From 40683658929e38905c87e72988c797180797501e Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 28 Jul 2011 08:34:27 -0700 Subject: make payload json serializable --- nova/notifier/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 98969fd3e..45105d00f 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -80,6 +80,12 @@ def notify(publisher_id, event_type, priority, payload): if priority not in log_levels: raise BadPriorityException( _('%s not in valid priorities' % priority)) + + # Ensure everything is JSON serializable. + for k, v in payload.iteritems(): + if not isinstance(v, (basestring, int, long, float)): + payload[k] = str(v) + driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), publisher_id=publisher_id, -- cgit From 1a97658a4bf7d0dab562f8b8c22a430f2da27f10 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 28 Jul 2011 08:44:01 -0700 Subject: unicode instead of str() --- nova/notifier/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 45105d00f..8eea2a032 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -84,7 +84,7 @@ def notify(publisher_id, event_type, priority, payload): # Ensure everything is JSON serializable. for k, v in payload.iteritems(): if not isinstance(v, (basestring, int, long, float)): - payload[k] = str(v) + payload[k] = unicode(v) driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), -- cgit From 7250fe0521ccb77e73563a0a36d23cac81956457 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 29 Jul 2011 07:05:27 -0700 Subject: added instance support to to_primitive and tests --- nova/notifier/api.py | 4 +-- nova/tests/test_utils.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 3 ++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 8eea2a032..70264efa8 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -82,9 +82,7 @@ def notify(publisher_id, event_type, priority, payload): _('%s not in valid priorities' % priority)) # Ensure everything is JSON serializable. - for k, v in payload.iteritems(): - if not isinstance(v, (basestring, int, long, float)): - payload[k] = unicode(v) + payload = utils.to_primitive(payload) driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index 0c359e981..3797478f3 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime import os import tempfile @@ -306,3 +307,73 @@ class IsUUIDLikeTestCase(test.TestCase): def test_non_uuid_string_passed(self): val = 'foo-fooo' self.assertUUIDLike(val, False) + + +class ToPrimitiveTestCase(test.TestCase): + def test_list(self): + self.assertEquals(utils.to_primitive([1, 2, 3]), [1, 2, 3]) + + def test_empty_list(self): + self.assertEquals(utils.to_primitive([]), []) + + def test_tuple(self): + self.assertEquals(utils.to_primitive((1, 2, 3)), [1, 2, 3]) + + def test_dict(self): + self.assertEquals(utils.to_primitive(dict(a=1, b=2, c=3)), + dict(a=1, b=2, c=3)) + + def test_empty_dict(self): + self.assertEquals(utils.to_primitive({}), {}) + + def test_datetime(self): + x = datetime.datetime(1,2,3,4,5,6,7) + self.assertEquals(utils.to_primitive(x), "0001-02-03 04:05:06.000007") + + def test_iter(self): + class IterClass(object): + def __init__(self): + self.data = [1, 2, 3, 4, 5] + self.index = 0 + + def __iter__(self): + return self + + def next(self): + if self.index == len(self.data): + raise StopIteration + self.index = self.index + 1 + return self.data[self.index - 1] + + x = IterClass() + self.assertEquals(utils.to_primitive(x), [1, 2, 3, 4, 5]) + + def test_iteritems(self): + class IterItemsClass(object): + def __init__(self): + self.data = dict(a=1, b=2, c=3).items() + self.index = 0 + + def __iter__(self): + return self + + def next(self): + if self.index == len(self.data): + raise StopIteration + self.index = self.index + 1 + return self.data[self.index - 1] + + x = IterItemsClass() + ordered = utils.to_primitive(x) + ordered.sort() + self.assertEquals(ordered, [['a', 1], ['b', 2], ['c', 3]]) + + def test_instance(self): + class MysteryClass(object): + a = 10 + + def __init__(self): + self.x = 1 + + x = MysteryClass() + self.assertEquals(utils.to_primitive(x), dict(x=1)) diff --git a/nova/utils.py b/nova/utils.py index 8784a227d..f54bf72ed 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -521,6 +521,9 @@ def to_primitive(value): return to_primitive(dict(value.iteritems())) elif hasattr(value, '__iter__'): return to_primitive(list(value)) + elif hasattr(value, '__dict__'): + # Class member variables not supported. + return to_primitive(value.__dict__) else: return value -- cgit From 51f4d4c2e0c7d9f066b328014aa955b150b62c3a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 29 Jul 2011 12:09:17 -0700 Subject: made the whole instance handling thing optional --- nova/notifier/api.py | 2 +- nova/tests/test_utils.py | 23 ++++++++++----- nova/utils.py | 77 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/nova/notifier/api.py b/nova/notifier/api.py index 70264efa8..e18f3e280 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -82,7 +82,7 @@ def notify(publisher_id, event_type, priority, payload): _('%s not in valid priorities' % priority)) # Ensure everything is JSON serializable. - payload = utils.to_primitive(payload) + payload = utils.to_primitive(payload, convert_instances=True) driver = utils.import_object(FLAGS.notification_driver) msg = dict(message_id=str(uuid.uuid4()), diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index 3797478f3..ec5098a37 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -318,16 +318,16 @@ class ToPrimitiveTestCase(test.TestCase): def test_tuple(self): self.assertEquals(utils.to_primitive((1, 2, 3)), [1, 2, 3]) - + def test_dict(self): - self.assertEquals(utils.to_primitive(dict(a=1, b=2, c=3)), + self.assertEquals(utils.to_primitive(dict(a=1, b=2, c=3)), dict(a=1, b=2, c=3)) def test_empty_dict(self): self.assertEquals(utils.to_primitive({}), {}) def test_datetime(self): - x = datetime.datetime(1,2,3,4,5,6,7) + x = datetime.datetime(1, 2, 3, 4, 5, 6, 7) self.assertEquals(utils.to_primitive(x), "0001-02-03 04:05:06.000007") def test_iter(self): @@ -335,7 +335,7 @@ class ToPrimitiveTestCase(test.TestCase): def __init__(self): self.data = [1, 2, 3, 4, 5] self.index = 0 - + def __iter__(self): return self @@ -347,13 +347,13 @@ class ToPrimitiveTestCase(test.TestCase): x = IterClass() self.assertEquals(utils.to_primitive(x), [1, 2, 3, 4, 5]) - + def test_iteritems(self): class IterItemsClass(object): def __init__(self): self.data = dict(a=1, b=2, c=3).items() self.index = 0 - + def __iter__(self): return self @@ -373,7 +373,14 @@ class ToPrimitiveTestCase(test.TestCase): a = 10 def __init__(self): - self.x = 1 + self.b = 1 x = MysteryClass() - self.assertEquals(utils.to_primitive(x), dict(x=1)) + self.assertEquals(utils.to_primitive(x, convert_instances=True), + dict(b=1)) + + self.assertEquals(utils.to_primitive(x), x) + + def test_typeerror(self): + x = bytearray # Class, not instance + self.assertEquals(utils.to_primitive(x), u"") diff --git a/nova/utils.py b/nova/utils.py index f54bf72ed..1a04c471c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -504,28 +504,61 @@ def utf8(value): return value -def to_primitive(value): - if type(value) is type([]) or type(value) is type((None,)): - o = [] - for v in value: - o.append(to_primitive(v)) - return o - elif type(value) is type({}): - o = {} - for k, v in value.iteritems(): - o[k] = to_primitive(v) - return o - elif isinstance(value, datetime.datetime): - return str(value) - elif hasattr(value, 'iteritems'): - return to_primitive(dict(value.iteritems())) - elif hasattr(value, '__iter__'): - return to_primitive(list(value)) - elif hasattr(value, '__dict__'): - # Class member variables not supported. - return to_primitive(value.__dict__) - else: - return value +def to_primitive(value, convert_instances=False, level=0): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + if inspect.isclass(value): + return unicode(value) + + if level > 3: + return [] + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + if type(value) is type([]) or type(value) is type((None,)): + o = [] + for v in value: + o.append(to_primitive(v, convert_instances=convert_instances, + level=level)) + return o + elif type(value) is type({}): + o = {} + for k, v in value.iteritems(): + o[k] = to_primitive(v, convert_instances=convert_instances, + level=level) + return o + elif isinstance(value, datetime.datetime): + return str(value) + elif hasattr(value, 'iteritems'): + return to_primitive(dict(value.iteritems()), + convert_instances=convert_instances, + level=level) + elif hasattr(value, '__iter__'): + return to_primitive(list(value), level) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return to_primitive(value.__dict__, + convert_instances=convert_instances, + level=level + 1) + else: + return value + except TypeError, e: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) def dumps(value): -- cgit