Add CRUD methods for tags to the EC2 API.

This is an incomplete implementation of the EC2 tags API.  In EC2, most
resources are able to be tagged.  See
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html

In openstack, the only currently 'taggable' resource is an instance, as
it has an instance_metadata table associated.  So far, only instance
tagging has been implemented, but it is relatively simple to extend this
to other resource types by creating the associated model and api calls.

Additionally, in EC2 searches, shell-style globs are allowed, eg:
fo* will match fo, foo, foobar
fo? will match foo

This has been left to do at a later date.

DocImpact: Adds new API calls: CreateTags, DeleteTags, DescribeTags
See:
http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-CreateTags.html
http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DeleteTags.html
http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeTags.html

Fixes: bug #1096821
Implements: blueprint ec2-tags-api

Change-Id: Idf1108f6a3476cabdbdb32ff41c00aa4bc2d9ffe
Signed-off-by: Stephen Gran <stephen.gran@guardian.co.uk>

8 files changed, 503 insertions, 0 deletions

diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py
index 0d06dde33..3d210404c 100644
--- a/nova/api/ec2/cloud.py
+++ b/nova/api/ec2/cloud.py
@@ -1174,6 +1174,10 @@ class CloudController(object):
             i['ipAddress'] = floating_ip or fixed_ip
             i['dnsName'] = i['publicDnsName'] or i['privateDnsName']
             i['keyName'] = instance['key_name']
+            i['tagSet'] = []
+            for k, v in self.compute_api.get_instance_metadata(
+                    context, instance).iteritems():
+                i['tagSet'].append({'key': k, 'value': v})
 
             if context.is_admin:
                 i['keyName'] = '%s (%s, %s)' % (i['keyName'],
@@ -1677,6 +1681,137 @@ class CloudController(object):
 
         return {'imageId': ec2_id}
 
+    def create_tags(self, context, **kwargs):
+        """Add tags to a resource
+
+        Returns True on success, error on failure.
+
+        :param context: context under which the method is called
+        """
+        resources = kwargs.get('resource_id', None)
+        tags = kwargs.get('tag', None)
+        if resources is None or tags is None:
+            raise exception.EC2APIError(_('resource_id and tag are required'))
+
+        if not isinstance(resources, (tuple, list, set)):
+            raise exception.EC2APIError(_('Expecting a list of resources'))
+
+        for r in resources:
+            if ec2utils.resource_type_from_id(context, r) != 'instance':
+                raise exception.EC2APIError(_('Only instances implemented'))
+
+        if not isinstance(tags, (tuple, list, set)):
+            raise exception.EC2APIError(_('Expecting a list of tagSets'))
+
+        metadata = {}
+        for tag in tags:
+            if not isinstance(tag, dict):
+                raise exception.EC2APIError(_
+                        ('Expecting tagSet to be key/value pairs'))
+
+            key = tag.get('key', None)
+            val = tag.get('value', None)
+
+            if key is None or val is None:
+                raise exception.EC2APIError(_
+                        ('Expecting both key and value to be set'))
+
+            metadata[key] = val
+
+        for ec2_id in resources:
+            instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id)
+            instance = self.compute_api.get(context, instance_uuid)
+            self.compute_api.update_instance_metadata(context,
+                instance, metadata)
+
+        return True
+
+    def delete_tags(self, context, **kwargs):
+        """Delete tags
+
+        Returns True on success, error on failure.
+
+        :param context: context under which the method is called
+        """
+        resources = kwargs.get('resource_id', None)
+        tags = kwargs.get('tag', None)
+        if resources is None or tags is None:
+            raise exception.EC2APIError(_('resource_id and tag are required'))
+
+        if not isinstance(resources, (tuple, list, set)):
+            raise exception.EC2APIError(_('Expecting a list of resources'))
+
+        for r in resources:
+            if ec2utils.resource_type_from_id(context, r) != 'instance':
+                raise exception.EC2APIError(_('Only instances implemented'))
+
+        if not isinstance(tags, (tuple, list, set)):
+            raise exception.EC2APIError(_('Expecting a list of tagSets'))
+
+        for ec2_id in resources:
+            instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id)
+            instance = self.compute_api.get(context, instance_uuid)
+            for tag in tags:
+                if not isinstance(tag, dict):
+                    raise exception.EC2APIError(_
+                            ('Expecting tagSet to be key/value pairs'))
+
+                key = tag.get('key', None)
+                if key is None:
+                    raise exception.EC2APIError(_('Expecting key to be set'))
+
+                self.compute_api.delete_instance_metadata(context,
+                        instance, key)
+
+        return True
+
+    def describe_tags(self, context, **kwargs):
+        """List tags
+
+        Returns a dict with a single key 'tagSet' on success, error on failure.
+
+        :param context: context under which the method is called
+        """
+        filters = kwargs.get('filter', None)
+
+        search_filts = []
+        if filters:
+            for filter_block in filters:
+                key_name = filter_block.get('name', None)
+                val = filter_block.get('value', None)
+                if val:
+                    if isinstance(val, dict):
+                        val = val.values()
+                    if not isinstance(val, (tuple, list, set)):
+                        val = (val,)
+                if key_name:
+                    search_block = {}
+                    if key_name == 'resource_id':
+                        search_block['resource_id'] = []
+                        for res_id in val:
+                            search_block['resource_id'].append(
+                                ec2utils.ec2_inst_id_to_uuid(context, res_id))
+                    elif key_name in ['key', 'value']:
+                        search_block[key_name] = val
+                    elif key_name == 'resource_type':
+                        for res_type in val:
+                            if res_type != 'instance':
+                                raise exception.EC2APIError(_
+                                        ('Only instances implemented'))
+                            search_block[key_name] = 'instance'
+                    if len(search_block.keys()) > 0:
+                        search_filts.append(search_block)
+        ts = []
+        for tag in self.compute_api.get_all_instance_metadata(context,
+                                                              search_filts):
+            ts.append({
+                'resource_id': ec2utils.id_to_ec2_inst_id(tag['instance_id']),
+                'resource_type': 'instance',
+                'key': tag['key'],
+                'value': tag['value']
+            })
+        return {"tagSet": ts}
+
 
 class EC2SecurityGroupExceptions(object):
     @staticmethod
diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py
index 1f4b8d6ee..8a73533c6 100644
--- a/nova/api/ec2/ec2utils.py
+++ b/nova/api/ec2/ec2utils.py
@@ -72,6 +72,31 @@ def image_type(image_type):
     return image_type
 
 
+def resource_type_from_id(context, resource_id):
+    """Get resource type by ID
+
+    Returns a string representation of the Amazon resource type, if known.
+    Returns None on failure.
+
+    :param context: context under which the method is called
+    :param resource_id: resource_id to evaluate
+    """
+
+    known_types = {
+        'i': 'instance',
+        'r': 'reservation',
+        'vol': 'volume',
+        'snap': 'snapshot',
+        'ami': 'image',
+        'aki': 'image',
+        'ari': 'image'
+    }
+
+    type_marker = resource_id.split('-')[0]
+
+    return known_types.get(type_marker)
+
+
 @memoize
 def id_to_glance_id(context, image_id):
     """Convert an internal (db) id to a glance id."""
diff --git a/nova/compute/api.py b/nova/compute/api.py
index 0d915bfc9..faaf13c92 100644
--- a/nova/compute/api.py
+++ b/nova/compute/api.py
@@ -2363,6 +2363,11 @@ class API(base.Base):
         return dict(rv.iteritems())
 
     @wrap_check_policy
+    def get_all_instance_metadata(self, context, search_filts):
+        """Get all metadata."""
+        return self.db.instance_metadata_get_all(context, search_filts)
+
+    @wrap_check_policy
     @check_instance_lock
     @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.PAUSED,
                                     vm_states.SUSPENDED, vm_states.STOPPED],
diff --git a/nova/db/api.py b/nova/db/api.py
index ae7b913cf..ba3957be6 100644
--- a/nova/db/api.py
+++ b/nova/db/api.py
@@ -1341,6 +1341,11 @@ def cell_get_all(context):
 ####################
 
 
+def instance_metadata_get_all(context, search_filts):
+    """Get all metadata for an instance."""
+    return IMPL.instance_metadata_get_all(context, search_filts)
+
+
 def instance_metadata_get(context, instance_uuid):
     """Get all metadata for an instance."""
     return IMPL.instance_metadata_get(context, instance_uuid)
diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py
index 081d6ede8..a6b40a989 100644
--- a/nova/db/sqlalchemy/api.py
+++ b/nova/db/sqlalchemy/api.py
@@ -3725,6 +3725,55 @@ def _instance_metadata_get_query(context, instance_uuid, session=None):
                     filter_by(instance_uuid=instance_uuid)
 
 
+def _instance_metadata_get_all_query(context, session=None,
+                                     read_deleted='no', search_filts=[]):
+
+    or_query = None
+    query = model_query(context, models.InstanceMetadata, session=session,
+                        read_deleted=read_deleted)
+
+    # We want to incrementally build an OR query out of the search filters.
+    # So:
+    # {'filter':
+    #     [{'resource_id': 'i-0000001'}],
+    #     [{'key': 'foo', 'value': 'bar'}]}
+    # Should produce:
+    # AND ((instance_metadata.uuid IN ('1')) OR
+    # (instance_metadata.key IN ('foo')) OR
+    # (instance_metadata.value IN ('bar')))
+
+    def make_tuple(item):
+        if isinstance(item, dict):
+            item = item.values()
+        if not isinstance(item, (tuple, list, set)):
+            item = (item,)
+        return item
+
+    for search_filt in search_filts:
+        subq = None
+
+        if search_filt.get('resource_id'):
+            uuid = make_tuple(search_filt['resource_id'])
+            subq = models.InstanceMetadata.instance_uuid.in_(uuid)
+        elif search_filt.get('key'):
+            key = make_tuple(search_filt['key'])
+            subq = models.InstanceMetadata.key.in_(key)
+        elif search_filt.get('value'):
+            value = make_tuple(search_filt['value'])
+            subq = models.InstanceMetadata.value.in_(value)
+
+        if subq is not None:
+            if or_query is None:
+                or_query = subq
+            else:
+                or_query = or_(or_query, subq)
+
+    if or_query is not None:
+        query = query.filter(or_query)
+
+    return query
+
+
 @require_context
 def instance_metadata_get(context, instance_uuid, session=None):
     rows = _instance_metadata_get_query(context, instance_uuid,
@@ -3738,6 +3787,18 @@ def instance_metadata_get(context, instance_uuid, session=None):
 
 
 @require_context
+def instance_metadata_get_all(context, search_filts=[], read_deleted="no"):
+    rows = _instance_metadata_get_all_query(context,
+                                       read_deleted=read_deleted,
+                                       search_filts=search_filts).all()
+
+    return [{'key': row['key'],
+             'value': row['value'],
+             'instance_id': row['instance_uuid']}
+             for row in rows]
+
+
+@require_context
 def instance_metadata_delete(context, instance_uuid, key):
     _instance_metadata_get_query(context, instance_uuid).\
         filter_by(key=key).\
diff --git a/nova/tests/api/ec2/test_cloud.py b/nova/tests/api/ec2/test_cloud.py
index 322bbb465..5ed143a2c 100644
--- a/nova/tests/api/ec2/test_cloud.py
+++ b/nova/tests/api/ec2/test_cloud.py
@@ -35,6 +35,7 @@ from nova.api.metadata import password
 from nova.compute import api as compute_api
 from nova.compute import instance_types
 from nova.compute import power_state
+from nova.compute import rpcapi as compute_rpcapi
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova import context
@@ -806,6 +807,7 @@ class CloudTestCase(test.TestCase):
         self.assertEqual(instance['publicDnsName'], '1.2.3.4')
         self.assertEqual(instance['ipAddress'], '1.2.3.4')
         self.assertEqual(instance['dnsName'], '1.2.3.4')
+        self.assertEqual(instance['tagSet'], [])
         self.assertEqual(instance['privateDnsName'], 'server-4321')
         self.assertEqual(instance['privateIpAddress'], '192.168.0.3')
         self.assertEqual(instance['dnsNameV6'],
@@ -2219,6 +2221,255 @@ class CloudTestCase(test.TestCase):
         test_dia_iisb('stop', image_id='ami-5')
         test_dia_iisb('stop', image_id='ami-6')
 
+    def test_create_delete_tags(self):
+
+        # We need to stub network calls
+        self._stub_instance_get_with_fixed_ips('get_all')
+        self._stub_instance_get_with_fixed_ips('get')
+
+        # We need to stub out the MQ call - it won't succeed.  We do want
+        # to check that the method is called, though
+        meta_changes = [None]
+
+        def fake_change_instance_metadata(inst, ctxt, diff, instance=None,
+                                          instance_uuid=None):
+            meta_changes[0] = diff
+
+        self.stubs.Set(compute_rpcapi.ComputeAPI, 'change_instance_metadata',
+                       fake_change_instance_metadata)
+
+        # Create a test image
+        image_uuid = 'cedef40a-ed67-4d10-800e-17455edce175'
+        inst1_kwargs = {
+                'reservation_id': 'a',
+                'image_ref': image_uuid,
+                'instance_type_id': 1,
+                'vm_state': 'active',
+                'hostname': 'server-1111',
+                'created_at': datetime.datetime(2012, 5, 1, 1, 1, 1)
+        }
+
+        inst1 = db.instance_create(self.context, inst1_kwargs)
+        ec2_id = ec2utils.id_to_ec2_inst_id(inst1['uuid'])
+
+        # Create some tags
+        md = {'key': 'foo', 'value': 'bar'}
+        md_result = {'foo': 'bar'}
+        self.cloud.create_tags(self.context, resource_id=[ec2_id],
+                tag=[md])
+
+        metadata = self.cloud.compute_api.get_instance_metadata(self.context,
+                inst1)
+        self.assertEqual(metadata, md_result)
+        self.assertEqual(meta_changes, [{'foo': ['+', 'bar']}])
+
+        # Delete them
+        self.cloud.delete_tags(self.context, resource_id=[ec2_id],
+                tag=[{'key': 'foo', 'value': 'bar'}])
+
+        metadata = self.cloud.compute_api.get_instance_metadata(self.context,
+                inst1)
+        self.assertEqual(metadata, {})
+        self.assertEqual(meta_changes, [{'foo': ['-']}])
+
+    def test_describe_tags(self):
+        # We need to stub network calls
+        self._stub_instance_get_with_fixed_ips('get_all')
+        self._stub_instance_get_with_fixed_ips('get')
+
+        # We need to stub out the MQ call - it won't succeed.  We do want
+        # to check that the method is called, though
+        meta_changes = [None]
+
+        def fake_change_instance_metadata(inst, ctxt, diff, instance=None,
+                                          instance_uuid=None):
+            meta_changes[0] = diff
+
+        self.stubs.Set(compute_rpcapi.ComputeAPI, 'change_instance_metadata',
+                       fake_change_instance_metadata)
+
+        # Create some test images
+        image_uuid = 'cedef40a-ed67-4d10-800e-17455edce175'
+        inst1_kwargs = {
+                'reservation_id': 'a',
+                'image_ref': image_uuid,
+                'instance_type_id': 1,
+                'vm_state': 'active',
+                'hostname': 'server-1111',
+                'created_at': datetime.datetime(2012, 5, 1, 1, 1, 1)
+        }
+
+        inst2_kwargs = {
+                'reservation_id': 'b',
+                'image_ref': image_uuid,
+                'instance_type_id': 1,
+                'vm_state': 'active',
+                'hostname': 'server-1112',
+                'created_at': datetime.datetime(2012, 5, 1, 1, 1, 2)
+        }
+
+        inst1 = db.instance_create(self.context, inst1_kwargs)
+        ec2_id1 = ec2utils.id_to_ec2_inst_id(inst1['uuid'])
+
+        inst2 = db.instance_create(self.context, inst2_kwargs)
+        ec2_id2 = ec2utils.id_to_ec2_inst_id(inst2['uuid'])
+
+        # Create some tags
+        # We get one overlapping pair, and each has a different key value pair
+        # inst1 : {'foo': 'bar', 'bax': 'wibble'}
+        # inst1 : {'foo': 'bar', 'baz': 'quux'}
+
+        md = {'key': 'foo', 'value': 'bar'}
+        md_result = {'foo': 'bar'}
+        self.cloud.create_tags(self.context, resource_id=[ec2_id1, ec2_id2],
+                tag=[md])
+
+        self.assertEqual(meta_changes, [{'foo': ['+', 'bar']}])
+
+        metadata = self.cloud.compute_api.get_instance_metadata(self.context,
+                inst1)
+        self.assertEqual(metadata, md_result)
+
+        metadata = self.cloud.compute_api.get_instance_metadata(self.context,
+                inst2)
+        self.assertEqual(metadata, md_result)
+
+        md2 = {'key': 'baz', 'value': 'quux'}
+        md2_result = {'baz': 'quux'}
+        md2_result.update(md_result)
+        self.cloud.create_tags(self.context, resource_id=[ec2_id2],
+                tag=[md2])
+
+        self.assertEqual(meta_changes, [{'baz': ['+', 'quux']}])
+
+        metadata = self.cloud.compute_api.get_instance_metadata(self.context,
+                inst2)
+        self.assertEqual(metadata, md2_result)
+
+        md3 = {'key': 'bax', 'value': 'wibble'}
+        md3_result = {'bax': 'wibble'}
+        md3_result.update(md_result)
+        self.cloud.create_tags(self.context, resource_id=[ec2_id1],
+                tag=[md3])
+
+        self.assertEqual(meta_changes, [{'bax': ['+', 'wibble']}])
+
+        metadata = self.cloud.compute_api.get_instance_metadata(self.context,
+                inst1)
+        self.assertEqual(metadata, md3_result)
+
+        inst1_key_foo = {'key': u'foo', 'resource_id': 'i-00000001',
+                         'resource_type': 'instance', 'value': u'bar'}
+        inst1_key_bax = {'key': u'bax', 'resource_id': 'i-00000001',
+                         'resource_type': 'instance', 'value': u'wibble'}
+        inst2_key_foo = {'key': u'foo', 'resource_id': 'i-00000002',
+                         'resource_type': 'instance', 'value': u'bar'}
+        inst2_key_baz = {'key': u'baz', 'resource_id': 'i-00000002',
+                         'resource_type': 'instance', 'value': u'quux'}
+
+        # We should be able to search by:
+        # No filter
+        tags = self.cloud.describe_tags(self.context)['tagSet']
+        self.assertEqual(tags, [inst1_key_foo, inst2_key_foo,
+                                inst2_key_baz, inst1_key_bax])
+
+        # Resource ID
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'resource_id',
+                         'value': [ec2_id1]}])['tagSet']
+        self.assertEqual(tags, [inst1_key_foo, inst1_key_bax])
+
+        # Resource Type
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'resource_type',
+                         'value': ['instance']}])['tagSet']
+        self.assertEqual(tags, [inst1_key_foo, inst2_key_foo,
+                                inst2_key_baz, inst1_key_bax])
+
+        # Key, either bare or with wildcards
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'key',
+                         'value': ['foo']}])['tagSet']
+        self.assertEqual(tags, [inst1_key_foo, inst2_key_foo])
+
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'key',
+                         'value': ['baz']}])['tagSet']
+        self.assertEqual(tags, [inst2_key_baz])
+
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'key',
+                         'value': ['ba?']}])['tagSet']
+        self.assertEqual(tags, [])
+
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'key',
+                         'value': ['b*']}])['tagSet']
+        self.assertEqual(tags, [])
+
+        # Value, either bare or with wildcards
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'value',
+                         'value': ['bar']}])['tagSet']
+        self.assertEqual(tags, [inst1_key_foo, inst2_key_foo])
+
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'value',
+                         'value': ['wi*']}])['tagSet']
+        self.assertEqual(tags, [])
+
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'value',
+                         'value': ['quu?']}])['tagSet']
+        self.assertEqual(tags, [])
+
+        # Multiple values
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'key',
+                         'value': ['baz', 'bax']}])['tagSet']
+        self.assertEqual(tags, [inst2_key_baz, inst1_key_bax])
+
+        # Multiple filters
+        tags = self.cloud.describe_tags(self.context,
+                filter=[{'name': 'key',
+                         'value': ['baz']},
+                        {'name': 'value',
+                         'value': ['wibble']}])['tagSet']
+        self.assertEqual(tags, [inst2_key_baz, inst1_key_bax])
+
+        # And we should fail on supported resource types
+        self.assertRaises(exception.EC2APIError,
+                          self.cloud.describe_tags,
+                          self.context,
+                          filter=[{'name': 'resource_type',
+                                   'value': ['instance', 'volume']}])
+
+    def test_resource_type_from_id(self):
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'i-12345'),
+                'instance')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'r-12345'),
+                'reservation')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'vol-12345'),
+                'volume')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'snap-12345'),
+                'snapshot')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'ami-12345'),
+                'image')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'ari-12345'),
+                'image')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'aki-12345'),
+                'image')
+        self.assertEqual(
+                ec2utils.resource_type_from_id(self.context, 'x-12345'),
+                None)
+
 
 class CloudTestCaseQuantumProxy(test.TestCase):
     def setUp(self):
diff --git a/nova/tests/fake_policy.py b/nova/tests/fake_policy.py
index 94380058d..d47ff629a 100644
--- a/nova/tests/fake_policy.py
+++ b/nova/tests/fake_policy.py
@@ -31,6 +31,7 @@ policy_data = """
     "compute:update": "",
 
     "compute:get_instance_metadata": "",
+    "compute:get_all_instance_metadata": "",
     "compute:update_instance_metadata": "",
     "compute:delete_instance_metadata": "",
 
diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py
index 19eb4d271..b5896399a 100644
--- a/nova/tests/test_db_api.py
+++ b/nova/tests/test_db_api.py
@@ -97,6 +97,26 @@ class DbApiTestCase(DbTestCase):
 
         self.flags(osapi_compute_unique_server_name_scope=None)
 
+    def test_instance_metadata_get_all_query(self):
+        self.create_instances_with_args(metadata={'foo': 'bar'})
+        self.create_instances_with_args(metadata={'baz': 'quux'})
+
+        result = db.instance_metadata_get_all(self.context, [])
+        self.assertEqual(2, len(result))
+
+        result = db.instance_metadata_get_all(self.context,
+                                              [{'key': 'foo'}])
+        self.assertEqual(1, len(result))
+
+        result = db.instance_metadata_get_all(self.context,
+                                              [{'value': 'quux'}])
+        self.assertEqual(1, len(result))
+
+        result = db.instance_metadata_get_all(self.context,
+                                              [{'value': 'quux'},
+                                               {'key': 'foo'}])
+        self.assertEqual(2, len(result))
+
     def test_ec2_ids_not_found_are_printable(self):
         def check_exc_format(method):
             try: