metadata speed - revert logic changes, just caching

 * reverts most of https://review.openstack.org/5042
 * keep changes that cache the results so subsequent calls are fast

 fixes bug 954377

Change-Id: Ia047f1d04dc360dd657d4d80d3dff8c8101635cd

1 files changed, 71 insertions, 152 deletions

diff --git a/nova/api/metadata/handler.py b/nova/api/metadata/handler.py
index 598dbf32c..1ad8a5413 100644
--- a/nova/api/metadata/handler.py
+++ b/nova/api/metadata/handler.py
@@ -19,7 +19,6 @@
 """Metadata request handler."""
 
 import base64
-import collections
 
 import webob.dec
 import webob.exc
@@ -33,12 +32,9 @@ from nova import exception
 from nova import flags
 from nova import log as logging
 from nova import network
-from nova.rpc import common as rpc_common
-from nova import utils
 from nova import volume
 from nova import wsgi
 
-
 LOG = logging.getLogger(__name__)
 FLAGS = flags.FLAGS
 flags.DECLARE('use_forwarded_for', 'nova.api.auth')
@@ -83,36 +79,9 @@ class MetadataRequestHandler(wsgi.Application):
         self.compute_api = compute.API(
                 network_api=self.network_api,
                 volume_api=volume.API())
-
-        self.metadata_mapper = {
-            'user-data': self.user_data,
-            'meta-data': {
-                'instance-id': self.instance_id,
-                'instance-type': self.instance_type,
-                'ami-id': self.ami_id,
-                'kernel-id': self.kernel_id,
-                'ramdisk-id': self.ramdisk_id,
-                'block-device-mapping': self.block_device_mapping,
-                'hostname': self.hostname,
-                'public-hostname': self.hostname,
-                'local-hostname': self.hostname,
-                'local-ipv4': self.local_ipv4,
-                'public-ipv4': self.public_ipv4,
-                'security-groups': self.security_groups,
-                'public-keys': self.public_keys,
-                'ami-launch-index': self.ami_launch_index,
-                'reservation-id': self.reservation_id,
-                'placement': self.placement,
-                'instance-action': self.instance_action,
-                'ami-manifest-path': self.ami_manifest_path,
-            }
-        }
-
         self._cache = memcache.Client(FLAGS.memcached_servers, debug=0)
 
-    def _format_instance_mapping(self, instance_ref):
-        ctxt = context.get_admin_context()
-
+    def _format_instance_mapping(self, ctxt, instance_ref):
         root_device_name = instance_ref['root_device_name']
         if root_device_name is None:
             return _DEFAULT_MAPPINGS
@@ -158,127 +127,84 @@ class MetadataRequestHandler(wsgi.Application):
 
         return mappings
 
-    def get_instance(self, address):
-        """get instance_ref for a given fixed_ip, raising
-        exception.NotFound if unable to find instance
-
-        this will attempt to use memcache or fake memcache (an in-memory
-        cache) to remove the DB query + RPC query for batched calls (eg
-        cloud-init making dozens of queries on boot)
-        """
-
-        cache_key = 'metadata-%s' % address
-        instance_dict = self._cache.get(cache_key)
-        if instance_dict:
-            return instance_dict
-
+    def get_metadata(self, address):
         if not address:
             raise exception.FixedIpNotFoundForAddress(address=address)
 
-        ctxt = context.get_admin_context()
+        cache_key = 'metadata-%s' % address
+        data = self._cache.get(cache_key)
+        if data:
+            return data
 
+        ctxt = context.get_admin_context()
         try:
             fixed_ip = self.network_api.get_fixed_ip_by_address(ctxt, address)
-        except rpc_common.RemoteError:
-            raise exception.FixedIpNotFoundForAddress(address=address)
-        instance_ref = db.instance_get(ctxt, fixed_ip['instance_id'])
-        instance_dict = utils.to_primitive(instance_ref)
-
-        self._cache.set(cache_key, instance_dict, 15)
-
-        return instance_dict
-
-    def user_data(self, address):
-        instance_ref = self.get_instance(address)
-        return base64.b64decode(instance_ref['user_data'])
-
-    def instance_id(self, address):
-        instance_ref = self.get_instance(address)
-        return ec2utils.id_to_ec2_id(instance_ref['id'])
+            instance_ref = db.instance_get(ctxt, fixed_ip['instance_id'])
+        except exception.NotFound:
+            return None
 
-    def instance_type(self, address):
-        instance_ref = self.get_instance(address)
-        return instance_ref['instance_type']['name']
+        hostname = "%s.%s" % (instance_ref['hostname'], FLAGS.dhcp_domain)
+        host = instance_ref['host']
+        services = db.service_get_all_by_host(ctxt.elevated(), host)
+        availability_zone = ec2utils.get_availability_zone_by_host(services,
+                                                                   host)
 
-    def ami_id(self, address):
-        instance_ref = self.get_instance(address)
-        image_id = instance_ref['image_ref']
-        ctxt = context.get_admin_context()
-        return ec2utils.glance_id_to_ec2_id(ctxt, image_id)
-
-    def kernel_id(self, address):
-        instance_ref = self.get_instance(address)
-        kernel_id = instance_ref.get('kernel_id')
-        if kernel_id:
-            image_type = ec2utils.image_type('kernel')
-            ctxt = context.get_admin_context()
-            return ec2utils.glance_id_to_ec2_id(ctxt, kernel_id, image_type)
-
-    def ramdisk_id(self, address):
-        instance_ref = self.get_instance(address)
-        ramdisk_id = instance_ref.get('ramdisk_id')
-        if ramdisk_id:
-            image_type = ec2utils.image_type('ramdisk')
-            ctxt = context.get_admin_context()
-            return ec2utils.image_ec2_id(ctxt, ramdisk_id, image_type)
-
-    def ami_launch_index(self, address):
-        instance_ref = self.get_instance(address)
-        return instance_ref['launch_index']
-
-    def block_device_mapping(self, address):
-        instance_ref = self.get_instance(address)
-        return self._format_instance_mapping(instance_ref)
-
-    def hostname(self, address):
-        instance_ref = self.get_instance(address)
-        return "%s.%s" % (instance_ref['hostname'], FLAGS.dhcp_domain)
-
-    def local_ipv4(self, address):
-        return address
-
-    def public_ipv4(self, address):
-        instance_ref = self.get_instance(address)
-        ctxt = context.get_admin_context()
         ip_info = ec2utils.get_ip_info_for_instance(ctxt, instance_ref)
         floating_ips = ip_info['floating_ips']
         floating_ip = floating_ips and floating_ips[0] or ''
-        return floating_ip
-
-    def reservation_id(self, address):
-        instance_ref = self.get_instance(address)
-        return instance_ref['reservation_id']
 
-    def placement(self, address):
-        instance_ref = self.get_instance(address)
-        host = instance_ref['host']
-        ctxt = context.get_admin_context()
-        # note(ja): original code had ctx.elevated?
-        services = db.service_get_all_by_host(ctxt, host)
-        zone = ec2utils.get_availability_zone_by_host(services, host)
-        return {'availability-zone': zone}
-
-    def security_groups(self, address):
-        instance_ref = self.get_instance(address)
+        ec2_id = ec2utils.id_to_ec2_id(instance_ref['id'])
+        image_id = instance_ref['image_ref']
         ctxt = context.get_admin_context()
-        groups = db.security_group_get_by_instance(ctxt,
-                                                   instance_ref['id'])
-        return [g['name'] for g in groups]
+        image_ec2_id = ec2utils.glance_id_to_ec2_id(ctxt, image_id)
+        security_groups = db.security_group_get_by_instance(ctxt,
+                                                            instance_ref['id'])
+        security_groups = [x['name'] for x in security_groups]
+        mappings = self._format_instance_mapping(ctxt, instance_ref)
+        data = {
+            'user-data': base64.b64decode(instance_ref['user_data']),
+            'meta-data': {
+                'ami-id': image_ec2_id,
+                'ami-launch-index': instance_ref['launch_index'],
+                'ami-manifest-path': 'FIXME',
+                'block-device-mapping': mappings,
+                'hostname': hostname,
+                'instance-action': 'none',
+                'instance-id': ec2_id,
+                'instance-type': instance_ref['instance_type']['name'],
+                'local-hostname': hostname,
+                'local-ipv4': address,
+                'placement': {'availability-zone': availability_zone},
+                'public-hostname': hostname,
+                'public-ipv4': floating_ip,
+                'reservation-id': instance_ref['reservation_id'],
+                'security-groups': security_groups}}
 
-    def public_keys(self, address):
-        instance_ref = self.get_instance(address)
         # public-keys should be in meta-data only if user specified one
         if instance_ref['key_name']:
-            return {'0': {'_name': instance_ref['key_name'],
-                          'openssh-key': instance_ref['key_data']}}
-
-    def ami_manifest_path(self, address):
-        return 'Not Implemented'
+            data['meta-data']['public-keys'] = {
+                '0': {'_name': instance_ref['key_name'],
+                      'openssh-key': instance_ref['key_data']}}
+
+        for image_type in ['kernel', 'ramdisk']:
+            if instance_ref.get('%s_id' % image_type):
+                image_id = instance_ref['%s_id' % image_type]
+                image_type = ec2utils.image_type(image_type)
+                ec2_id = ec2utils.glance_id_to_ec2_id(ctxt,
+                                                      image_id,
+                                                      image_type)
+                data['meta-data']['%s-id' % image_type] = ec2_id
+
+        if False:  # TODO(vish): store ancestor ids
+            data['ancestor-ami-ids'] = []
+        if False:  # TODO(vish): store product codes
+            data['product-codes'] = []
+
+        self._cache.set(cache_key, data, 15)
 
-    def instance_action(self, address):
-        return 'none'
+        return data
 
-    def format_data(self, data):
+    def print_data(self, data):
         if isinstance(data, dict):
             output = ''
             for key in data:
@@ -298,21 +224,15 @@ class MetadataRequestHandler(wsgi.Application):
         else:
             return str(data)
 
-    def lookup(self, path, address):
+    def lookup(self, path, data):
         items = path.split('/')
-        data = self.metadata_mapper
         for item in items:
             if item:
                 if not isinstance(data, dict):
-                    # FIXME(ja): should we check that we are at the end
-                    # of the path as well before we just return?
                     return data
                 if not item in data:
                     return None
                 data = data[item]
-                if isinstance(data, collections.Callable):
-                    # lazy evaluation
-                    data = data(address)
         return data
 
     @webob.dec.wsgify(RequestClass=wsgi.Request)
@@ -320,20 +240,19 @@ class MetadataRequestHandler(wsgi.Application):
         remote_address = req.remote_addr
         if FLAGS.use_forwarded_for:
             remote_address = req.headers.get('X-Forwarded-For', remote_address)
-
         try:
-            data = self.lookup(req.path_info,
-                               remote_address)
-        except (exception.NotFound, exception.FixedIpNotFoundForAddress):
-            LOG.error(_('Failed to get metadata for ip: %s'), remote_address)
-            return webob.exc.HTTPNotFound()
-        except:
+            meta_data = self.get_metadata(remote_address)
+        except Exception:
             LOG.exception(_('Failed to get metadata for ip: %s'),
                           remote_address)
             msg = _('An unknown error has occurred. '
                     'Please try your request again.')
-            return webob.exc.HTTPInternalServerError(explanation=unicode(msg))
-
+            exc = webob.exc.HTTPInternalServerError(explanation=unicode(msg))
+            return exc
+        if meta_data is None:
+            LOG.error(_('Failed to get metadata for ip: %s'), remote_address)
+            raise webob.exc.HTTPNotFound()
+        data = self.lookup(req.path_info, meta_data)
         if data is None:
             raise webob.exc.HTTPNotFound()
-        return self.format_data(data)
+        return self.print_data(data)