From 4c8dbf81d9a9f27faeb19271b774ea4f130e516c Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Wed, 14 Mar 2012 13:20:07 -0700 Subject: metadata speed - revert logic changes, just caching * reverts most of https://review.openstack.org/5042 * keep changes that cache the results so subsequent calls are fast fixes bug 954377 Change-Id: Ia047f1d04dc360dd657d4d80d3dff8c8101635cd --- nova/api/metadata/handler.py | 223 ++++++++++++++----------------------------- 1 file changed, 71 insertions(+), 152 deletions(-) (limited to 'nova/api') diff --git a/nova/api/metadata/handler.py b/nova/api/metadata/handler.py index 598dbf32c..1ad8a5413 100644 --- a/nova/api/metadata/handler.py +++ b/nova/api/metadata/handler.py @@ -19,7 +19,6 @@ """Metadata request handler.""" import base64 -import collections import webob.dec import webob.exc @@ -33,12 +32,9 @@ from nova import exception from nova import flags from nova import log as logging from nova import network -from nova.rpc import common as rpc_common -from nova import utils from nova import volume from nova import wsgi - LOG = logging.getLogger(__name__) FLAGS = flags.FLAGS flags.DECLARE('use_forwarded_for', 'nova.api.auth') @@ -83,36 +79,9 @@ class MetadataRequestHandler(wsgi.Application): self.compute_api = compute.API( network_api=self.network_api, volume_api=volume.API()) - - self.metadata_mapper = { - 'user-data': self.user_data, - 'meta-data': { - 'instance-id': self.instance_id, - 'instance-type': self.instance_type, - 'ami-id': self.ami_id, - 'kernel-id': self.kernel_id, - 'ramdisk-id': self.ramdisk_id, - 'block-device-mapping': self.block_device_mapping, - 'hostname': self.hostname, - 'public-hostname': self.hostname, - 'local-hostname': self.hostname, - 'local-ipv4': self.local_ipv4, - 'public-ipv4': self.public_ipv4, - 'security-groups': self.security_groups, - 'public-keys': self.public_keys, - 'ami-launch-index': self.ami_launch_index, - 'reservation-id': self.reservation_id, - 'placement': self.placement, - 'instance-action': self.instance_action, - 'ami-manifest-path': self.ami_manifest_path, - } - } - self._cache = memcache.Client(FLAGS.memcached_servers, debug=0) - def _format_instance_mapping(self, instance_ref): - ctxt = context.get_admin_context() - + def _format_instance_mapping(self, ctxt, instance_ref): root_device_name = instance_ref['root_device_name'] if root_device_name is None: return _DEFAULT_MAPPINGS @@ -158,127 +127,84 @@ class MetadataRequestHandler(wsgi.Application): return mappings - def get_instance(self, address): - """get instance_ref for a given fixed_ip, raising - exception.NotFound if unable to find instance - - this will attempt to use memcache or fake memcache (an in-memory - cache) to remove the DB query + RPC query for batched calls (eg - cloud-init making dozens of queries on boot) - """ - - cache_key = 'metadata-%s' % address - instance_dict = self._cache.get(cache_key) - if instance_dict: - return instance_dict - + def get_metadata(self, address): if not address: raise exception.FixedIpNotFoundForAddress(address=address) - ctxt = context.get_admin_context() + cache_key = 'metadata-%s' % address + data = self._cache.get(cache_key) + if data: + return data + ctxt = context.get_admin_context() try: fixed_ip = self.network_api.get_fixed_ip_by_address(ctxt, address) - except rpc_common.RemoteError: - raise exception.FixedIpNotFoundForAddress(address=address) - instance_ref = db.instance_get(ctxt, fixed_ip['instance_id']) - instance_dict = utils.to_primitive(instance_ref) - - self._cache.set(cache_key, instance_dict, 15) - - return instance_dict - - def user_data(self, address): - instance_ref = self.get_instance(address) - return base64.b64decode(instance_ref['user_data']) - - def instance_id(self, address): - instance_ref = self.get_instance(address) - return ec2utils.id_to_ec2_id(instance_ref['id']) + instance_ref = db.instance_get(ctxt, fixed_ip['instance_id']) + except exception.NotFound: + return None - def instance_type(self, address): - instance_ref = self.get_instance(address) - return instance_ref['instance_type']['name'] + hostname = "%s.%s" % (instance_ref['hostname'], FLAGS.dhcp_domain) + host = instance_ref['host'] + services = db.service_get_all_by_host(ctxt.elevated(), host) + availability_zone = ec2utils.get_availability_zone_by_host(services, + host) - def ami_id(self, address): - instance_ref = self.get_instance(address) - image_id = instance_ref['image_ref'] - ctxt = context.get_admin_context() - return ec2utils.glance_id_to_ec2_id(ctxt, image_id) - - def kernel_id(self, address): - instance_ref = self.get_instance(address) - kernel_id = instance_ref.get('kernel_id') - if kernel_id: - image_type = ec2utils.image_type('kernel') - ctxt = context.get_admin_context() - return ec2utils.glance_id_to_ec2_id(ctxt, kernel_id, image_type) - - def ramdisk_id(self, address): - instance_ref = self.get_instance(address) - ramdisk_id = instance_ref.get('ramdisk_id') - if ramdisk_id: - image_type = ec2utils.image_type('ramdisk') - ctxt = context.get_admin_context() - return ec2utils.image_ec2_id(ctxt, ramdisk_id, image_type) - - def ami_launch_index(self, address): - instance_ref = self.get_instance(address) - return instance_ref['launch_index'] - - def block_device_mapping(self, address): - instance_ref = self.get_instance(address) - return self._format_instance_mapping(instance_ref) - - def hostname(self, address): - instance_ref = self.get_instance(address) - return "%s.%s" % (instance_ref['hostname'], FLAGS.dhcp_domain) - - def local_ipv4(self, address): - return address - - def public_ipv4(self, address): - instance_ref = self.get_instance(address) - ctxt = context.get_admin_context() ip_info = ec2utils.get_ip_info_for_instance(ctxt, instance_ref) floating_ips = ip_info['floating_ips'] floating_ip = floating_ips and floating_ips[0] or '' - return floating_ip - - def reservation_id(self, address): - instance_ref = self.get_instance(address) - return instance_ref['reservation_id'] - def placement(self, address): - instance_ref = self.get_instance(address) - host = instance_ref['host'] - ctxt = context.get_admin_context() - # note(ja): original code had ctx.elevated? - services = db.service_get_all_by_host(ctxt, host) - zone = ec2utils.get_availability_zone_by_host(services, host) - return {'availability-zone': zone} - - def security_groups(self, address): - instance_ref = self.get_instance(address) + ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) + image_id = instance_ref['image_ref'] ctxt = context.get_admin_context() - groups = db.security_group_get_by_instance(ctxt, - instance_ref['id']) - return [g['name'] for g in groups] + image_ec2_id = ec2utils.glance_id_to_ec2_id(ctxt, image_id) + security_groups = db.security_group_get_by_instance(ctxt, + instance_ref['id']) + security_groups = [x['name'] for x in security_groups] + mappings = self._format_instance_mapping(ctxt, instance_ref) + data = { + 'user-data': base64.b64decode(instance_ref['user_data']), + 'meta-data': { + 'ami-id': image_ec2_id, + 'ami-launch-index': instance_ref['launch_index'], + 'ami-manifest-path': 'FIXME', + 'block-device-mapping': mappings, + 'hostname': hostname, + 'instance-action': 'none', + 'instance-id': ec2_id, + 'instance-type': instance_ref['instance_type']['name'], + 'local-hostname': hostname, + 'local-ipv4': address, + 'placement': {'availability-zone': availability_zone}, + 'public-hostname': hostname, + 'public-ipv4': floating_ip, + 'reservation-id': instance_ref['reservation_id'], + 'security-groups': security_groups}} - def public_keys(self, address): - instance_ref = self.get_instance(address) # public-keys should be in meta-data only if user specified one if instance_ref['key_name']: - return {'0': {'_name': instance_ref['key_name'], - 'openssh-key': instance_ref['key_data']}} - - def ami_manifest_path(self, address): - return 'Not Implemented' + data['meta-data']['public-keys'] = { + '0': {'_name': instance_ref['key_name'], + 'openssh-key': instance_ref['key_data']}} + + for image_type in ['kernel', 'ramdisk']: + if instance_ref.get('%s_id' % image_type): + image_id = instance_ref['%s_id' % image_type] + image_type = ec2utils.image_type(image_type) + ec2_id = ec2utils.glance_id_to_ec2_id(ctxt, + image_id, + image_type) + data['meta-data']['%s-id' % image_type] = ec2_id + + if False: # TODO(vish): store ancestor ids + data['ancestor-ami-ids'] = [] + if False: # TODO(vish): store product codes + data['product-codes'] = [] + + self._cache.set(cache_key, data, 15) - def instance_action(self, address): - return 'none' + return data - def format_data(self, data): + def print_data(self, data): if isinstance(data, dict): output = '' for key in data: @@ -298,21 +224,15 @@ class MetadataRequestHandler(wsgi.Application): else: return str(data) - def lookup(self, path, address): + def lookup(self, path, data): items = path.split('/') - data = self.metadata_mapper for item in items: if item: if not isinstance(data, dict): - # FIXME(ja): should we check that we are at the end - # of the path as well before we just return? return data if not item in data: return None data = data[item] - if isinstance(data, collections.Callable): - # lazy evaluation - data = data(address) return data @webob.dec.wsgify(RequestClass=wsgi.Request) @@ -320,20 +240,19 @@ class MetadataRequestHandler(wsgi.Application): remote_address = req.remote_addr if FLAGS.use_forwarded_for: remote_address = req.headers.get('X-Forwarded-For', remote_address) - try: - data = self.lookup(req.path_info, - remote_address) - except (exception.NotFound, exception.FixedIpNotFoundForAddress): - LOG.error(_('Failed to get metadata for ip: %s'), remote_address) - return webob.exc.HTTPNotFound() - except: + meta_data = self.get_metadata(remote_address) + except Exception: LOG.exception(_('Failed to get metadata for ip: %s'), remote_address) msg = _('An unknown error has occurred. ' 'Please try your request again.') - return webob.exc.HTTPInternalServerError(explanation=unicode(msg)) - + exc = webob.exc.HTTPInternalServerError(explanation=unicode(msg)) + return exc + if meta_data is None: + LOG.error(_('Failed to get metadata for ip: %s'), remote_address) + raise webob.exc.HTTPNotFound() + data = self.lookup(req.path_info, meta_data) if data is None: raise webob.exc.HTTPNotFound() - return self.format_data(data) + return self.print_data(data) -- cgit