diff options
Diffstat (limited to 'nova')
85 files changed, 7196 insertions, 1570 deletions
diff --git a/nova/api/direct.py b/nova/api/direct.py index 208b6d086..dfca250e0 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -187,7 +187,7 @@ class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): self.service_handle = service_handle - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] action = arg_dict['action'] @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req) + return self._serialize(result, req.best_match_content_type()) else: return result @@ -218,7 +218,7 @@ class Proxy(object): self.prefix = prefix def __do_request(self, path, context, **kwargs): - req = webob.Request.blank(path) + req = wsgi.Request.blank(path) req.method = 'POST' req.body = urllib.urlencode({'json': utils.dumps(kwargs)}) req.environ['openstack.context'] = context diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 4425ba3cd..fccebca5d 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -53,7 +53,7 @@ flags.DEFINE_list('lockout_memcached_servers', None, class RequestLogging(wsgi.Middleware): """Access-Log akin logging for all EC2 API requests.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): start = utils.utcnow() rv = req.get_response(self.application) @@ -112,7 +112,7 @@ class Lockout(wsgi.Middleware): debug=0) super(Lockout, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): access_key = str(req.params['AWSAccessKeyId']) failures_key = "authfailures-%s" % access_key @@ -141,7 +141,7 @@ class Authenticate(wsgi.Middleware): """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): # Read request signature and access id. try: @@ -190,7 +190,7 @@ class Requestify(wsgi.Middleware): super(Requestify, self).__init__(app) self.controller = utils.import_class(controller)() - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Version', 'Timestamp'] @@ -275,7 +275,7 @@ class Authorizer(wsgi.Middleware): }, } - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] controller = req.environ['ec2.request'].controller.__class__.__name__ @@ -309,7 +309,7 @@ class Executor(wsgi.Application): response, or a 400 upon failure. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] api_request = req.environ['ec2.request'] @@ -371,7 +371,7 @@ class Executor(wsgi.Application): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all EC2 versions.""" # available api versions diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0d22a3f46..40a9da0e7 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -39,7 +39,9 @@ from nova import log as logging from nova import network from nova import utils from nova import volume +from nova.api.ec2 import ec2utils from nova.compute import instance_types +from nova.image import s3 FLAGS = flags.FLAGS @@ -73,30 +75,19 @@ def _gen_key(context, user_id, key_name): return {'private_key': private_key, 'fingerprint': fingerprint} -def ec2_id_to_id(ec2_id): - """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" - return int(ec2_id.split('-')[-1], 16) - - -def id_to_ec2_id(instance_id, template='i-%08x'): - """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" - return template % instance_id - - class CloudController(object): """ CloudController provides the critical dispatch between inbound API calls through the endpoint and messages sent to the other nodes. """ def __init__(self): - self.image_service = utils.import_object(FLAGS.image_service) + self.image_service = s3.S3ImageService() self.network_api = network.API() self.volume_api = volume.API() self.compute_api = compute.API( network_api=self.network_api, - image_service=self.image_service, volume_api=self.volume_api, - hostname_factory=id_to_ec2_id) + hostname_factory=ec2utils.id_to_ec2_id) self.setup() def __str__(self): @@ -115,7 +106,7 @@ class CloudController(object): start = os.getcwd() os.chdir(FLAGS.ca_path) # TODO(vish): Do this with M2Crypto instead - utils.runthis(_("Generating root CA: %s"), "sh genrootca.sh") + utils.runthis(_("Generating root CA: %s"), "sh", "genrootca.sh") os.chdir(start) def _get_mpi_data(self, context, project_id): @@ -154,11 +145,12 @@ class CloudController(object): availability_zone = self._get_availability_zone_by_host(ctxt, host) floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) - ec2_id = id_to_ec2_id(instance_ref['id']) + ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) + image_ec2_id = self._image_ec2_id(instance_ref['image_id'], 'machine') data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { - 'ami-id': instance_ref['image_id'], + 'ami-id': image_ec2_id, 'ami-launch-index': instance_ref['launch_index'], 'ami-manifest-path': 'FIXME', 'block-device-mapping': { @@ -173,15 +165,20 @@ class CloudController(object): 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, 'local-ipv4': address, - 'kernel-id': instance_ref['kernel_id'], 'placement': {'availability-zone': availability_zone}, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', 'public-keys': keys, - 'ramdisk-id': instance_ref['ramdisk_id'], 'reservation-id': instance_ref['reservation_id'], 'security-groups': '', 'mpi': mpi}} + + for image_type in ['kernel', 'ramdisk']: + if '%s_id' % image_type in instance_ref: + ec2_id = self._image_ec2_id(instance_ref['%s_id' % image_type], + image_type) + data['meta-data']['%s-id' % image_type] = ec2_id + if False: # TODO(vish): store ancestor ids data['ancestor-ami-ids'] = [] if False: # TODO(vish): store product codes @@ -525,7 +522,7 @@ class CloudController(object): ec2_id = instance_id[0] else: ec2_id = instance_id - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) output = self.compute_api.get_console_output( context, instance_id=instance_id) now = datetime.datetime.utcnow() @@ -535,7 +532,7 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) return self.compute_api.get_ajax_console(context, instance_id=instance_id) @@ -543,7 +540,7 @@ class CloudController(object): if volume_id: volumes = [] for ec2_id in volume_id: - internal_id = ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) volume = self.volume_api.get(context, internal_id) volumes.append(volume) else: @@ -556,11 +553,11 @@ class CloudController(object): instance_data = None if volume.get('instance', None): instance_id = volume['instance']['id'] - instance_ec2_id = id_to_ec2_id(instance_id) + instance_ec2_id = ec2utils.id_to_ec2_id(instance_id) instance_data = '%s[%s]' % (instance_ec2_id, volume['instance']['host']) v = {} - v['volumeId'] = id_to_ec2_id(volume['id'], 'vol-%08x') + v['volumeId'] = ec2utils.id_to_ec2_id(volume['id'], 'vol-%08x') v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] @@ -568,7 +565,7 @@ class CloudController(object): if context.is_admin: v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], - volume['user_id'], + volume['project_id'], volume['host'], instance_data, volume['mountpoint']) @@ -578,8 +575,7 @@ class CloudController(object): 'device': volume['mountpoint'], 'instanceId': instance_ec2_id, 'status': 'attached', - 'volumeId': id_to_ec2_id(volume['id'], - 'vol-%08x')}] + 'volumeId': v['volumeId']}] else: v['attachmentSet'] = [{}] @@ -598,12 +594,12 @@ class CloudController(object): return {'volumeSet': [self._format_volume(context, dict(volume))]} def delete_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) self.volume_api.delete(context, volume_id=volume_id) return True def update_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) updatable_fields = ['display_name', 'display_description'] changes = {} for field in updatable_fields: @@ -614,8 +610,8 @@ class CloudController(object): return True def attach_volume(self, context, volume_id, instance_id, device, **kwargs): - volume_id = ec2_id_to_id(volume_id) - instance_id = ec2_id_to_id(instance_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) msg = _("Attach volume %(volume_id)s to instance %(instance_id)s" " at %(device)s") % locals() LOG.audit(msg, context=context) @@ -626,22 +622,22 @@ class CloudController(object): volume = self.volume_api.get(context, volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], - 'instanceId': id_to_ec2_id(instance_id), + 'instanceId': ec2utils.id_to_ec2_id(instance_id), 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': id_to_ec2_id(volume_id, 'vol-%08x')} + 'volumeId': ec2utils.id_to_ec2_id(volume_id, 'vol-%08x')} def detach_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) LOG.audit(_("Detach volume %s"), volume_id, context=context) volume = self.volume_api.get(context, volume_id) instance = self.compute_api.detach_volume(context, volume_id=volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], - 'instanceId': id_to_ec2_id(instance['id']), + 'instanceId': ec2utils.id_to_ec2_id(instance['id']), 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': id_to_ec2_id(volume_id, 'vol-%08x')} + 'volumeId': ec2utils.id_to_ec2_id(volume_id, 'vol-%08x')} def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -675,7 +671,7 @@ class CloudController(object): if instance_id: instances = [] for ec2_id in instance_id: - internal_id = ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) instance = self.compute_api.get(context, instance_id=internal_id) instances.append(instance) @@ -687,9 +683,9 @@ class CloudController(object): continue i = {} instance_id = instance['id'] - ec2_id = id_to_ec2_id(instance_id) + ec2_id = ec2utils.id_to_ec2_id(instance_id) i['instanceId'] = ec2_id - i['imageId'] = instance['image_id'] + i['imageId'] = self._image_ec2_id(instance['image_id']) i['instanceState'] = { 'code': instance['state'], 'name': instance['state_description']} @@ -755,7 +751,7 @@ class CloudController(object): if (floating_ip_ref['fixed_ip'] and floating_ip_ref['fixed_ip']['instance']): instance_id = floating_ip_ref['fixed_ip']['instance']['id'] - ec2_id = id_to_ec2_id(instance_id) + ec2_id = ec2utils.id_to_ec2_id(instance_id) address_rv = {'public_ip': address, 'instance_id': ec2_id} if context.is_admin: @@ -778,7 +774,7 @@ class CloudController(object): def associate_address(self, context, instance_id, public_ip, **kwargs): LOG.audit(_("Associate address %(public_ip)s to" " instance %(instance_id)s") % locals(), context=context) - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.associate_floating_ip(context, instance_id=instance_id, address=public_ip) @@ -791,13 +787,19 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) + if kwargs.get('kernel_id'): + kernel = self._get_image(context, kwargs['kernel_id']) + kwargs['kernel_id'] = kernel['id'] + if kwargs.get('ramdisk_id'): + ramdisk = self._get_image(context, kwargs['ramdisk_id']) + kwargs['ramdisk_id'] = ramdisk['id'] instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id=kwargs['image_id'], + image_id=self._get_image(context, kwargs['image_id'])['id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, - kernel_id=kwargs.get('kernel_id', None), + kernel_id=kwargs.get('kernel_id'), ramdisk_id=kwargs.get('ramdisk_id'), display_name=kwargs.get('display_name'), display_description=kwargs.get('display_description'), @@ -814,7 +816,7 @@ class CloudController(object): instance_id is a kwarg so its name cannot be modified.""" LOG.debug(_("Going to start terminating instances")) for ec2_id in instance_id: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.delete(context, instance_id=instance_id) return True @@ -822,19 +824,19 @@ class CloudController(object): """instance_id is a list of instance ids""" LOG.audit(_("Reboot instance %r"), instance_id, context=context) for ec2_id in instance_id: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.reboot(context, instance_id=instance_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.rescue(context, instance_id=instance_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.unrescue(context, instance_id=instance_id) return True @@ -845,41 +847,80 @@ class CloudController(object): if field in kwargs: changes[field] = kwargs[field] if changes: - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.update(context, instance_id=instance_id, **kwargs) return True - def _format_image(self, context, image): + _type_prefix_map = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + + def _image_ec2_id(self, image_id, image_type='machine'): + prefix = self._type_prefix_map[image_type] + template = prefix + '-%08x' + return ec2utils.id_to_ec2_id(int(image_id), template=template) + + def _get_image(self, context, ec2_id): + try: + internal_id = ec2utils.ec2_id_to_id(ec2_id) + return self.image_service.show(context, internal_id) + except exception.NotFound: + return self.image_service.show_by_name(context, ec2_id) + + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - i['imageId'] = image.get('id') - i['kernelId'] = image.get('kernel_id') - i['ramdiskId'] = image.get('ramdisk_id') - i['imageOwnerId'] = image.get('owner_id') - i['imageLocation'] = image.get('location') - i['imageState'] = image.get('status') - i['type'] = image.get('type') - i['isPublic'] = image.get('is_public') - i['architecture'] = image.get('architecture') + image_type = image['properties'].get('type') + ec2_id = self._image_ec2_id(image.get('id'), image_type) + name = image.get('name') + if name: + i['imageId'] = "%s (%s)" % (ec2_id, name) + else: + i['imageId'] = ec2_id + kernel_id = image['properties'].get('kernel_id') + if kernel_id: + i['kernelId'] = self._image_ec2_id(kernel_id, 'kernel') + ramdisk_id = image['properties'].get('ramdisk_id') + if ramdisk_id: + i['ramdiskId'] = self._image_ec2_id(ramdisk_id, 'ramdisk') + i['imageOwnerId'] = image['properties'].get('owner_id') + i['imageLocation'] = image['properties'].get('image_location') + i['imageState'] = image['properties'].get('image_state') + i['type'] = image_type + i['isPublic'] = str(image['properties'].get('is_public', '')) == 'True' + i['architecture'] = image['properties'].get('architecture') return i def describe_images(self, context, image_id=None, **kwargs): # NOTE: image_id is a list! - images = self.image_service.index(context) if image_id: - images = filter(lambda x: x['id'] in image_id, images) - images = [self._format_image(context, i) for i in images] + images = [] + for ec2_id in image_id: + try: + image = self._get_image(context, ec2_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % + ec2_id) + images.append(image) + else: + images = self.image_service.detail(context) + images = [self._format_image(i) for i in images] return {'imagesSet': images} def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - self.image_service.deregister(context, image_id) + image = self._get_image(context, image_id) + internal_id = image['id'] + self.image_service.delete(context, internal_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - image_id = self.image_service.register(context, image_location) + metadata = {'properties': {'image_location': image_location}} + image = self.image_service.create(context, metadata) + image_id = self._image_ec2_id(image['id'], + image['properties']['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -890,13 +931,11 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: - image = self._format_image(context, - self.image_service.show(context, - image_id)) - except IndexError: - raise exception.ApiError(_('invalid id: %s') % image_id) - result = {'image_id': image_id, 'launchPermission': []} - if image['isPublic']: + image = self._get_image(context, image_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % image_id) + result = {'imageId': image_id, 'launchPermission': []} + if image['properties']['is_public']: result['launchPermission'].append({'group': 'all'}) return result @@ -913,8 +952,18 @@ class CloudController(object): if not operation_type in ['add', 'remove']: raise exception.ApiError(_('operation_type must be add or remove')) LOG.audit(_("Updating image %s publicity"), image_id, context=context) - return self.image_service.modify(context, image_id, operation_type) + + try: + image = self._get_image(context, image_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % image_id) + internal_id = image['id'] + del(image['id']) + raise Exception(image) + image['properties']['is_public'] = (operation_type == 'add') + return self.image_service.update(context, internal_id, image) def update_image(self, context, image_id, **kwargs): - result = self.image_service.update(context, image_id, dict(kwargs)) + internal_id = ec2utils.ec2_id_to_id(image_id) + result = self.image_service.update(context, internal_id, dict(kwargs)) return result diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py new file mode 100644 index 000000000..3b34f6ea5 --- /dev/null +++ b/nova/api/ec2/ec2utils.py @@ -0,0 +1,32 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import exception + + +def ec2_id_to_id(ec2_id): + """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" + try: + return int(ec2_id.split('-')[-1], 16) + except ValueError: + raise exception.NotFound(_("Id %s Not Found") % ec2_id) + + +def id_to_ec2_id(instance_id, template='i-%08x'): + """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" + return template % instance_id diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 6fb441656..28f99b0ef 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -65,7 +65,7 @@ class MetadataRequestHandler(wsgi.Application): data = data[item] return data - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): cc = cloud.CloudController() remote_address = req.remote_addr diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index bdebb6a1e..ce3cff337 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -27,6 +27,7 @@ import webob.exc from nova import flags from nova import log as logging from nova import wsgi +from nova.api.openstack import accounts from nova.api.openstack import faults from nova.api.openstack import backup_schedules from nova.api.openstack import consoles @@ -34,6 +35,7 @@ from nova.api.openstack import flavors from nova.api.openstack import images from nova.api.openstack import servers from nova.api.openstack import shared_ip_groups +from nova.api.openstack import users from nova.api.openstack import zones @@ -47,7 +49,7 @@ flags.DEFINE_bool('allow_admin_api', class FaultWrapper(wsgi.Middleware): """Calls down the middleware stack, making exceptions into faults.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): try: return req.get_response(self.application) @@ -89,6 +91,13 @@ class APIRouter(wsgi.Router): mapper.resource("zone", "zones", controller=zones.Controller(), collection={'detail': 'GET', 'info': 'GET'}), + mapper.resource("user", "users", controller=users.Controller(), + collection={'detail': 'GET'}) + + mapper.resource("account", "accounts", + controller=accounts.Controller(), + collection={'detail': 'GET'}) + mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, member=server_members) @@ -115,7 +124,7 @@ class APIRouter(wsgi.Router): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all OpenStack API versions.""" response = { @@ -124,4 +133,6 @@ class Versions(wsgi.Application): metadata = { "application/xml": { "attributes": dict(version=["status", "id"])}} - return wsgi.Serializer(req.environ, metadata).to_content_type(response) + + content_type = req.best_match_content_type() + return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py new file mode 100644 index 000000000..2510ffb61 --- /dev/null +++ b/nova/api/openstack/accounts.py @@ -0,0 +1,85 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import common + +from nova import exception +from nova import flags +from nova import log as logging +from nova import wsgi + +from nova.auth import manager +from nova.api.openstack import faults + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.api.openstack') + + +def _translate_keys(account): + return dict(id=account.id, + name=account.name, + description=account.description, + manager=account.project_manager_id) + + +class Controller(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "account": ["id", "name", "description", "manager"]}}} + + def __init__(self): + self.manager = manager.AuthManager() + + def _check_admin(self, context): + """We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here""" + if not context.is_admin: + raise exception.NotAuthorized(_("Not admin user.")) + + def index(self, req): + raise faults.Fault(exc.HTTPNotImplemented()) + + def detail(self, req): + raise faults.Fault(exc.HTTPNotImplemented()) + + def show(self, req, id): + """Return data about the given account id""" + account = self.manager.get_project(id) + return dict(account=_translate_keys(account)) + + def delete(self, req, id): + self._check_admin(req.environ['nova.context']) + self.manager.delete_project(id) + return {} + + def create(self, req): + """We use update with create-or-update semantics + because the id comes from an external source""" + raise faults.Fault(exc.HTTPNotImplemented()) + + def update(self, req, id): + """This is really create or update.""" + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req.get_content_type()) + description = env['account'].get('description') + manager = env['account'].get('manager') + try: + account = self.manager.get_project(id) + self.manager.modify_project(id, manager, description) + except exception.NotFound: + account = self.manager.create_project(id, manager, description) + return dict(account=_translate_keys(account)) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6011e6115..f3a9bdeca 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -28,11 +28,13 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import log as logging from nova import manager from nova import utils from nova import wsgi from nova.api.openstack import faults +LOG = logging.getLogger('nova.api.openstack') FLAGS = flags.FLAGS @@ -46,18 +48,27 @@ class AuthMiddleware(wsgi.Middleware): self.auth = auth.manager.AuthManager() super(AuthMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): if not self.has_authentication(req): return self.authenticate(req) - user = self.get_user_by_authentication(req) - + accounts = self.auth.get_projects(user=user) if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - project = self.auth.get_project(FLAGS.default_project) - req.environ['nova.context'] = context.RequestContext(user, project) + if accounts: + #we are punting on this til auth is settled, + #and possibly til api v1.1 (mdragon) + account = accounts[0] + else: + return faults.Fault(webob.exc.HTTPUnauthorized()) + + if not self.auth.is_admin(user) and \ + not self.auth.is_project_member(user, account): + return faults.Fault(webob.exc.HTTPUnauthorized()) + + req.environ['nova.context'] = context.RequestContext(user, account) return self.application def has_authentication(self, req): @@ -121,18 +132,23 @@ class AuthMiddleware(wsgi.Middleware): username - string key - string API key - req - webob.Request object + req - wsgi.Request object """ ctxt = context.get_admin_context() - user = self.auth.get_user_from_access_key(key) + + try: + user = self.auth.get_user_from_access_key(key) + except exception.NotFound: + user = None + if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, time.time())).hexdigest() token_dict = {} token_dict['token_hash'] = token_hash token_dict['cdn_management_url'] = '' - # Same as auth url, e.g. http://foo.org:8774/baz/v1.0 - token_dict['server_management_url'] = req.url + os_url = req.url + token_dict['server_management_url'] = os_url token_dict['storage_url'] = '' token_dict['user_id'] = user.id token = self.db.auth_token_create(ctxt, token_dict) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 9f85c5c8a..74ac21024 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -25,7 +25,7 @@ def limited(items, request, max_limit=1000): Return a slice of items according to requested offset and limit. @param items: A sliceable entity - @param request: `webob.Request` possibly containing 'offset' and 'limit' + @param request: `wsgi.Request` possibly containing 'offset' and 'limit' GET variables. 'offset' is where to start in the list, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default @@ -36,15 +36,18 @@ def limited(items, request, max_limit=1000): try: offset = int(request.GET.get('offset', 0)) except ValueError: - offset = 0 + raise webob.exc.HTTPBadRequest(_('offset param must be an integer')) try: limit = int(request.GET.get('limit', max_limit)) except ValueError: - limit = max_limit + raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) - if offset < 0 or limit < 0: - raise webob.exc.HTTPBadRequest() + if limit < 0: + raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + + if offset < 0: + raise webob.exc.HTTPBadRequest(_('offset param must be positive')) limit = min(max_limit, limit or max_limit) range_end = offset + limit diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 9ebdbe710..8c291c2eb 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -65,7 +65,7 @@ class Controller(wsgi.Controller): def create(self, req, server_id): """Creates a new console""" - #info = self._deserialize(req.body, req) + #info = self._deserialize(req.body, req.get_content_type()) self.console_api.create_console( req.environ['nova.context'], int(server_id)) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 224a7ef0b..2fd733299 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -42,7 +42,7 @@ class Fault(webob.exc.HTTPException): """Create a Fault for the given webob.exc.exception.""" self.wrapped_exc = exception - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Generate a WSGI response based on the exception passed to ctor.""" # Replace the body with fault details. @@ -57,6 +57,7 @@ class Fault(webob.exc.HTTPException): fault_data[fault_name]['retryAfter'] = retry # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} - serializer = wsgi.Serializer(req.environ, metadata) - self.wrapped_exc.body = serializer.to_content_type(fault_data) + serializer = wsgi.Serializer(metadata) + content_type = req.best_match_content_type() + self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cf85a496f..98f0dd96b 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -151,7 +151,7 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) instance_id = env["image"]["serverId"] name = env["image"]["name"] diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index cbb4b897e..88ffc3246 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -57,7 +57,7 @@ class RateLimitingMiddleware(wsgi.Middleware): self.limiter = WSGIAppProxy(service_host) super(RateLimitingMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Rate limit the request. @@ -183,7 +183,7 @@ class WSGIApp(object): """Create the WSGI application using the given Limiter instance.""" self.limiter = limiter - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): parts = req.path_info.split('/') # format: /limiter/<username>/<urlencoded action> diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 85999764f..9f14a6b82 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -13,9 +13,11 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 import hashlib import json import traceback +from xml.dom import minidom from webob import exc @@ -98,7 +100,7 @@ class Controller(wsgi.Controller): 'application/xml': { "attributes": { "server": ["id", "imageId", "name", "flavorId", "hostId", - "status", "progress"]}}} + "status", "progress", "adminPass"]}}} def __init__(self): self.compute_api = compute.API() @@ -126,7 +128,9 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - instance = self.compute_api.get(req.environ['nova.context'], id) + LOG.debug(_("***SHOW")) + instance = self.compute_api.routing_get(req.environ['nova.context'], id) + LOG.debug(_("***SHOW")) return _translate_detail_keys(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -141,15 +145,19 @@ class Controller(wsgi.Controller): def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) + env = self._deserialize_create(req) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) context = req.environ['nova.context'] + + key_name = None + key_data = None key_pairs = auth_manager.AuthManager.get_key_pairs(context) - if not key_pairs: - raise exception.NotFound(_("No keypairs defined")) - key_pair = key_pairs[0] + if key_pairs: + key_pair = key_pairs[0] + key_name = key_pair['name'] + key_data = key_pair['public_key'] image_id = common.get_image_id_from_image_hash(self._image_service, context, env['server']['imageId']) @@ -166,23 +174,94 @@ class Controller(wsgi.Controller): for k, v in env['server']['metadata'].items(): metadata.append({'key': k, 'value': v}) - instances = self.compute_api.create( - context, - instance_types.get_by_flavor_id(env['server']['flavorId']), - image_id, - kernel_id=kernel_id, - ramdisk_id=ramdisk_id, - display_name=env['server']['name'], - display_description=env['server']['name'], - key_name=key_pair['name'], - key_data=key_pair['public_key'], - metadata=metadata, - onset_files=env.get('onset_files', [])) - return _translate_keys(instances[0]) + personality = env['server'].get('personality', []) + injected_files = self._get_injected_files(personality) + + try: + instances = self.compute_api.create( + context, + instance_types.get_by_flavor_id(env['server']['flavorId']), + image_id, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + display_name=env['server']['name'], + display_description=env['server']['name'], + key_name=key_name, + key_data=key_data, + metadata=metadata, + injected_files=injected_files) + except QuotaError as error: + self._handle_quota_error(error) + + server = _translate_keys(instances[0]) + password = "%s%s" % (server['server']['name'][:4], + utils.generate_password(12)) + server['server']['adminPass'] = password + self.compute_api.set_admin_password(context, server['server']['id'], + password) + return server + + def _deserialize_create(self, request): + """ + Deserialize a create request + + Overrides normal behavior in the case of xml content + """ + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request.get_content_type()) + + def _get_injected_files(self, personality): + """ + Create a list of injected files from the personality attribute + + At this time, injected_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + injected_files = [] + for item in personality: + try: + path = item['path'] + contents = item['contents'] + except KeyError as key: + expl = _('Bad personality format: missing %s') % key + raise exc.HTTPBadRequest(explanation=expl) + except TypeError: + expl = _('Bad personality format') + raise exc.HTTPBadRequest(explanation=expl) + try: + contents = base64.b64decode(contents) + except TypeError: + expl = _('Personality content for %s cannot be decoded') % path + raise exc.HTTPBadRequest(explanation=expl) + injected_files.append((path, contents)) + return injected_files + + def _handle_quota_errors(self, error): + """ + Reraise quota errors as api-specific http exceptions + """ + if error.code == "OnsetFileLimitExceeded": + expl = _("Personality file limit exceeded") + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFilePathLimitExceeded": + expl = _("Personality file path too long") + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFileContentLimitExceeded": + expl = _("Personality file content too long") + raise exc.HTTPBadRequest(explanation=expl) + # if the original error is okay, just reraise it + raise error def update(self, req, id): """ Updates the server name or password """ - inst_dict = self._deserialize(req.body, req) + if len(req.body) == 0: + raise exc.HTTPUnprocessableEntity() + + inst_dict = self._deserialize(req.body, req.get_content_type()) if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -214,7 +293,7 @@ class Controller(wsgi.Controller): 'rebuild': self._action_rebuild, } - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) for key in actions.keys(): if key in input_dict: return actions[key](input_dict, req, id) @@ -466,3 +545,79 @@ class Controller(wsgi.Controller): _("Ramdisk not found for image %(image_id)s") % locals()) return kernel_id, ramdisk_id + + +class ServerCreateRequestXMLDeserializer(object): + """ + Deserializer to handle xml-formatted server create requests. + + Handles standard server attributes as well as optional metadata + and personality attributes + """ + + def deserialize(self, string): + """Deserialize an xml-formatted server create request""" + dom = minidom.parseString(string) + server = self._extract_server(dom) + return {'server': server} + + def _extract_server(self, node): + """Marshal the server attribute of a parsed request""" + server = {} + server_node = self._find_first_child_named(node, 'server') + for attr in ["name", "imageId", "flavorId"]: + server[attr] = server_node.getAttribute(attr) + metadata = self._extract_metadata(server_node) + if metadata is not None: + server["metadata"] = metadata + personality = self._extract_personality(server_node) + if personality is not None: + server["personality"] = personality + return server + + def _extract_metadata(self, server_node): + """Marshal the metadata attribute of a parsed request""" + metadata_node = self._find_first_child_named(server_node, "metadata") + if metadata_node is None: + return None + metadata = {} + for meta_node in self._find_children_named(metadata_node, "meta"): + key = meta_node.getAttribute("key") + metadata[key] = self._extract_text(meta_node) + return metadata + + def _extract_personality(self, server_node): + """Marshal the personality attribute of a parsed request""" + personality_node = \ + self._find_first_child_named(server_node, "personality") + if personality_node is None: + return None + personality = [] + for file_node in self._find_children_named(personality_node, "file"): + item = {} + if file_node.hasAttribute("path"): + item["path"] = file_node.getAttribute("path") + item["contents"] = self._extract_text(file_node) + personality.append(item) + return personality + + def _find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def _find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + + def _extract_text(self, node): + """Get the text field contained by the given node""" + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py new file mode 100644 index 000000000..ebd0f4512 --- /dev/null +++ b/nova/api/openstack/users.py @@ -0,0 +1,93 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import common + +from nova import exception +from nova import flags +from nova import log as logging +from nova import wsgi + +from nova.auth import manager + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.api.openstack') + + +def _translate_keys(user): + return dict(id=user.id, + name=user.name, + access=user.access, + secret=user.secret, + admin=user.admin) + + +class Controller(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "user": ["id", "name", "access", "secret", "admin"]}}} + + def __init__(self): + self.manager = manager.AuthManager() + + def _check_admin(self, context): + """We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here""" + if not context.is_admin: + raise exception.NotAuthorized(_("Not admin user")) + + def index(self, req): + """Return all users in brief""" + users = self.manager.get_users() + users = common.limited(users, req) + users = [_translate_keys(user) for user in users] + return dict(users=users) + + def detail(self, req): + """Return all users in detail""" + return self.index(req) + + def show(self, req, id): + """Return data about the given user id""" + user = self.manager.get_user(id) + return dict(user=_translate_keys(user)) + + def delete(self, req, id): + self._check_admin(req.environ['nova.context']) + self.manager.delete_user(id) + return {} + + def create(self, req): + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req.get_content_type()) + is_admin = env['user'].get('admin') in ('T', 'True', True) + name = env['user'].get('name') + access = env['user'].get('access') + secret = env['user'].get('secret') + user = self.manager.create_user(name, access, secret, is_admin) + return dict(user=_translate_keys(user)) + + def update(self, req, id): + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req.get_content_type()) + is_admin = env['user'].get('admin') + if is_admin is not None: + is_admin = is_admin in ('T', 'True', True) + access = env['user'].get('access') + secret = env['user'].get('secret') + self.manager.modify_user(id, access, secret, is_admin) + return dict(user=_translate_keys(self.manager.get_user(id))) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index fecbd6fa3..ebfc7743c 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -71,9 +71,9 @@ class Controller(wsgi.Controller): items = api.API.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) - caps = FLAGS.zone_capabilities.split(';') + caps = FLAGS.zone_capabilities for cap in caps: - key_values = cap.split(':') + key_values = cap.split('=') zone[key_values[0]] = key_values[1] for item, (min_value, max_value) in items.iteritems(): zone[item] = "%s,%s" % (min_value, max_value) @@ -92,13 +92,13 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) zone = db.zone_update(context, zone_id, env["zone"]) return dict(zone=_scrub_zone(zone)) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py deleted file mode 100644 index 7ebae1401..000000000 --- a/nova/api/zone_redirect.py +++ /dev/null @@ -1,120 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License.import datetime - -"""Reroutes calls to child zones on ZoneRouteException's.""" - -import httplib -import re -import webob -import webob.dec -import webob.exc -import urlparse -import urllib - -from nova import exception -from nova import log as logging -from nova import wsgi -from nova.scheduler import api - -import novaclient.client as client -import novaclient.exceptions as osexceptions - -try: - import json -except ImportError: - import simplejson as json - - -LOG = logging.getLogger('server') - - -class RequestForwarder(api.ChildZoneHelper): - """Worker for sending an OpenStack Request to each child zone.""" - def __init__(self, resource, method, body): - self.resource = resource - self.method = method - self.body = body - - def process(self, client, zone): - api_url = zone.api_url - LOG.debug(_("Zone redirect to: %(api_url)s, " % locals())) - try: - if self.method == 'GET': - response, body = client.get(self.resource, body=self.body) - elif self.method == 'POST': - response, body = client.post(self.resource, body=self.body) - elif self.method == 'PUT': - response, body = client.put(self.resource, body=self.body) - elif self.method == 'DELETE': - response, body = client.delete(self.resource, body=self.body) - except osexceptions.OpenStackException, e: - LOG.info(_("Zone returned error: %s ('%s', '%s')"), - e.code, e.message, e.details) - res = webob.Response() - res.status = "404" - return res - - status = response.status - LOG.debug(_("Zone %(api_url)s response: " - "%(response)s [%(status)s]/ %(body)s") % - locals()) - res = webob.Response() - res.status = response['status'] - res.content_type = response['content-type'] - res.body = json.dumps(body) - return res - - -class ZoneRedirectMiddleware(wsgi.Middleware): - """Catches Zone Routing exceptions and delegates the call - to child zones.""" - - @webob.dec.wsgify - def __call__(self, req): - try: - return req.get_response(self.application) - except exception.ZoneRouteException as e: - if not e.zones: - exc = webob.exc.HTTPInternalServerError(explanation=_( - "No zones to reroute to.")) - return faults.Fault(exc) - - # Todo(sandy): This only works for OpenStack API currently. - # Needs to be broken out into a driver. - scheme, netloc, path, query, frag = \ - urlparse.urlsplit(req.path_qs) - query = urlparse.parse_qsl(query) - # Remove any cache busters from old novaclient calls ... - query = [(key, value) for key, value in query if key != 'fresh'] - query = urllib.urlencode(query) - url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - - # Strip off the API version, since this is given when the - # child zone was added. - m = re.search('/v\d+\.\d+/(.+)', url) - resource = m.group(1) - - forwarder = RequestForwarder(resource, req.method, req.body) - for result in forwarder.start(e.zones): - # Todo(sandy): We need to aggregate multiple successes. - if result.status_int == 200: - return result - - LOG.debug(_("Zone Redirect Middleware returning 404 ...")) - res = webob.Response() - res.status = "404" - return res diff --git a/nova/compute/api.py b/nova/compute/api.py index f4bfe720c..9fb4c8ae2 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -34,6 +34,7 @@ from nova import rpc from nova import utils from nova import volume from nova.compute import instance_types +from nova.scheduler import api as scheduler_api from nova.db import base FLAGS = flags.FLAGS @@ -80,13 +81,32 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) + def _check_injected_file_quota(self, context, injected_files): + """ + Enforce quota limits on injected files + + Raises a QuotaError if any limit is exceeded + """ + if injected_files is None: + return + limit = quota.allowed_injected_files(context) + if len(injected_files) > limit: + raise quota.QuotaError(code="OnsetFileLimitExceeded") + path_limit = quota.allowed_injected_file_path_bytes(context) + content_limit = quota.allowed_injected_file_content_bytes(context) + for path, content in injected_files: + if len(path) > path_limit: + raise quota.QuotaError(code="OnsetFilePathLimitExceeded") + if len(content) > content_limit: + raise quota.QuotaError(code="OnsetFileContentLimitExceeded") + def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - onset_files=None): + injected_files=None): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -124,11 +144,18 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") + self._check_injected_file_quota(context, injected_files) + image = self.image_service.show(context, image_id) + + os_type = None + if 'properties' in image and 'os_type' in image['properties']: + os_type = image['properties']['os_type'] + if kernel_id is None: - kernel_id = image.get('kernel_id', None) + kernel_id = image['properties'].get('kernel_id', None) if ramdisk_id is None: - ramdisk_id = image.get('ramdisk_id', None) + ramdisk_id = image['properties'].get('ramdisk_id', None) # FIXME(sirp): is there a way we can remove null_kernel? # No kernel and ramdisk for raw images if kernel_id == str(FLAGS.null_kernel): @@ -165,6 +192,7 @@ class API(base.Base): 'image_id': image_id, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', + 'state': 0, 'state_description': 'scheduling', 'user_id': context.user_id, 'project_id': context.project_id, @@ -180,7 +208,8 @@ class API(base.Base): 'key_data': key_data, 'locked': False, 'metadata': metadata, - 'availability_zone': availability_zone} + 'availability_zone': availability_zone, + 'os_type': os_type} elevated = context.elevated() instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) @@ -218,7 +247,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "onset_files": onset_files}}) + "injected_files": injected_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) @@ -314,6 +343,7 @@ class API(base.Base): rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) + #@scheduler_api.reroute_if_not_found("delete") def delete(self, context, instance_id): LOG.debug(_("Going to try to terminate %s"), instance_id) try: @@ -343,9 +373,18 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" + LOG.debug("*** COMPUTE.API::GET") rv = self.db.instance_get(context, instance_id) + LOG.debug("*** COMPUTE.API::GET OUT CLEAN") return dict(rv.iteritems()) + @scheduler_api.reroute_if_not_found("get") + def routing_get(self, context, instance_id): + """Use this method instead of get() if this is the only + operation you intend to to. It will route to novaclient.get + if the instance is not found.""" + return self.get(context, instance_id) + def get_all(self, context, project_id=None, reservation_id=None, fixed_ip=None): """Get all instances, possibly filtered by one of the @@ -463,14 +502,17 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, }},) + #@scheduler_api.reroute_if_not_found("pause") def pause(self, context, instance_id): """Pause the given instance.""" self._cast_compute_message('pause_instance', context, instance_id) + #@scheduler_api.reroute_if_not_found("unpause") def unpause(self, context, instance_id): """Unpause the given instance.""" self._cast_compute_message('unpause_instance', context, instance_id) + #@scheduler_api.reroute_if_not_found("diagnostics") def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for the given instance.""" return self._call_compute_message( @@ -482,25 +524,30 @@ class API(base.Base): """Retrieve actions for the given instance.""" return self.db.instance_get_actions(context, instance_id) + #@scheduler_api.reroute_if_not_found("suspend") def suspend(self, context, instance_id): """suspend the instance with instance_id""" self._cast_compute_message('suspend_instance', context, instance_id) + #@scheduler_api.reroute_if_not_found("resume") def resume(self, context, instance_id): """resume the instance with instance_id""" self._cast_compute_message('resume_instance', context, instance_id) + #@scheduler_api.reroute_if_not_found("rescue") def rescue(self, context, instance_id): """Rescue the given instance.""" self._cast_compute_message('rescue_instance', context, instance_id) + #@scheduler_api.reroute_if_not_found("unrescue") def unrescue(self, context, instance_id): """Unrescue the given instance.""" self._cast_compute_message('unrescue_instance', context, instance_id) - def set_admin_password(self, context, instance_id): + def set_admin_password(self, context, instance_id, password=None): """Set the root/admin password for the given instance.""" - self._cast_compute_message('set_admin_password', context, instance_id) + self._cast_compute_message('set_admin_password', context, instance_id, + password) def inject_file(self, context, instance_id): """Write a file to the given instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ebe1ce6f0..a31df0895 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -34,17 +34,19 @@ terminating it. :func:`nova.utils.import_object` """ -import base64 import datetime +import os import random import string import socket +import tempfile +import time import functools from nova import exception from nova import flags from nova import log as logging -from nova import scheduler_manager +from nova import manager from nova import rpc from nova import utils from nova.compute import power_state @@ -61,6 +63,9 @@ flags.DEFINE_integer('password_length', 12, flags.DEFINE_string('console_host', socket.gethostname(), 'Console proxy host to use to connect to instances on' 'this host.') +flags.DEFINE_integer('live_migration_retry_count', 30, + ("Retry count needed in live_migration." + " sleep 1 sec for each count")) LOG = logging.getLogger('nova.compute.manager') @@ -99,7 +104,7 @@ def checks_instance_lock(function): return decorated_function -class ComputeManager(scheduler_manager.SchedulerDependentManager): +class ComputeManager(manager.SchedulerDependentManager): """Manages the running instances from creation to destruction.""" @@ -175,14 +180,14 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('onset_files', []) + instance_ref.injected_files = kwargs.get('injected_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, context=context) self.db.instance_update(context, instance_id, - {'host': self.host}) + {'host': self.host, 'launched_on': self.host}) self.db.instance_set_state(context, instance_id, @@ -354,15 +359,10 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): LOG.warn(_('trying to inject a file into a non-running ' 'instance: %(instance_id)s (state: %(instance_state)s ' 'expected: %(expected_state)s)') % locals()) - # Files/paths *should* be base64-encoded at this point, but - # double-check to make sure. - b64_path = utils.ensure_b64_encoding(path) - b64_contents = utils.ensure_b64_encoding(file_contents) - plain_path = base64.b64decode(b64_path) nm = instance_ref['name'] - msg = _('instance %(nm)s: injecting file to %(plain_path)s') % locals() + msg = _('instance %(nm)s: injecting file to %(path)s') % locals() LOG.audit(msg) - self.driver.inject_file(instance_ref, b64_path, b64_contents) + self.driver.inject_file(instance_ref, path, file_contents) @exception.wrap_exception @checks_instance_lock @@ -724,3 +724,248 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): self.volume_manager.remove_compute_volume(context, volume_id) self.db.volume_detached(context, volume_id) return True + + @exception.wrap_exception + def compare_cpu(self, context, cpu_info): + """Checks the host cpu is compatible to a cpu given by xml. + + :param context: security context + :param cpu_info: json string obtained from virConnect.getCapabilities + :returns: See driver.compare_cpu + + """ + return self.driver.compare_cpu(cpu_info) + + @exception.wrap_exception + def create_shared_storage_test_file(self, context): + """Makes tmpfile under FLAGS.instance_path. + + This method enables compute nodes to recognize that they mounts + same shared storage. (create|check|creanup)_shared_storage_test_file() + is a pair. + + :param context: security context + :returns: tmpfile name(basename) + + """ + + dirpath = FLAGS.instances_path + fd, tmp_file = tempfile.mkstemp(dir=dirpath) + LOG.debug(_("Creating tmpfile %s to notify to other " + "compute nodes that they should mount " + "the same storage.") % tmp_file) + os.close(fd) + return os.path.basename(tmp_file) + + @exception.wrap_exception + def check_shared_storage_test_file(self, context, filename): + """Confirms existence of the tmpfile under FLAGS.instances_path. + + :param context: security context + :param filename: confirm existence of FLAGS.instances_path/thisfile + + """ + + tmp_file = os.path.join(FLAGS.instances_path, filename) + if not os.path.exists(tmp_file): + raise exception.NotFound(_('%s not found') % tmp_file) + + @exception.wrap_exception + def cleanup_shared_storage_test_file(self, context, filename): + """Removes existence of the tmpfile under FLAGS.instances_path. + + :param context: security context + :param filename: remove existence of FLAGS.instances_path/thisfile + + """ + + tmp_file = os.path.join(FLAGS.instances_path, filename) + os.remove(tmp_file) + + @exception.wrap_exception + def update_available_resource(self, context): + """See comments update_resource_info. + + :param context: security context + :returns: See driver.update_available_resource() + + """ + + return self.driver.update_available_resource(context, self.host) + + def pre_live_migration(self, context, instance_id): + """Preparations for live migration at dest host. + + :param context: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + + """ + + # Getting instance info + instance_ref = self.db.instance_get(context, instance_id) + ec2_id = instance_ref['hostname'] + + # Getting fixed ips + fixed_ip = self.db.instance_get_fixed_address(context, instance_id) + if not fixed_ip: + msg = _("%(instance_id)s(%(ec2_id)s) does not have fixed_ip.") + raise exception.NotFound(msg % locals()) + + # If any volume is mounted, prepare here. + if not instance_ref['volumes']: + LOG.info(_("%s has no volume."), ec2_id) + else: + for v in instance_ref['volumes']: + self.volume_manager.setup_compute_volume(context, v['id']) + + # Bridge settings. + # Call this method prior to ensure_filtering_rules_for_instance, + # since bridge is not set up, ensure_filtering_rules_for instance + # fails. + # + # Retry operation is necessary because continuously request comes, + # concorrent request occurs to iptables, then it complains. + max_retry = FLAGS.live_migration_retry_count + for cnt in range(max_retry): + try: + self.network_manager.setup_compute_network(context, + instance_id) + break + except exception.ProcessExecutionError: + if cnt == max_retry - 1: + raise + else: + LOG.warn(_("setup_compute_network() failed %(cnt)d." + "Retry up to %(max_retry)d for %(ec2_id)s.") + % locals()) + time.sleep(1) + + # Creating filters to hypervisors and firewalls. + # An example is that nova-instance-instance-xxx, + # which is written to libvirt.xml(Check "virsh nwfilter-list") + # This nwfilter is necessary on the destination host. + # In addition, this method is creating filtering rule + # onto destination host. + self.driver.ensure_filtering_rules_for_instance(instance_ref) + + def live_migration(self, context, instance_id, dest): + """Executing live migration. + + :param context: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param dest: destination host + + """ + + # Get instance for error handling. + instance_ref = self.db.instance_get(context, instance_id) + i_name = instance_ref.name + + try: + # Checking volume node is working correctly when any volumes + # are attached to instances. + if instance_ref['volumes']: + rpc.call(context, + FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': instance_id}}) + + # Asking dest host to preparing live migration. + rpc.call(context, + self.db.queue_get_for(context, FLAGS.compute_topic, dest), + {"method": "pre_live_migration", + "args": {'instance_id': instance_id}}) + + except Exception: + msg = _("Pre live migration for %(i_name)s failed at %(dest)s") + LOG.error(msg % locals()) + self.recover_live_migration(context, instance_ref) + raise + + # Executing live migration + # live_migration might raises exceptions, but + # nothing must be recovered in this version. + self.driver.live_migration(context, instance_ref, dest, + self.post_live_migration, + self.recover_live_migration) + + def post_live_migration(self, ctxt, instance_ref, dest): + """Post operations for live migration. + + This method is called from live_migration + and mainly updating database record. + + :param ctxt: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param dest: destination host + + """ + + LOG.info(_('post_live_migration() is started..')) + instance_id = instance_ref['id'] + + # Detaching volumes. + try: + for vol in self.db.volume_get_all_by_instance(ctxt, instance_id): + self.volume_manager.remove_compute_volume(ctxt, vol['id']) + except exception.NotFound: + pass + + # Releasing vlan. + # (not necessary in current implementation?) + + # Releasing security group ingress rule. + self.driver.unfilter_instance(instance_ref) + + # Database updating. + i_name = instance_ref.name + try: + # Not return if floating_ip is not found, otherwise, + # instance never be accessible.. + floating_ip = self.db.instance_get_floating_address(ctxt, + instance_id) + if not floating_ip: + LOG.info(_('No floating_ip is found for %s.'), i_name) + else: + floating_ip_ref = self.db.floating_ip_get_by_address(ctxt, + floating_ip) + self.db.floating_ip_update(ctxt, + floating_ip_ref['address'], + {'host': dest}) + except exception.NotFound: + LOG.info(_('No floating_ip is found for %s.'), i_name) + except: + LOG.error(_("Live migration: Unexpected error:" + "%s cannot inherit floating ip..") % i_name) + + # Restore instance/volume state + self.recover_live_migration(ctxt, instance_ref, dest) + + LOG.info(_('Migrating %(i_name)s to %(dest)s finished successfully.') + % locals()) + LOG.info(_("You may see the error \"libvirt: QEMU error: " + "Domain not found: no domain with matching name.\" " + "This error can be safely ignored.")) + + def recover_live_migration(self, ctxt, instance_ref, host=None): + """Recovers Instance/volume state from migrating -> running. + + :param ctxt: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param host: + DB column value is updated by this hostname. + if none, the host instance currently running is selected. + + """ + + if not host: + host = instance_ref['host'] + + self.db.instance_update(ctxt, + instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, + 'host': host}) + + for volume in instance_ref['volumes']: + self.db.volume_update(ctxt, volume['id'], {'status': 'in-use'}) diff --git a/nova/console/manager.py b/nova/console/manager.py index 57c75cf4f..bfa571ea9 100644 --- a/nova/console/manager.py +++ b/nova/console/manager.py @@ -69,7 +69,7 @@ class ConsoleProxyManager(manager.Manager): except exception.NotFound: logging.debug(_("Adding console")) if not password: - password = self.driver.generate_password() + password = utils.generate_password(8) if not port: port = self.driver.get_port(context) console_data = {'instance_name': name, diff --git a/nova/console/xvp.py b/nova/console/xvp.py index cd257e0a6..0cedfbb13 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -91,10 +91,6 @@ class XVPConsoleProxy(object): """Trim password to length, and encode""" return self._xvp_encrypt(password) - def generate_password(self, length=8): - """Returns random console password""" - return os.urandom(length * 2).encode('base64')[:length] - def _rebuild_xvp_conf(self, context): logging.debug(_("Rebuilding xvp conf")) pools = [pool for pool in @@ -133,10 +129,10 @@ class XVPConsoleProxy(object): return logging.debug(_("Starting xvp")) try: - utils.execute('xvp -p %s -c %s -l %s' % - (FLAGS.console_xvp_pid, - FLAGS.console_xvp_conf, - FLAGS.console_xvp_log)) + utils.execute('xvp', + '-p', FLAGS.console_xvp_pid, + '-c', FLAGS.console_xvp_conf, + '-l', FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) @@ -190,5 +186,5 @@ class XVPConsoleProxy(object): flag = '-x' #xvp will blow up on passwords that are too long (mdragon) password = password[:maxlen] - out, err = utils.execute('xvp %s' % flag, process_input=password) + out, err = utils.execute('xvp', flag, process_input=password) return out.strip() diff --git a/nova/crypto.py b/nova/crypto.py index a34b940f5..2a8d4abca 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,10 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen -q -b %d -N "" -f %s' % (bits, keyfile)) - (out, err) = utils.execute('ssh-keygen -q -l -f %s.pub' % (keyfile)) + utils.execute('ssh-keygen', '-q', '-b', bits, '-N', '', + '-f', keyfile) + (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', + '%s.pub' % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +120,8 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen -y -f /dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', + # '/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,9 +146,10 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute("openssl ca -config ./openssl.cnf -revoke '%s'" % file_name) - utils.execute("openssl ca -gencrl -config ./openssl.cnf -out '%s'" % - FLAGS.crl_file) + utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', + file_name) + utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', + '-out', FLAGS.crl_file) os.chdir(start) @@ -193,9 +197,9 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.execute("openssl req -new -key %s -out %s -batch -subj %s" % - (keyfile, csrfile, subject)) + utils.execute('openssl', 'genrsa', '-out', keyfile, str(bits)) + utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, + '-batch', '-subj', subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -212,8 +216,8 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute("sh geninter.sh %s %s" % - (project_id, _project_cert_subject(project_id))) + utils.execute('sh', 'geninter.sh', project_id, + _project_cert_subject(project_id)) os.chdir(start) @@ -228,8 +232,8 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute("sh genvpn.sh %s %s" % - (project_id, _vpn_cert_subject(project_id))) + utils.execute('sh', 'genvpn.sh', + project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() (serial, signed_csr) = sign_csr(csr_text, project_id) @@ -259,9 +263,10 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute("openssl ca -batch -out %s -config " - "./openssl.cnf -infiles %s" % (outbound, inbound)) - out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) + utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config', + './openssl.cnf', '-infiles', inbound) + out, _err = utils.execute('openssl', 'x509', '-in', outbound, + '-serial', '-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: diff --git a/nova/db/api.py b/nova/db/api.py index d56d6f404..47a8ca1cb 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -79,33 +79,6 @@ class NoMoreTargets(exception.Error): ################### -def reroute_if_not_found(key_args_index=None): - """Decorator used to indicate that the method should throw - a RouteRedirectException if the query can't find anything. - """ - def wrap(f): - def wrapped_f(*args, **kwargs): - try: - return f(*args, **kwargs) - except exception.InstanceNotFound, e: - context = args[0] - key = None - if key_args_index: - key = args[key_args_index] - LOG.debug(_("Instance %(key)s not found locally: '%(e)s'" % - locals())) - - # Throw a reroute Exception for the middleware to pick up. - LOG.debug("Firing ZoneRouteException") - zones = zone_get_all(context) - raise exception.ZoneRouteException(zones, e) - return wrapped_f - return wrap - - -################### - - def service_destroy(context, instance_id): """Destroy the service or raise if it does not exist.""" return IMPL.service_destroy(context, instance_id) @@ -136,6 +109,11 @@ def service_get_all_by_host(context, host): return IMPL.service_get_all_by_host(context, host) +def service_get_all_compute_by_host(context, host): + """Get all compute services for a given host.""" + return IMPL.service_get_all_compute_by_host(context, host) + + def service_get_all_compute_sorted(context): """Get all compute services sorted by instance count. @@ -185,6 +163,29 @@ def service_update(context, service_id, values): ################### +def compute_node_get(context, compute_id, session=None): + """Get an computeNode or raise if it does not exist.""" + return IMPL.compute_node_get(context, compute_id) + + +def compute_node_create(context, values): + """Create a computeNode from the values dictionary.""" + return IMPL.compute_node_create(context, values) + + +def compute_node_update(context, compute_id, values): + """Set the given properties on an computeNode and update it. + + Raises NotFound if computeNode does not exist. + + """ + + return IMPL.compute_node_update(context, compute_id, values) + + +################### + + def certificate_create(context, values): """Create a certificate from the values dictionary.""" return IMPL.certificate_create(context, values) @@ -289,6 +290,11 @@ def floating_ip_get_by_address(context, address): return IMPL.floating_ip_get_by_address(context, address) +def floating_ip_update(context, address, values): + """Update a floating ip by address or raise if it doesn't exist.""" + return IMPL.floating_ip_update(context, address, values) + + #################### def migration_update(context, id, values): @@ -352,6 +358,11 @@ def fixed_ip_get_all(context): return IMPL.fixed_ip_get_all(context) +def fixed_ip_get_all_by_host(context, host): + """Get all defined fixed ips used by a host.""" + return IMPL.fixed_ip_get_all_by_host(context, host) + + def fixed_ip_get_by_address(context, address): """Get a fixed ip by address or raise if it does not exist.""" return IMPL.fixed_ip_get_by_address(context, address) @@ -399,7 +410,6 @@ def instance_destroy(context, instance_id): return IMPL.instance_destroy(context, instance_id) -@reroute_if_not_found(key_args_index=1) def instance_get(context, instance_id): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) @@ -474,6 +484,27 @@ def instance_add_security_group(context, instance_id, security_group_id): security_group_id) +def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): + """Get instances.vcpus by host and project.""" + return IMPL.instance_get_vcpu_sum_by_host_and_project(context, + hostname, + proj_id) + + +def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): + """Get amount of memory by host and project.""" + return IMPL.instance_get_memory_sum_by_host_and_project(context, + hostname, + proj_id) + + +def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): + """Get total amount of disk by host and project.""" + return IMPL.instance_get_disk_sum_by_host_and_project(context, + hostname, + proj_id) + + def instance_action_create(context, values): """Create an instance action from the values dictionary.""" return IMPL.instance_action_create(context, values) @@ -550,6 +581,13 @@ def network_create_safe(context, values): return IMPL.network_create_safe(context, values) +def network_delete_safe(context, network_id): + """Delete network with key network_id. + This method assumes that the network is not associated with any project + """ + return IMPL.network_delete_safe(context, network_id) + + def network_create_fixed_ips(context, network_id, num_vpn_clients): """Create the ips for the network, reserving sepecified ips.""" return IMPL.network_create_fixed_ips(context, network_id, num_vpn_clients) @@ -586,6 +624,11 @@ def network_get_by_bridge(context, bridge): return IMPL.network_get_by_bridge(context, bridge) +def network_get_by_cidr(context, cidr): + """Get a network by cidr or raise if it does not exist""" + return IMPL.network_get_by_cidr(context, cidr) + + def network_get_by_instance(context, instance_id): """Get a network by instance id or raise if it does not exist.""" return IMPL.network_get_by_instance(context, instance_id) @@ -786,6 +829,11 @@ def volume_get_all_by_host(context, host): return IMPL.volume_get_all_by_host(context, host) +def volume_get_all_by_instance(context, instance_id): + """Get all volumes belonging to a instance.""" + return IMPL.volume_get_all_by_instance(context, instance_id) + + def volume_get_all_by_project(context, project_id): """Get all volumes belonging to a project.""" return IMPL.volume_get_all_by_project(context, project_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 5e498fc6f..44540617f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -34,6 +34,7 @@ from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload_all from sqlalchemy.sql import exists from sqlalchemy.sql import func +from sqlalchemy.sql.expression import literal_column FLAGS = flags.FLAGS @@ -118,6 +119,11 @@ def service_destroy(context, service_id): service_ref = service_get(context, service_id, session=session) service_ref.delete(session=session) + if service_ref.topic == 'compute' and \ + len(service_ref.compute_node) != 0: + for c in service_ref.compute_node: + c.delete(session=session) + @require_admin_context def service_get(context, service_id, session=None): @@ -125,6 +131,7 @@ def service_get(context, service_id, session=None): session = get_session() result = session.query(models.Service).\ + options(joinedload('compute_node')).\ filter_by(id=service_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -175,6 +182,24 @@ def service_get_all_by_host(context, host): @require_admin_context +def service_get_all_compute_by_host(context, host): + topic = 'compute' + session = get_session() + result = session.query(models.Service).\ + options(joinedload('compute_node')).\ + filter_by(deleted=False).\ + filter_by(host=host).\ + filter_by(topic=topic).\ + all() + + if not result: + raise exception.NotFound(_("%s does not exist or is not " + "a compute node.") % host) + + return result + + +@require_admin_context def _service_get_all_topic_subquery(context, session, topic, subq, label): sort_value = getattr(subq.c, label) return session.query(models.Service, func.coalesce(sort_value, 0)).\ @@ -285,6 +310,42 @@ def service_update(context, service_id, values): @require_admin_context +def compute_node_get(context, compute_id, session=None): + if not session: + session = get_session() + + result = session.query(models.ComputeNode).\ + filter_by(id=compute_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + + if not result: + raise exception.NotFound(_('No computeNode for id %s') % compute_id) + + return result + + +@require_admin_context +def compute_node_create(context, values): + compute_node_ref = models.ComputeNode() + compute_node_ref.update(values) + compute_node_ref.save() + return compute_node_ref + + +@require_admin_context +def compute_node_update(context, compute_id, values): + session = get_session() + with session.begin(): + compute_ref = compute_node_get(context, compute_id, session=session) + compute_ref.update(values) + compute_ref.save(session=session) + + +################### + + +@require_admin_context def certificate_get(context, certificate_id, session=None): if not session: session = get_session() @@ -505,6 +566,16 @@ def floating_ip_get_by_address(context, address, session=None): return result +@require_context +def floating_ip_update(context, address, values): + session = get_session() + with session.begin(): + floating_ip_ref = floating_ip_get_by_address(context, address, session) + for (key, value) in values.iteritems(): + floating_ip_ref[key] = value + floating_ip_ref.save(session=session) + + ################### @@ -577,18 +648,17 @@ def fixed_ip_disassociate(context, address): @require_admin_context def fixed_ip_disassociate_all_by_timeout(_context, host, time): session = get_session() - # NOTE(vish): The nested select is because sqlite doesn't support - # JOINs in UPDATEs. - result = session.execute('UPDATE fixed_ips SET instance_id = NULL, ' - 'leased = 0 ' - 'WHERE network_id IN (SELECT id FROM networks ' - 'WHERE host = :host) ' - 'AND updated_at < :time ' - 'AND instance_id IS NOT NULL ' - 'AND allocated = 0', - {'host': host, - 'time': time}) - return result.rowcount + inner_q = session.query(models.Network.id).\ + filter_by(host=host).\ + subquery() + result = session.query(models.FixedIp).\ + filter(models.FixedIp.network_id.in_(inner_q)).\ + filter(models.FixedIp.updated_at < time).\ + filter(models.FixedIp.instance_id != None).\ + filter_by(allocated=0).\ + update({'instance_id': None, + 'leased': 0}) + return result @require_admin_context @@ -602,6 +672,22 @@ def fixed_ip_get_all(context, session=None): return result +@require_admin_context +def fixed_ip_get_all_by_host(context, host=None): + session = get_session() + + result = session.query(models.FixedIp).\ + join(models.FixedIp.instance).\ + filter_by(state=1).\ + filter_by(host=host).\ + all() + + if not result: + raise exception.NotFound(_('No fixed ips for this host defined')) + + return result + + @require_context def fixed_ip_get_by_address(context, address, session=None): if not session: @@ -701,14 +787,16 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.execute('update instances set deleted=1,' - 'deleted_at=:at where id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) - session.execute('update security_group_instance_association ' - 'set deleted=1,deleted_at=:at where instance_id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(instance_id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -905,6 +993,45 @@ def instance_add_security_group(context, instance_id, security_group_id): @require_context +def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.vcpus)) + if not result: + return 0 + return result + + +@require_context +def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.memory_mb)) + if not result: + return 0 + return result + + +@require_context +def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.local_gb)) + if not result: + return 0 + return result + + +@require_context def instance_action_create(context, values): """Create an instance action from the values dictionary.""" action_ref = models.InstanceActions() @@ -950,9 +1077,11 @@ def key_pair_destroy_all_by_user(context, user_id): authorize_user_context(context, user_id) session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update key_pairs set deleted=1 where user_id=:id', - {'id': user_id}) + session.query(models.KeyPair).\ + filter_by(user_id=user_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1055,6 +1184,15 @@ def network_create_safe(context, values): @require_admin_context +def network_delete_safe(context, network_id): + session = get_session() + with session.begin(): + network_ref = network_get(context, network_id=network_id, \ + session=session) + session.delete(network_ref) + + +@require_admin_context def network_disassociate(context, network_id): network_update(context, network_id, {'project_id': None, 'host': None}) @@ -1063,7 +1201,9 @@ def network_disassociate(context, network_id): @require_admin_context def network_disassociate_all(context): session = get_session() - session.execute('update networks set project_id=NULL') + session.query(models.Network).\ + update({'project_id': None, + 'updated_at': literal_column('updated_at')}) @require_context @@ -1128,6 +1268,18 @@ def network_get_by_bridge(context, bridge): @require_admin_context +def network_get_by_cidr(context, cidr): + session = get_session() + result = session.query(models.Network).\ + filter_by(cidr=cidr).first() + + if not result: + raise exception.NotFound(_('Network with cidr %s does not exist') % + cidr) + return result + + +@require_admin_context def network_get_by_instance(_context, instance_id): session = get_session() rv = session.query(models.Network).\ @@ -1433,15 +1585,17 @@ def volume_data_get_for_project(context, project_id): def volume_destroy(context, volume_id): session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update volumes set deleted=1 where id=:id', - {'id': volume_id}) - session.execute('update export_devices set volume_id=NULL ' - 'where volume_id=:id', - {'id': volume_id}) - session.execute('update iscsi_targets set volume_id=NULL ' - 'where volume_id=:id', - {'id': volume_id}) + session.query(models.Volume).\ + filter_by(id=volume_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + session.query(models.ExportDevice).\ + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) + session.query(models.IscsiTarget).\ + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) @require_admin_context @@ -1501,6 +1655,18 @@ def volume_get_all_by_host(context, host): all() +@require_admin_context +def volume_get_all_by_instance(context, instance_id): + session = get_session() + result = session.query(models.Volume).\ + filter_by(instance_id=instance_id).\ + filter_by(deleted=False).\ + all() + if not result: + raise exception.NotFound(_('No volume for instance %s') % instance_id) + return result + + @require_context def volume_get_all_by_project(context, project_id): authorize_project_context(context, project_id) @@ -1661,17 +1827,21 @@ def security_group_create(context, values): def security_group_destroy(context, security_group_id): session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update security_groups set deleted=1 where id=:id', - {'id': security_group_id}) - session.execute('update security_group_instance_association ' - 'set deleted=1,deleted_at=:at ' - 'where security_group_id=:id', - {'id': security_group_id, - 'at': datetime.datetime.utcnow()}) - session.execute('update security_group_rules set deleted=1 ' - 'where group_id=:id', - {'id': security_group_id}) + session.query(models.SecurityGroup).\ + filter_by(id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(security_group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + session.query(models.SecurityGroupIngressRule).\ + filter_by(group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1679,9 +1849,14 @@ def security_group_destroy_all(context, session=None): if not session: session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update security_groups set deleted=1') - session.execute('update security_group_rules set deleted=1') + session.query(models.SecurityGroup).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + session.query(models.SecurityGroupIngressRule).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) ################### @@ -1810,12 +1985,15 @@ def user_create(_context, values): def user_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association ' - 'where user_id=:id', {'id': id}) - session.execute('delete from user_role_association ' - 'where user_id=:id', {'id': id}) - session.execute('delete from user_project_role_association ' - 'where user_id=:id', {'id': id}) + session.query(models.UserProjectAssociation).\ + filter_by(user_id=id).\ + delete() + session.query(models.UserRoleAssociation).\ + filter_by(user_id=id).\ + delete() + session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=id).\ + delete() user_ref = user_get(context, id, session=session) session.delete(user_ref) @@ -1872,8 +2050,11 @@ def project_get_by_user(context, user_id): session = get_session() user = session.query(models.User).\ filter_by(deleted=can_read_deleted(context)).\ + filter_by(id=user_id).\ options(joinedload_all('projects')).\ first() + if not user: + raise exception.NotFound(_('Invalid user_id %s') % user_id) return user.projects @@ -1906,10 +2087,12 @@ def project_update(context, project_id, values): def project_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association ' - 'where project_id=:id', {'id': id}) - session.execute('delete from user_project_role_association ' - 'where project_id=:id', {'id': id}) + session.query(models.UserProjectAssociation).\ + filter_by(project_id=id).\ + delete() + session.query(models.UserProjectRoleAssociation).\ + filter_by(project_id=id).\ + delete() project_ref = project_get(context, id, session=session) session.delete(project_ref) @@ -1934,11 +2117,11 @@ def user_get_roles_for_project(context, user_id, project_id): def user_remove_project_role(context, user_id, project_id, role): session = get_session() with session.begin(): - session.execute('delete from user_project_role_association where ' - 'user_id=:user_id and project_id=:project_id and ' - 'role=:role', {'user_id': user_id, - 'project_id': project_id, - 'role': role}) + session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(role=role).\ + delete() def user_remove_role(context, user_id, role): @@ -2089,8 +2272,9 @@ def console_delete(context, console_id): session = get_session() with session.begin(): # consoles are meant to be transient. (mdragon) - session.execute('delete from consoles ' - 'where id=:id', {'id': console_id}) + session.query(models.Console).\ + filter_by(id=console_id).\ + delete() def console_get_by_pool_instance(context, pool_id, instance_id): @@ -2246,8 +2430,9 @@ def zone_update(context, zone_id, values): def zone_delete(context, zone_id): session = get_session() with session.begin(): - session.execute('delete from zones ' - 'where id=:id', {'id': zone_id}) + session.query(models.Zone).\ + filter_by(id=zone_id).\ + delete() @require_admin_context diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py new file mode 100644 index 000000000..eb3066894 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from sqlalchemy.sql import text +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) + migrate_engine.execute(instances.update()\ + .where(instances.c.os_type == None)\ + .values(os_type='linux')) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances.drop_column('os_type') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/011_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/011_live_migration.py new file mode 100644 index 000000000..23ccccb4e --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/011_live_migration.py @@ -0,0 +1,83 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate import * +from nova import log as logging +from sqlalchemy import * + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +compute_nodes = Table('compute_nodes', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('service_id', Integer(), nullable=False), + + Column('vcpus', Integer(), nullable=False), + Column('memory_mb', Integer(), nullable=False), + Column('local_gb', Integer(), nullable=False), + Column('vcpus_used', Integer(), nullable=False), + Column('memory_mb_used', Integer(), nullable=False), + Column('local_gb_used', Integer(), nullable=False), + Column('hypervisor_type', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('hypervisor_version', Integer(), nullable=False), + Column('cpu_info', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + ) + + +# +# Tables to alter +# +instances_launched_on = Column( + 'launched_on', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + try: + compute_nodes.create() + except Exception: + logging.info(repr(compute_nodes)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[compute_nodes]) + raise + + instances.create_column(instances_launched_on) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 6ef284e65..1845e85eb 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -113,6 +113,41 @@ class Service(BASE, NovaBase): availability_zone = Column(String(255), default='nova') +class ComputeNode(BASE, NovaBase): + """Represents a running compute service on a host.""" + + __tablename__ = 'compute_nodes' + id = Column(Integer, primary_key=True) + service_id = Column(Integer, ForeignKey('services.id'), nullable=True) + service = relationship(Service, + backref=backref('compute_node'), + foreign_keys=service_id, + primaryjoin='and_(' + 'ComputeNode.service_id == Service.id,' + 'ComputeNode.deleted == False)') + + vcpus = Column(Integer, nullable=True) + memory_mb = Column(Integer, nullable=True) + local_gb = Column(Integer, nullable=True) + vcpus_used = Column(Integer, nullable=True) + memory_mb_used = Column(Integer, nullable=True) + local_gb_used = Column(Integer, nullable=True) + hypervisor_type = Column(Text, nullable=True) + hypervisor_version = Column(Integer, nullable=True) + + # Note(masumotok): Expected Strings example: + # + # '{"arch":"x86_64", + # "model":"Nehalem", + # "topology":{"sockets":1, "threads":2, "cores":3}, + # "features":["tdtscp", "xtpr"]}' + # + # Points are "json translatable" and it must have all dictionary keys + # above, since it is copied from <cpu> tag of getCapabilities() + # (See libvirt.virtConnection). + cpu_info = Column(Text, nullable=True) + + class Certificate(BASE, NovaBase): """Represents a an x509 certificate""" __tablename__ = 'certificates' @@ -126,7 +161,7 @@ class Certificate(BASE, NovaBase): class Instance(BASE, NovaBase): """Represents a guest vm.""" __tablename__ = 'instances' - onset_files = [] + injected_files = [] id = Column(Integer, primary_key=True, autoincrement=True) @@ -191,8 +226,13 @@ class Instance(BASE, NovaBase): display_name = Column(String(255)) display_description = Column(String(255)) + # To remember on which host a instance booted. + # An instance may have moved to another host by live migraiton. + launched_on = Column(Text) locked = Column(Boolean) + os_type = Column(String(255)) + # TODO(vish): see Ewan's email about state improvements, probably # should be in a driver base class or some such # vmstate_state = running, halted, suspended, paused diff --git a/nova/exception.py b/nova/exception.py index cfed32a72..93c5fe3d7 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -88,16 +88,12 @@ class InvalidInputException(Error): pass -class TimeoutException(Error): +class InvalidContentType(Error): pass -class ZoneRouteException(Error): - """Thrown in API to reroute request to child zones.""" - def __init__(self, zones, original_exception, *args, **kwargs): - self.zones = zones - self.original_exception = original_exception - super(ZoneRouteException, self).__init__(*args, **kwargs) +class TimeoutException(Error): + pass class DBError(Error): diff --git a/nova/flags.py b/nova/flags.py index 7036180fc..3a8ec1a39 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -321,6 +321,8 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), "Top-level directory for maintaining nova's state") +DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'), + "Directory for lock files") DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') @@ -346,7 +348,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.s3.S3ImageService', +DEFINE_string('image_service', 'nova.image.local.LocalImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), @@ -356,5 +358,6 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'hypervisor:xenserver;os:linux', - 'Key/Value tags which represent capabilities of this zone') +DEFINE_list('zone_capabilities', + ['hypervisor=xenserver;kvm', 'os=linux;windows'], + 'Key/Multi-value list representng capabilities of this zone') diff --git a/nova/image/glance.py b/nova/image/glance.py index 593c4bce6..15fca69b8 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -17,9 +17,8 @@ """Implementation of an image service that uses Glance as the backend""" from __future__ import absolute_import -import httplib -import json -import urlparse + +from glance.common import exception as glance_exception from nova import exception from nova import flags @@ -53,31 +52,64 @@ class GlanceImageService(service.BaseImageService): """ return self.client.get_images_detailed() - def show(self, context, id): + def show(self, context, image_id): """ Returns a dict containing image data for the given opaque image id. """ - image = self.client.get_image_meta(id) - if image: - return image - raise exception.NotFound + try: + image = self.client.get_image_meta(image_id) + except glance_exception.NotFound: + raise exception.NotFound + return image - def create(self, context, data): + def show_by_name(self, context, name): + """ + Returns a dict containing image data for the given name. + """ + # TODO(vish): replace this with more efficient call when glance + # supports it. + images = self.detail(context) + image = None + for cantidate in images: + if name == cantidate.get('name'): + image = cantidate + break + if image is None: + raise exception.NotFound + return image + + def get(self, context, image_id, data): + """ + Calls out to Glance for metadata and data and writes data. + """ + try: + metadata, image_chunks = self.client.get_image(image_id) + except glance_exception.NotFound: + raise exception.NotFound + for chunk in image_chunks: + data.write(chunk) + return metadata + + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. :raises AlreadyExists if the image already exist. """ - return self.client.add_image(image_meta=data) + return self.client.add_image(metadata, data) - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. :raises NotFound if the image does not exist. """ - return self.client.update_image(image_id, data) + try: + result = self.client.update_image(image_id, metadata, data) + except glance_exception.NotFound: + raise exception.NotFound + return result def delete(self, context, image_id): """ @@ -86,7 +118,11 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - return self.client.delete_image(image_id) + try: + result = self.client.delete_image(image_id) + except glance_exception.NotFound: + raise exception.NotFound + return result def delete_all(self): """ diff --git a/nova/image/local.py b/nova/image/local.py index f78b9aa89..c4ac3baaa 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -15,57 +15,110 @@ # License for the specific language governing permissions and limitations # under the License. -import cPickle as pickle +import json import os.path import random -import tempfile +import shutil +from nova import flags from nova import exception from nova.image import service -class LocalImageService(service.BaseImageService): +FLAGS = flags.FLAGS +flags.DEFINE_string('images_path', '$state_path/images', + 'path to decrypted images') + +class LocalImageService(service.BaseImageService): """Image service storing images to local disk. + It assumes that image_ids are integers. """ def __init__(self): - self._path = tempfile.mkdtemp() + self._path = FLAGS.images_path - def _path_to(self, image_id): - return os.path.join(self._path, str(image_id)) + def _path_to(self, image_id, fname='info.json'): + if fname: + return os.path.join(self._path, '%08x' % int(image_id), fname) + return os.path.join(self._path, '%08x' % int(image_id)) def _ids(self): """The list of all image ids.""" - return [int(i) for i in os.listdir(self._path)] + return [int(i, 16) for i in os.listdir(self._path)] def index(self, context): - return [dict(id=i['id'], name=i['name']) for i in self.detail(context)] + return [dict(image_id=i['id'], name=i.get('name')) + for i in self.detail(context)] def detail(self, context): - return [self.show(context, id) for id in self._ids()] + images = [] + for image_id in self._ids(): + try: + image = self.show(context, image_id) + images.append(image) + except exception.NotFound: + continue + return images + + def show(self, context, image_id): + try: + with open(self._path_to(image_id)) as metadata_file: + return json.load(metadata_file) + except (IOError, ValueError): + raise exception.NotFound - def show(self, context, id): + def show_by_name(self, context, name): + """Returns a dict containing image data for the given name.""" + # NOTE(vish): Not very efficient, but the local image service + # is for testing so it should be fine. + images = self.detail(context) + image = None + for cantidate in images: + if name == cantidate.get('name'): + image = cantidate + break + if image == None: + raise exception.NotFound + return image + + def get(self, context, image_id, data): + """Get image and metadata.""" try: - return pickle.load(open(self._path_to(id))) - except IOError: + with open(self._path_to(image_id)) as metadata_file: + metadata = json.load(metadata_file) + with open(self._path_to(image_id, 'image')) as image_file: + shutil.copyfileobj(image_file, data) + except (IOError, ValueError): raise exception.NotFound + return metadata - def create(self, context, data): - """Store the image data and return the new image id.""" - id = random.randint(0, 2 ** 31 - 1) - data['id'] = id - self.update(context, id, data) - return id + def create(self, context, metadata, data=None): + """Store the image data and return the new image.""" + image_id = random.randint(0, 2 ** 31 - 1) + image_path = self._path_to(image_id, None) + if not os.path.exists(image_path): + os.mkdir(image_path) + return self.update(context, image_id, metadata, data) - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data.""" + metadata['id'] = image_id try: - pickle.dump(data, open(self._path_to(image_id), 'w')) - except IOError: + if data: + location = self._path_to(image_id, 'image') + with open(location, 'w') as image_file: + shutil.copyfileobj(data, image_file) + # NOTE(vish): update metadata similarly to glance + metadata['status'] = 'active' + metadata['location'] = location + with open(self._path_to(image_id), 'w') as metadata_file: + json.dump(metadata, metadata_file) + except (IOError, ValueError): raise exception.NotFound + return metadata def delete(self, context, image_id): """Delete the given image. @@ -73,18 +126,11 @@ class LocalImageService(service.BaseImageService): """ try: - os.unlink(self._path_to(image_id)) - except IOError: + shutil.rmtree(self._path_to(image_id, None)) + except (IOError, ValueError): raise exception.NotFound def delete_all(self): """Clears out all images in local directory.""" - for id in self._ids(): - os.unlink(self._path_to(id)) - - def delete_imagedir(self): - """Deletes the local directory. - Raises OSError if directory is not empty. - - """ - os.rmdir(self._path) + for image_id in self._ids(): + shutil.rmtree(self._path_to(image_id, None)) diff --git a/nova/image/s3.py b/nova/image/s3.py index 14135a1ee..85a2c651c 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -21,8 +21,13 @@ Proxy AMI-related calls from the cloud controller, to the running objectstore service. """ -import json -import urllib +import binascii +import eventlet +import os +import shutil +import tarfile +import tempfile +from xml.etree import ElementTree import boto.s3.connection @@ -31,84 +36,78 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service +from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS +flags.DEFINE_string('image_decryption_dir', '/tmp', + 'parent dir for tempdir used for image decryption') -def map_s3_to_base(image): - """Convert from S3 format to format defined by BaseImageService.""" - i = {} - i['id'] = image.get('imageId') - i['name'] = image.get('imageId') - i['kernel_id'] = image.get('kernelId') - i['ramdisk_id'] = image.get('ramdiskId') - i['location'] = image.get('imageLocation') - i['owner_id'] = image.get('imageOwnerId') - i['status'] = image.get('imageState') - i['type'] = image.get('type') - i['is_public'] = image.get('isPublic') - i['architecture'] = image.get('architecture') - return i +class S3ImageService(service.BaseImageService): + def __init__(self, service=None, *args, **kwargs): + if service == None: + service = utils.import_object(FLAGS.image_service) + self.service = service + self.service.__init__(*args, **kwargs) + def create(self, context, metadata, data=None): + """metadata['properties'] should contain image_location""" + image = self._s3_create(context, metadata) + return image -class S3ImageService(service.BaseImageService): + def delete(self, context, image_id): + # FIXME(vish): call to show is to check filter + self.show(context, image_id) + self.service.delete(context, image_id) - def modify(self, context, image_id, operation): - self._conn(context).make_request( - method='POST', - bucket='_images', - query_args=self._qs({'image_id': image_id, - 'operation': operation})) - return True - - def update(self, context, image_id, attributes): - """update an image's attributes / info.json""" - attributes.update({"image_id": image_id}) - self._conn(context).make_request( - method='POST', - bucket='_images', - query_args=self._qs(attributes)) - return True - - def register(self, context, image_location): - """ rpc call to register a new image based from a manifest """ - image_id = utils.generate_uid('ami') - self._conn(context).make_request( - method='PUT', - bucket='_images', - query_args=self._qs({'image_location': image_location, - 'image_id': image_id})) - return image_id + def update(self, context, image_id, metadata, data=None): + # FIXME(vish): call to show is to check filter + self.show(context, image_id) + image = self.service.update(context, image_id, metadata, data) + return image def index(self, context): - """Return a list of all images that a user can see.""" - response = self._conn(context).make_request( - method='GET', - bucket='_images') - images = json.loads(response.read()) - return [map_s3_to_base(i) for i in images] + images = self.service.index(context) + # FIXME(vish): index doesn't filter so we do it manually + return self._filter(context, images) + + def detail(self, context): + images = self.service.detail(context) + # FIXME(vish): detail doesn't filter so we do it manually + return self._filter(context, images) + + @classmethod + def _is_visible(cls, context, image): + return (context.is_admin + or context.project_id == image['properties']['owner_id'] + or image['properties']['is_public'] == 'True') + + @classmethod + def _filter(cls, context, images): + filtered = [] + for image in images: + if not cls._is_visible(context, image): + continue + filtered.append(image) + return filtered def show(self, context, image_id): - """return a image object if the context has permissions""" - if FLAGS.connection_type == 'fake': - return {'imageId': 'bar'} - result = self.index(context) - result = [i for i in result if i['id'] == image_id] - if not result: - raise exception.NotFound(_('Image %s could not be found') - % image_id) - image = result[0] + image = self.service.show(context, image_id) + if not self._is_visible(context, image): + raise exception.NotFound return image - def deregister(self, context, image_id): - """ unregister an image """ - self._conn(context).make_request( - method='DELETE', - bucket='_images', - query_args=self._qs({'image_id': image_id})) + def show_by_name(self, context, name): + image = self.service.show_by_name(context, name) + if not self._is_visible(context, image): + raise exception.NotFound + return image - def _conn(self, context): + @staticmethod + def _conn(context): + # TODO(vish): is there a better way to get creds to sign + # for the user? access = manager.AuthManager().get_access_key(context.user, context.project) secret = str(context.user.secret) @@ -120,8 +119,159 @@ class S3ImageService(service.BaseImageService): port=FLAGS.s3_port, host=FLAGS.s3_host) - def _qs(self, params): - pairs = [] - for key in params.keys(): - pairs.append(key + '=' + urllib.quote(params[key])) - return '&'.join(pairs) + @staticmethod + def _download_file(bucket, filename, local_dir): + key = bucket.get_key(filename) + local_filename = os.path.join(local_dir, filename) + key.get_contents_to_filename(local_filename) + return local_filename + + def _s3_create(self, context, metadata): + """Gets a manifext from s3 and makes an image""" + + image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir) + + image_location = metadata['properties']['image_location'] + bucket_name = image_location.split("/")[0] + manifest_path = image_location[len(bucket_name) + 1:] + bucket = self._conn(context).get_bucket(bucket_name) + key = bucket.get_key(manifest_path) + manifest = key.get_contents_as_string() + + manifest = ElementTree.fromstring(manifest) + image_format = 'ami' + image_type = 'machine' + + try: + kernel_id = manifest.find("machine_configuration/kernel_id").text + if kernel_id == 'true': + image_format = 'aki' + image_type = 'kernel' + kernel_id = None + except Exception: + kernel_id = None + + try: + ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text + if ramdisk_id == 'true': + image_format = 'ari' + image_type = 'ramdisk' + ramdisk_id = None + except Exception: + ramdisk_id = None + + try: + arch = manifest.find("machine_configuration/architecture").text + except Exception: + arch = 'x86_64' + + properties = metadata['properties'] + properties['owner_id'] = context.project_id + properties['architecture'] = arch + + if kernel_id: + properties['kernel_id'] = ec2utils.ec2_id_to_id(kernel_id) + + if ramdisk_id: + properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id) + + properties['is_public'] = False + properties['type'] = image_type + metadata.update({'disk_format': image_format, + 'container_format': image_format, + 'status': 'queued', + 'is_public': True, + 'properties': properties}) + metadata['properties']['image_state'] = 'pending' + image = self.service.create(context, metadata) + image_id = image['id'] + + def delayed_create(): + """This handles the fetching and decrypting of the part files.""" + parts = [] + for fn_element in manifest.find("image").getiterator("filename"): + part = self._download_file(bucket, fn_element.text, image_path) + parts.append(part) + + # NOTE(vish): this may be suboptimal, should we use cat? + encrypted_filename = os.path.join(image_path, 'image.encrypted') + with open(encrypted_filename, 'w') as combined: + for filename in parts: + with open(filename) as part: + shutil.copyfileobj(part, combined) + + metadata['properties']['image_state'] = 'decrypting' + self.service.update(context, image_id, metadata) + + hex_key = manifest.find("image/ec2_encrypted_key").text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find("image/ec2_encrypted_iv").text + encrypted_iv = binascii.a2b_hex(hex_iv) + + # FIXME(vish): grab key from common service so this can run on + # any host. + cloud_pk = os.path.join(FLAGS.ca_path, "private/cakey.pem") + + decrypted_filename = os.path.join(image_path, 'image.tar.gz') + self._decrypt_image(encrypted_filename, encrypted_key, + encrypted_iv, cloud_pk, decrypted_filename) + + metadata['properties']['image_state'] = 'untarring' + self.service.update(context, image_id, metadata) + + unz_filename = self._untarzip_image(image_path, decrypted_filename) + + metadata['properties']['image_state'] = 'uploading' + with open(unz_filename) as image_file: + self.service.update(context, image_id, metadata, image_file) + metadata['properties']['image_state'] = 'available' + self.service.update(context, image_id, metadata) + + shutil.rmtree(image_path) + + eventlet.spawn_n(delayed_create) + + return image + + @staticmethod + def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename): + key, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt private key: %s") + % err) + iv, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt initialization " + "vector: %s") % err) + + _out, err = utils.execute('openssl', 'enc', + '-d', '-aes-128-cbc', + '-in', '%s' % (encrypted_filename,), + '-K', '%s' % (key,), + '-iv', '%s' % (iv,), + '-out', '%s' % (decrypted_filename,), + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt image file " + "%(image_file)s: %(err)s") % + {'image_file': encrypted_filename, + 'err': err}) + + @staticmethod + def _untarzip_image(path, filename): + tar_file = tarfile.open(filename, "r|gz") + tar_file.extractall(path) + image_file = tar_file.getnames()[0] + tar_file.close() + return os.path.join(path, image_file) diff --git a/nova/image/service.py b/nova/image/service.py index ebee2228d..c09052cab 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -56,9 +56,9 @@ class BaseImageService(object): """ raise NotImplementedError - def show(self, context, id): + def show(self, context, image_id): """ - Returns a dict containing image data for the given opaque image id. + Returns a dict containing image metadata for the given opaque image id. :retval a mapping with the following signature: @@ -76,17 +76,27 @@ class BaseImageService(object): """ raise NotImplementedError - def create(self, context, data): + def get(self, context, data): """ - Store the image data and return the new image id. + Returns a dict containing image metadata and writes image data to data. + + :param data: a file-like object to hold binary image data + + :raises NotFound if the image does not exist + """ + raise NotImplementedError + + def create(self, context, metadata, data=None): + """ + Store the image metadata and data and return the new image id. :raises AlreadyExists if the image already exist. """ raise NotImplementedError - def update(self, context, image_id, data): - """Replace the contents of the given image with the new data. + def update(self, context, image_id, metadata, data=None): + """Update the given image with the new metadata and data. :raises NotFound if the image does not exist. diff --git a/nova/manager.py b/nova/manager.py index 3d38504bd..f384e3f0f 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -53,8 +53,9 @@ This module provides Manager, a base class for managers. from nova import utils from nova import flags +from nova import log as logging from nova.db import base - +from nova.scheduler import api FLAGS = flags.FLAGS @@ -74,3 +75,28 @@ class Manager(base.Base): """Do any initialization that needs to be run if this is a standalone service. Child classes should override this method.""" pass + + +class SchedulerDependentManager(Manager): + """Periodically send capability updates to the Scheduler services. + Services that need to update the Scheduler of their capabilities + should derive from this class. Otherwise they can derive from + manager.Manager directly. Updates are only sent after + update_service_capabilities is called with non-None values.""" + def __init__(self, host=None, db_driver=None, service_name="undefined"): + self.last_capabilities = None + self.service_name = service_name + super(SchedulerDependentManager, self).__init__(host, db_driver) + + def update_service_capabilities(self, capabilities): + """Remember these capabilities to send on next periodic update.""" + self.last_capabilities = capabilities + + def periodic_tasks(self, context=None): + """Pass data back to the scheduler at a periodic interval""" + if self.last_capabilities: + logging.debug(_("Notifying Schedulers of capabilities ...")) + api.API.update_service_capabilities(context, self.service_name, + self.host, self.last_capabilities) + + super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 535ce87bc..7106e6164 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -17,7 +17,11 @@ Implements vlans, bridges, and iptables rules using linux utilities. """ +import inspect import os +import calendar + +from eventlet import semaphore from nova import db from nova import exception @@ -25,7 +29,6 @@ from nova import flags from nova import log as logging from nova import utils - LOG = logging.getLogger("nova.linux_net") @@ -52,10 +55,10 @@ flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') flags.DEFINE_string('routing_source_ip', '$my_ip', 'Public IP of network host') -flags.DEFINE_bool('use_nova_chains', False, - 'use the nova_ routing chains instead of default') flags.DEFINE_string('input_chain', 'INPUT', 'chain to add nova_input to') +flags.DEFINE_integer('dhcp_lease_time', 120, + 'Lifetime of a DHCP lease') flags.DEFINE_string('dns_server', None, 'if set, uses specific dns server for dnsmasq') @@ -63,115 +66,379 @@ flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', 'dmz range that should be accepted') +binary_name = os.path.basename(inspect.stack()[-1][1]) + + +class IptablesRule(object): + """An iptables rule + + You shouldn't need to use this class directly, it's only used by + IptablesManager + """ + def __init__(self, chain, rule, wrap=True, top=False): + self.chain = chain + self.rule = rule + self.wrap = wrap + self.top = top + + def __eq__(self, other): + return ((self.chain == other.chain) and + (self.rule == other.rule) and + (self.top == other.top) and + (self.wrap == other.wrap)) + + def __ne__(self, other): + return not self == other + + def __str__(self): + if self.wrap: + chain = '%s-%s' % (binary_name, self.chain) + else: + chain = self.chain + return '-A %s %s' % (chain, self.rule) + + +class IptablesTable(object): + """An iptables table""" + + def __init__(self): + self.rules = [] + self.chains = set() + self.unwrapped_chains = set() + + def add_chain(self, name, wrap=True): + """Adds a named chain to the table + + The chain name is wrapped to be unique for the component creating + it, so different components of Nova can safely create identically + named chains without interfering with one another. + + At the moment, its wrapped name is <binary name>-<chain name>, + so if nova-compute creates a chain named "OUTPUT", it'll actually + end up named "nova-compute-OUTPUT". + """ + if wrap: + self.chains.add(name) + else: + self.unwrapped_chains.add(name) + + def remove_chain(self, name, wrap=True): + """Remove named chain + + This removal "cascades". All rule in the chain are removed, as are + all rules in other chains that jump to it. + + If the chain is not found, this is merely logged. + """ + if wrap: + chain_set = self.chains + else: + chain_set = self.unwrapped_chains + + if name not in chain_set: + LOG.debug(_("Attempted to remove chain %s which doesn't exist"), + name) + return + + chain_set.remove(name) + self.rules = filter(lambda r: r.chain != name, self.rules) + + if wrap: + jump_snippet = '-j %s-%s' % (binary_name, name) + else: + jump_snippet = '-j %s' % (name,) + + self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) + + def add_rule(self, chain, rule, wrap=True, top=False): + """Add a rule to the table + + This is just like what you'd feed to iptables, just without + the "-A <chain name>" bit at the start. + + However, if you need to jump to one of your wrapped chains, + prepend its name with a '$' which will ensure the wrapping + is applied correctly. + """ + if wrap and chain not in self.chains: + raise ValueError(_("Unknown chain: %r") % chain) + + if '$' in rule: + rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) + + self.rules.append(IptablesRule(chain, rule, wrap, top)) + + def _wrap_target_chain(self, s): + if s.startswith('$'): + return '%s-%s' % (binary_name, s[1:]) + return s + + def remove_rule(self, chain, rule, wrap=True, top=False): + """Remove a rule from a chain + + Note: The rule must be exactly identical to the one that was added. + You cannot switch arguments around like you can with the iptables + CLI tool. + """ + try: + self.rules.remove(IptablesRule(chain, rule, wrap, top)) + except ValueError: + LOG.debug(_("Tried to remove rule that wasn't there:" + " %(chain)r %(rule)r %(wrap)r %(top)r"), + {'chain': chain, 'rule': rule, + 'top': top, 'wrap': wrap}) + + +class IptablesManager(object): + """Wrapper for iptables + + See IptablesTable for some usage docs + + A number of chains are set up to begin with. + + First, nova-filter-top. It's added at the top of FORWARD and OUTPUT. Its + name is not wrapped, so it's shared between the various nova workers. It's + intended for rules that need to live at the top of the FORWARD and OUTPUT + chains. It's in both the ipv4 and ipv6 set of tables. + + For ipv4 and ipv6, the builtin INPUT, OUTPUT, and FORWARD filter chains are + wrapped, meaning that the "real" INPUT chain has a rule that jumps to the + wrapped INPUT chain, etc. Additionally, there's a wrapped chain named + "local" which is jumped to from nova-filter-top. + + For ipv4, the builtin PREROUTING, OUTPUT, and POSTROUTING nat chains are + wrapped in the same was as the builtin filter chains. Additionally, there's + a snat chain that is applied after the POSTROUTING chain. + """ + def __init__(self, execute=None): + if not execute: + if FLAGS.fake_network: + self.execute = lambda *args, **kwargs: ('', '') + else: + self.execute = utils.execute + else: + self.execute = execute + + self.ipv4 = {'filter': IptablesTable(), + 'nat': IptablesTable()} + self.ipv6 = {'filter': IptablesTable()} + + # Add a nova-filter-top chain. It's intended to be shared + # among the various nova components. It sits at the very top + # of FORWARD and OUTPUT. + for tables in [self.ipv4, self.ipv6]: + tables['filter'].add_chain('nova-filter-top', wrap=False) + tables['filter'].add_rule('FORWARD', '-j nova-filter-top', + wrap=False, top=True) + tables['filter'].add_rule('OUTPUT', '-j nova-filter-top', + wrap=False, top=True) + + tables['filter'].add_chain('local') + tables['filter'].add_rule('nova-filter-top', '-j $local', + wrap=False) + + # Wrap the builtin chains + builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} + + for ip_version in builtin_chains: + if ip_version == 4: + tables = self.ipv4 + elif ip_version == 6: + tables = self.ipv6 + + for table, chains in builtin_chains[ip_version].iteritems(): + for chain in chains: + tables[table].add_chain(chain) + tables[table].add_rule(chain, '-j $%s' % (chain,), + wrap=False) + + # Add a nova-postrouting-bottom chain. It's intended to be shared + # among the various nova components. We set it as the last chain + # of POSTROUTING chain. + self.ipv4['nat'].add_chain('nova-postrouting-bottom', wrap=False) + self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', + wrap=False) + + # We add a snat chain to the shared nova-postrouting-bottom chain + # so that it's applied last. + self.ipv4['nat'].add_chain('snat') + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $snat', + wrap=False) + + # And then we add a floating-snat chain and jump to first thing in + # the snat chain. + self.ipv4['nat'].add_chain('floating-snat') + self.ipv4['nat'].add_rule('snat', '-j $floating-snat') + + self.semaphore = semaphore.Semaphore() + + @utils.synchronized('iptables') + def apply(self): + """Apply the current in-memory set of iptables rules + + This will blow away any rules left over from previous runs of the + same component of Nova, and replace them with our current set of + rules. This happens atomically, thanks to iptables-restore. + + We wrap the call in a semaphore lock, so that we don't race with + ourselves. In the event of a race with another component running + an iptables-* command at the same time, we retry up to 5 times. + """ + with self.semaphore: + s = [('iptables', self.ipv4)] + if FLAGS.use_ipv6: + s += [('ip6tables', self.ipv6)] + + for cmd, tables in s: + for table in tables: + current_table, _ = self.execute('sudo', + '%s-save' % (cmd,), + '-t', '%s' % (table,), + attempts=5) + current_lines = current_table.split('\n') + new_filter = self._modify_rules(current_lines, + tables[table]) + self.execute('sudo', '%s-restore' % (cmd,), + process_input='\n'.join(new_filter), + attempts=5) + + def _modify_rules(self, current_lines, table, binary=None): + unwrapped_chains = table.unwrapped_chains + chains = table.chains + rules = table.rules + + # Remove any trace of our rules + new_filter = filter(lambda line: binary_name not in line, + current_lines) + + seen_chains = False + rules_index = 0 + for rules_index, rule in enumerate(new_filter): + if not seen_chains: + if rule.startswith(':'): + seen_chains = True + else: + if not rule.startswith(':'): + break + + our_rules = [] + for rule in rules: + rule_str = str(rule) + if rule.top: + # rule.top == True means we want this rule to be at the top. + # Further down, we weed out duplicates from the bottom of the + # list, so here we remove the dupes ahead of time. + new_filter = filter(lambda s: s.strip() != rule_str.strip(), + new_filter) + our_rules += [rule_str] + + new_filter[rules_index:rules_index] = our_rules + + new_filter[rules_index:rules_index] = [':%s - [0:0]' % \ + (name,) \ + for name in unwrapped_chains] + new_filter[rules_index:rules_index] = [':%s-%s - [0:0]' % \ + (binary_name, name,) \ + for name in chains] + + seen_lines = set() + + def _weed_out_duplicates(line): + line = line.strip() + if line in seen_lines: + return False + else: + seen_lines.add(line) + return True + + # We filter duplicates, letting the *last* occurrence take + # precendence. + new_filter.reverse() + new_filter = filter(_weed_out_duplicates, new_filter) + new_filter.reverse() + return new_filter + + +iptables_manager = IptablesManager() + + def metadata_forward(): """Create forwarding rule for metadata""" - _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " - "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + iptables_manager.ipv4['nat'].add_rule("PREROUTING", + "-s 0.0.0.0/0 -d 169.254.169.254/32 " + "-p tcp -m tcp --dport 80 -j DNAT " + "--to-destination %s:%s" % \ + (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + iptables_manager.apply() def init_host(): """Basic networking setup goes here""" - - if FLAGS.use_nova_chains: - _execute("sudo iptables -N nova_input", check_exit_code=False) - _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, - check_exit_code=False) - _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) - - _execute("sudo iptables -N nova_forward", check_exit_code=False) - _execute("sudo iptables -D FORWARD -j nova_forward", - check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _execute("sudo iptables -N nova_output", check_exit_code=False) - _execute("sudo iptables -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -A OUTPUT -j nova_output") - - _execute("sudo iptables -t nat -N nova_prerouting", - check_exit_code=False) - _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", - check_exit_code=False) - _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") - - _execute("sudo iptables -t nat -N nova_postrouting", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") - - _execute("sudo iptables -t nat -N nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") - - _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) - _execute("sudo iptables -t nat -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -t nat -A OUTPUT -j nova_output") - else: - # NOTE(vish): This makes it easy to ensure snatting rules always - # come after the accept rules in the postrouting chain - _execute("sudo iptables -t nat -N SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") - # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("SNATTING", "-t nat -s %s " - "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) + iptables_manager.ipv4['nat'].add_rule("snat", + "-s %s -j SNAT --to-source %s" % \ + (FLAGS.fixed_range, + FLAGS.routing_source_ip)) + + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %s -d %s -j ACCEPT" % \ + (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % - (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % - {'range': FLAGS.fixed_range}) + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %(range)s -d %(range)s " + "-j ACCEPT" % \ + {'range': FLAGS.fixed_range}) + iptables_manager.apply() def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" - _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface), + _execute('sudo', 'ip', 'addr', 'add', floating_ip, + 'dev', FLAGS.public_interface, check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" - _execute("sudo ip addr del %s dev %s" % (floating_ip, - FLAGS.public_interface)) + _execute('sudo', 'ip', 'addr', 'del', floating_ip, + 'dev', FLAGS.public_interface) def ensure_vlan_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan""" - _confirm_rule("FORWARD", "-d %s -p udp --dport 1194 -j ACCEPT" % - private_ip) - _confirm_rule("PREROUTING", - "-t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (public_ip, port, private_ip)) + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "-d %s -p udp " + "--dport 1194 " + "-j ACCEPT" % private_ip) + iptables_manager.ipv4['nat'].add_rule("PREROUTING", + "-d %s -p udp " + "--dport %s -j DNAT --to %s:1194" % + (public_ip, port, private_ip)) + iptables_manager.apply() def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" - _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + for chain, rule in floating_forward_rules(floating_ip, fixed_ip): + iptables_manager.ipv4['nat'].add_rule(chain, rule) + iptables_manager.apply() def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" - _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + for chain, rule in floating_forward_rules(floating_ip, fixed_ip): + iptables_manager.ipv4['nat'].remove_rule(chain, rule) + iptables_manager.apply() + + +def floating_forward_rules(floating_ip, fixed_ip): + return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), + ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), + ("floating-snat", + "-s %s -j SNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): @@ -185,9 +452,9 @@ def ensure_vlan(vlan_num): interface = "vlan%s" % vlan_num if not _device_exists(interface): LOG.debug(_("Starting VLAN inteface %s"), interface) - _execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") - _execute("sudo vconfig add %s %s" % (FLAGS.vlan_interface, vlan_num)) - _execute("sudo ip link set %s up" % interface) + _execute('sudo', 'vconfig', 'set_name_type', 'VLAN_PLUS_VID_NO_PAD') + _execute('sudo', 'vconfig', 'add', FLAGS.vlan_interface, vlan_num) + _execute('sudo', 'ip', 'link', 'set', interface, 'up') return interface @@ -206,75 +473,80 @@ def ensure_bridge(bridge, interface, net_attrs=None): """ if not _device_exists(bridge): LOG.debug(_("Starting Bridge interface for %s"), interface) - _execute("sudo brctl addbr %s" % bridge) - _execute("sudo brctl setfd %s 0" % bridge) + _execute('sudo', 'brctl', 'addbr', bridge) + _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute("sudo brctl stp %s off" % bridge) - _execute("sudo ip link set %s up" % bridge) + _execute('sudo', 'brctl', 'stp', bridge, 'off') + _execute('sudo', 'ip', 'link', 'set', bridge, 'up') if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] - out, err = _execute("sudo ip addr add %s/%s brd %s dev %s" % - (net_attrs['gateway'], - suffix, - net_attrs['broadcast'], - bridge), + out, err = _execute('sudo', 'ip', 'addr', 'add', + "%s/%s" % + (net_attrs['gateway'], suffix), + 'brd', + net_attrs['broadcast'], + 'dev', + bridge, check_exit_code=False) if err and err != "RTNETLINK answers: File exists\n": raise exception.Error("Failed to add ip: %s" % err) if(FLAGS.use_ipv6): - _execute("sudo ip -f inet6 addr change %s dev %s" % - (net_attrs['cidr_v6'], bridge)) + _execute('sudo', 'ip', '-f', 'inet6', 'addr', + 'change', net_attrs['cidr_v6'], + 'dev', bridge) # NOTE(vish): If the public interface is the same as the # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute("sudo ip link set dev %s promisc on" % bridge) + _execute('sudo', 'ip', 'link', 'set', + 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge gateway = None - out, err = _execute("sudo route -n") + out, err = _execute('sudo', 'route', '-n') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "0.0.0.0" and fields[-1] == interface: gateway = fields[1] - out, err = _execute("sudo ip addr show dev %s scope global" % - interface) + out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, + 'scope', 'global') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "inet": - params = ' '.join(fields[1:-1]) - _execute("sudo ip addr del %s dev %s" % (params, fields[-1])) - _execute("sudo ip addr add %s dev %s" % (params, bridge)) + params = fields[1:-1] + _execute(*_ip_bridge_cmd('del', params, fields[-1])) + _execute(*_ip_bridge_cmd('add', params, bridge)) if gateway: - _execute("sudo route add 0.0.0.0 gw %s" % gateway) - out, err = _execute("sudo brctl addif %s %s" % - (bridge, interface), + _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) + out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " "enslave it to bridge %s.\n" % (interface, bridge)): raise exception.Error("Failed to add interface: %s" % err) - if FLAGS.use_nova_chains: - (out, err) = _execute("sudo iptables -N nova_forward", - check_exit_code=False) - if err != 'iptables: Chain already exists.\n': - # NOTE(vish): chain didn't exist link chain - _execute("sudo iptables -D FORWARD -j nova_forward", - check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "--in-interface %s -j ACCEPT" % \ + bridge) + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "--out-interface %s -j ACCEPT" % \ + bridge) - _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) - _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) - _execute("sudo iptables -N nova-local", check_exit_code=False) - _confirm_rule("FORWARD", "-j nova-local") + +def get_dhcp_leases(context, network_id): + """Return a network's hosts config in dnsmasq leasefile format""" + hosts = [] + for fixed_ip_ref in db.network_get_associated_fixed_ips(context, + network_id): + hosts.append(_host_lease(fixed_ip_ref)) + return '\n'.join(hosts) def get_dhcp_hosts(context, network_id): - """Get a string containing a network's hosts config in dnsmasq format""" + """Get a string containing a network's hosts config in dhcp-host format""" hosts = [] for fixed_ip_ref in db.network_get_associated_fixed_ips(context, network_id): @@ -304,11 +576,11 @@ def update_dhcp(context, network_id): # if dnsmasq is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' % pid, + out, _err = _execute('cat', "/proc/%d/cmdline" % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill -HUP %d' % pid) + _execute('sudo', 'kill', '-HUP', pid) return except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Hupping dnsmasq threw %s"), exc) @@ -319,7 +591,7 @@ def update_dhcp(context, network_id): env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, 'DNSMASQ_INTERFACE': network_ref['bridge']} command = _dnsmasq_cmd(network_ref) - _execute(command, addl_env=env) + _execute(*command, addl_env=env) def update_ra(context, network_id): @@ -349,24 +621,40 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' + out, _err = _execute('cat', '/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill %d' % pid) + _execute('sudo', 'kill', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("killing radvd threw %s"), exc) else: LOG.debug(_("Pid %d is stale, relaunching radvd"), pid) command = _ra_cmd(network_ref) - _execute(command) + _execute(*command) db.network_update(context, network_id, {"ra_server": utils.get_my_linklocal(network_ref['bridge'])}) +def _host_lease(fixed_ip_ref): + """Return a host string for an address in leasefile format""" + instance_ref = fixed_ip_ref['instance'] + if instance_ref['updated_at']: + timestamp = instance_ref['updated_at'] + else: + timestamp = instance_ref['created_at'] + + seconds_since_epoch = calendar.timegm(timestamp.utctimetuple()) + + return "%d %s %s %s *" % (seconds_since_epoch + FLAGS.dhcp_lease_time, + instance_ref['mac_address'], + fixed_ip_ref['address'], + instance_ref['hostname'] or '*') + + def _host_dhcp(fixed_ip_ref): - """Return a host string for an address""" + """Return a host string for an address in dhcp-host format""" instance_ref = fixed_ip_ref['instance'] return "%s,%s.%s,%s" % (instance_ref['mac_address'], instance_ref['hostname'], @@ -374,68 +662,48 @@ def _host_dhcp(fixed_ip_ref): fixed_ip_ref['address']) -def _execute(cmd, *args, **kwargs): +def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", cmd) + LOG.debug("FAKE NET: %s", " ".join(map(str, cmd))) return "fake", 0 else: - return utils.execute(cmd, *args, **kwargs) + return utils.execute(*cmd, **kwargs) def _device_exists(device): """Check if ethernet device exists""" - (_out, err) = _execute("ip link show dev %s" % device, + (_out, err) = _execute('ip', 'link', 'show', 'dev', device, check_exit_code=False) return not err -def _confirm_rule(chain, cmd, append=False): - """Delete and re-add iptables rule""" - if FLAGS.use_nova_chains: - chain = "nova_%s" % chain.lower() - if append: - loc = "-A" - else: - loc = "-I" - _execute("sudo iptables --delete %s %s" % (chain, cmd), - check_exit_code=False) - _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) - - -def _remove_rule(chain, cmd): - """Remove iptables rule""" - if FLAGS.use_nova_chains: - chain = "%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd)) - - def _dnsmasq_cmd(net): """Builds dnsmasq command""" - cmd = ['sudo -E dnsmasq', - ' --strict-order', - ' --bind-interfaces', - ' --conf-file=', - ' --domain=%s' % FLAGS.dhcp_domain, - ' --pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), - ' --listen-address=%s' % net['gateway'], - ' --except-interface=lo', - ' --dhcp-range=%s,static,120s' % net['dhcp_start'], - ' --dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), - ' --dhcp-script=%s' % FLAGS.dhcpbridge, - ' --leasefile-ro'] + cmd = ['sudo', '-E', 'dnsmasq', + '--strict-order', + '--bind-interfaces', + '--conf-file=', + '--domain=%s' % FLAGS.dhcp_domain, + '--pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), + '--listen-address=%s' % net['gateway'], + '--except-interface=lo', + '--dhcp-range=%s,static,120s' % net['dhcp_start'], + '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), + '--dhcp-script=%s' % FLAGS.dhcpbridge, + '--leasefile-ro'] if FLAGS.dns_server: - cmd.append(' -h -R --server=%s' % FLAGS.dns_server) - return ''.join(cmd) + cmd += ['-h', '-R', '--server=%s' % FLAGS.dns_server] + return cmd def _ra_cmd(net): """Builds radvd command""" - cmd = ['sudo -E radvd', -# ' -u nobody', - ' -C %s' % _ra_file(net['bridge'], 'conf'), - ' -p %s' % _ra_file(net['bridge'], 'pid')] - return ''.join(cmd) + cmd = ['sudo', '-E', 'radvd', +# '-u', 'nobody', + '-C', '%s' % _ra_file(net['bridge'], 'conf'), + '-p', '%s' % _ra_file(net['bridge'], 'pid')] + return cmd def _stop_dnsmasq(network): @@ -444,7 +712,7 @@ def _stop_dnsmasq(network): if pid: try: - _execute('sudo kill -TERM %d' % pid) + _execute('sudo', 'kill', '-TERM', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Killing dnsmasq threw %s"), exc) @@ -497,3 +765,12 @@ def _ra_pid_for(bridge): if os.path.exists(pid_file): with open(pid_file, 'r') as f: return int(f.read()) + + +def _ip_bridge_cmd(action, params, device): + """Build commands to add/del ips to bridges/devices""" + + cmd = ['sudo', 'ip', 'addr', action] + cmd.extend(params) + cmd.extend(['dev', device]) + return cmd diff --git a/nova/network/manager.py b/nova/network/manager.py index f5f5b17aa..e97d2a375 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -55,7 +55,7 @@ from nova import db from nova import exception from nova import flags from nova import log as logging -from nova import scheduler_manager +from nova import manager from nova import utils from nova import rpc @@ -105,7 +105,7 @@ class AddressAlreadyAllocated(exception.Error): pass -class NetworkManager(scheduler_manager.SchedulerDependentManager): +class NetworkManager(manager.SchedulerDependentManager): """Implements common network manager functionality. This class must be subclassed to support specific topologies. @@ -564,6 +564,16 @@ class VlanManager(NetworkManager): # NOTE(vish): This makes ports unique accross the cloud, a more # robust solution would be to make them unique per ip net['vpn_public_port'] = vpn_start + index + network_ref = None + try: + network_ref = db.network_get_by_cidr(context, cidr) + except exception.NotFound: + pass + + if network_ref is not None: + raise ValueError(_('Network with cidr %s already exists' % + cidr)) + network_ref = self.db.network_create_safe(context, net) if network_ref: self._create_fixed_ips(context, network_ref['id']) diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 27227e2ca..c90b5b54b 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -37,8 +37,7 @@ from nova.objectstore import bucket FLAGS = flags.FLAGS -flags.DEFINE_string('images_path', '$state_path/images', - 'path to decrypted images') +flags.DECLARE('images_path', 'nova.image.local') class Image(object): @@ -254,25 +253,34 @@ class Image(object): @staticmethod def decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename): - key, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_key, - check_exit_code=False) + key, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt private key: %s") % err) - iv, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_iv, - check_exit_code=False) + iv, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt initialization " "vector: %s") % err) - _out, err = utils.execute( - 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' - % (encrypted_filename, key, iv, decrypted_filename), - check_exit_code=False) + _out, err = utils.execute('openssl', + 'enc', + '-d', + '-aes-128-cbc', + '-in', '%s' % (encrypted_filename,), + '-K', '%s' % (key,), + '-iv', '%s' % (iv,), + '-out', '%s' % (decrypted_filename,), + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt image file " "%(image_file)s: %(err)s") % diff --git a/nova/quota.py b/nova/quota.py index 6b52a97fa..2b24c0b5b 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,6 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') +flags.DEFINE_integer('quota_max_injected_files', 5, + 'number of injected files allowed') +flags.DEFINE_integer('quota_max_injected_file_content_bytes', 10 * 1024, + 'number of bytes allowed per injected file') +flags.DEFINE_integer('quota_max_injected_file_path_bytes', 255, + 'number of bytes allowed per injected file path') def get_quota(context, project_id): @@ -46,6 +52,7 @@ def get_quota(context, project_id): 'gigabytes': FLAGS.quota_gigabytes, 'floating_ips': FLAGS.quota_floating_ips, 'metadata_items': FLAGS.quota_metadata_items} + try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -106,6 +113,21 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) +def allowed_injected_files(context): + """Return the number of injected files allowed""" + return FLAGS.quota_max_injected_files + + +def allowed_injected_file_content_bytes(context): + """Return the number of bytes allowed per injected file content""" + return FLAGS.quota_max_injected_file_content_bytes + + +def allowed_injected_file_path_bytes(context): + """Return the number of bytes allowed in an injected file path""" + return FLAGS.quota_max_injected_file_path_bytes + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index f0b645c09..8c9fd2298 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -86,27 +86,66 @@ def _wrap_method(function, self): return _wrap -def _process(self, zone): +def _process(func, zone): """Worker stub for green thread pool""" nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() - return self.process(nova, zone) - - -class ChildZoneHelper(object): - """Delegate a call to a set of Child Zones and wait for their - responses. Could be used for Zone Redirect or by the Scheduler - plug-ins to query the children.""" - - def start(self, zone_list): - """Spawn a green thread for each child zone, calling the - derived classes process() method as the worker. Returns - a list of HTTP Responses. 1 per child.""" - self.green_pool = greenpool.GreenPool() - return [result for result in self.green_pool.imap( - _wrap_method(_process, self), zone_list)] - - def process(self, client, zone): - """Worker Method. Derived class must override.""" - pass + return func(nova, zone) + + +def child_zone_helper(zone_list, func): + green_pool = greenpool.GreenPool() + return [result for result in green_pool.imap( + _wrap_method(_process, func), zone_list)] + + +def _issue_novaclient_command(nova, zone, method_name, instance_id): + server = None + try: + if isinstance(instance_id, int) or instance_id.isdigit(): + server = manager.get(int(instance_id)) + else: + server = manager.find(name=instance_id) + except novaclient.NotFound: + url = zone.api_url + LOG.debug(_("Instance %(instance_id)s not found on '%(url)s'" % + locals())) + return + + return getattr(server, method_name)() + + +def wrap_novaclient_function(f, method_name, instance_id): + def inner(nova, zone): + return f(nova, zone, method_name, instance_id) + + return inner + + +class reroute_if_not_found(object): + """Decorator used to indicate that the method should + delegate the call the child zones if the db query + can't find anything. + """ + def __init__(self, method_name): + self.method_name = method_name + + def __call__(self, f): + def wrapped_f(*args, **kwargs): + LOG.debug("***REROUTE-3: %s / %s" % (args, kwargs)) + context = args[1] + instance_id = args[2] + try: + return f(*args, **kwargs) + except exception.InstanceNotFound, e: + LOG.debug(_("Instance %(instance_id)s not found " + "locally: '%(e)s'" % locals())) + + zones = db.zone_get_all(context) + result = child_zone_helper(zones, + wrap_novaclient_function(_issue_novaclient_command, + self.method_name, instance_id)) + LOG.debug("***REROUTE: %s" % result) + return result + return wrapped_f diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 317a039cc..ce05d9f6a 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -26,10 +26,14 @@ import datetime from nova import db from nova import exception from nova import flags +from nova import log as logging +from nova import rpc +from nova.compute import power_state FLAGS = flags.FLAGS flags.DEFINE_integer('service_down_time', 60, 'maximum time since last checkin for up service') +flags.DECLARE('instances_path', 'nova.compute.manager') class NoValidHost(exception.Error): @@ -71,3 +75,236 @@ class Scheduler(object): def schedule(self, context, topic, *_args, **_kwargs): """Must override at least this method for scheduler to work.""" raise NotImplementedError(_("Must implement a fallback schedule")) + + def schedule_live_migration(self, context, instance_id, dest): + """Live migration scheduling method. + + :param context: + :param instance_id: + :param dest: destination host + :return: + The host where instance is running currently. + Then scheduler send request that host. + + """ + + # Whether instance exists and is running. + instance_ref = db.instance_get(context, instance_id) + + # Checking instance. + self._live_migration_src_check(context, instance_ref) + + # Checking destination host. + self._live_migration_dest_check(context, instance_ref, dest) + + # Common checking. + self._live_migration_common_check(context, instance_ref, dest) + + # Changing instance_state. + db.instance_set_state(context, + instance_id, + power_state.PAUSED, + 'migrating') + + # Changing volume state + for volume_ref in instance_ref['volumes']: + db.volume_update(context, + volume_ref['id'], + {'status': 'migrating'}) + + # Return value is necessary to send request to src + # Check _schedule() in detail. + src = instance_ref['host'] + return src + + def _live_migration_src_check(self, context, instance_ref): + """Live migration check routine (for src host). + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + + """ + + # Checking instance is running. + if (power_state.RUNNING != instance_ref['state'] or \ + 'running' != instance_ref['state_description']): + ec2_id = instance_ref['hostname'] + raise exception.Invalid(_('Instance(%s) is not running') % ec2_id) + + # Checing volume node is running when any volumes are mounted + # to the instance. + if len(instance_ref['volumes']) != 0: + services = db.service_get_all_by_topic(context, 'volume') + if len(services) < 1 or not self.service_is_up(services[0]): + raise exception.Invalid(_("volume node is not alive" + "(time synchronize problem?)")) + + # Checking src host exists and compute node + src = instance_ref['host'] + services = db.service_get_all_compute_by_host(context, src) + + # Checking src host is alive. + if not self.service_is_up(services[0]): + raise exception.Invalid(_("%s is not alive(time " + "synchronize problem?)") % src) + + def _live_migration_dest_check(self, context, instance_ref, dest): + """Live migration check routine (for destination host). + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + + """ + + # Checking dest exists and compute node. + dservice_refs = db.service_get_all_compute_by_host(context, dest) + dservice_ref = dservice_refs[0] + + # Checking dest host is alive. + if not self.service_is_up(dservice_ref): + raise exception.Invalid(_("%s is not alive(time " + "synchronize problem?)") % dest) + + # Checking whether The host where instance is running + # and dest is not same. + src = instance_ref['host'] + if dest == src: + ec2_id = instance_ref['hostname'] + raise exception.Invalid(_("%(dest)s is where %(ec2_id)s is " + "running now. choose other host.") + % locals()) + + # Checking dst host still has enough capacities. + self.assert_compute_node_has_enough_resources(context, + instance_ref, + dest) + + def _live_migration_common_check(self, context, instance_ref, dest): + """Live migration common check routine. + + Below checkings are followed by + http://wiki.libvirt.org/page/TodoPreMigrationChecks + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + + """ + + # Checking shared storage connectivity + self.mounted_on_same_shared_storage(context, instance_ref, dest) + + # Checking dest exists. + dservice_refs = db.service_get_all_compute_by_host(context, dest) + dservice_ref = dservice_refs[0]['compute_node'][0] + + # Checking original host( where instance was launched at) exists. + try: + oservice_refs = db.service_get_all_compute_by_host(context, + instance_ref['launched_on']) + except exception.NotFound: + raise exception.Invalid(_("host %s where instance was launched " + "does not exist.") + % instance_ref['launched_on']) + oservice_ref = oservice_refs[0]['compute_node'][0] + + # Checking hypervisor is same. + orig_hypervisor = oservice_ref['hypervisor_type'] + dest_hypervisor = dservice_ref['hypervisor_type'] + if orig_hypervisor != dest_hypervisor: + raise exception.Invalid(_("Different hypervisor type" + "(%(orig_hypervisor)s->" + "%(dest_hypervisor)s)')" % locals())) + + # Checkng hypervisor version. + orig_hypervisor = oservice_ref['hypervisor_version'] + dest_hypervisor = dservice_ref['hypervisor_version'] + if orig_hypervisor > dest_hypervisor: + raise exception.Invalid(_("Older hypervisor version" + "(%(orig_hypervisor)s->" + "%(dest_hypervisor)s)") % locals()) + + # Checking cpuinfo. + try: + rpc.call(context, + db.queue_get_for(context, FLAGS.compute_topic, dest), + {"method": 'compare_cpu', + "args": {'cpu_info': oservice_ref['cpu_info']}}) + + except rpc.RemoteError: + src = instance_ref['host'] + logging.exception(_("host %(dest)s is not compatible with " + "original host %(src)s.") % locals()) + raise + + def assert_compute_node_has_enough_resources(self, context, + instance_ref, dest): + """Checks if destination host has enough resource for live migration. + + Currently, only memory checking has been done. + If storage migration(block migration, meaning live-migration + without any shared storage) will be available, local storage + checking is also necessary. + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + + """ + + # Getting instance information + ec2_id = instance_ref['hostname'] + + # Getting host information + service_refs = db.service_get_all_compute_by_host(context, dest) + compute_node_ref = service_refs[0]['compute_node'][0] + + mem_total = int(compute_node_ref['memory_mb']) + mem_used = int(compute_node_ref['memory_mb_used']) + mem_avail = mem_total - mem_used + mem_inst = instance_ref['memory_mb'] + if mem_avail <= mem_inst: + raise exception.NotEmpty(_("Unable to migrate %(ec2_id)s " + "to destination: %(dest)s " + "(host:%(mem_avail)s " + "<= instance:%(mem_inst)s)") + % locals()) + + def mounted_on_same_shared_storage(self, context, instance_ref, dest): + """Check if the src and dest host mount same shared storage. + + At first, dest host creates temp file, and src host can see + it if they mounts same shared storage. Then src host erase it. + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + + """ + + src = instance_ref['host'] + dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) + src_t = db.queue_get_for(context, FLAGS.compute_topic, src) + + try: + # create tmpfile at dest host + filename = rpc.call(context, dst_t, + {"method": 'create_shared_storage_test_file'}) + + # make sure existence at src host. + rpc.call(context, src_t, + {"method": 'check_shared_storage_test_file', + "args": {'filename': filename}}) + + except rpc.RemoteError: + ipath = FLAGS.instances_path + logging.error(_("Cannot confirm tmpfile at %(ipath)s is on " + "same shared storage between %(src)s " + "and %(dest)s.") % locals()) + raise + + finally: + rpc.call(context, dst_t, + {"method": 'cleanup_shared_storage_test_file', + "args": {'filename': filename}}) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index d3d338943..7d62cfc4e 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -89,3 +89,55 @@ class SchedulerManager(manager.Manager): {"method": method, "args": kwargs}) LOG.debug(_("Casting to %(topic)s %(host)s for %(method)s") % locals()) + + # NOTE (masumotok) : This method should be moved to nova.api.ec2.admin. + # Based on bexar design summit discussion, + # just put this here for bexar release. + def show_host_resources(self, context, host, *args): + """Shows the physical/usage resource given by hosts. + + :param context: security context + :param host: hostname + :returns: + example format is below. + {'resource':D, 'usage':{proj_id1:D, proj_id2:D}} + D: {'vcpus':3, 'memory_mb':2048, 'local_gb':2048} + + """ + + compute_ref = db.service_get_all_compute_by_host(context, host) + compute_ref = compute_ref[0] + + # Getting physical resource information + compute_node_ref = compute_ref['compute_node'][0] + resource = {'vcpus': compute_node_ref['vcpus'], + 'memory_mb': compute_node_ref['memory_mb'], + 'local_gb': compute_node_ref['local_gb'], + 'vcpus_used': compute_node_ref['vcpus_used'], + 'memory_mb_used': compute_node_ref['memory_mb_used'], + 'local_gb_used': compute_node_ref['local_gb_used']} + + # Getting usage resource information + usage = {} + instance_refs = db.instance_get_all_by_host(context, + compute_ref['host']) + if not instance_refs: + return {'resource': resource, 'usage': usage} + + project_ids = [i['project_id'] for i in instance_refs] + project_ids = list(set(project_ids)) + for project_id in project_ids: + vcpus = db.instance_get_vcpu_sum_by_host_and_project(context, + host, + project_id) + mem = db.instance_get_memory_sum_by_host_and_project(context, + host, + project_id) + hdd = db.instance_get_disk_sum_by_host_and_project(context, + host, + project_id) + usage[project_id] = {'vcpus': int(vcpus), + 'memory_mb': int(mem), + 'local_gb': int(hdd)} + + return {'resource': resource, 'usage': usage} diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py deleted file mode 100644 index ca39b85dd..000000000 --- a/nova/scheduler_manager.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2011 OpenStack, LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -This module provides SchedulerDependentManager, a base class for -any Manager that has Capabilities that should be related to the -Scheduler. - -These Capabilities are hints that can help the scheduler route -requests to the appropriate service instance. -""" - -import sys - -from nova import flags -from nova import manager -from nova.scheduler import api -from nova import log as logging - - -FLAGS = flags.FLAGS - - -class SchedulerDependentManager(manager.Manager): - """Periodically send capability updates to the Scheduler services. - Services that need to update the Scheduler of their capabilities - should derive from this class. Otherwise they can derive from - manager.Manager directly. Updates are only sent after - update_service_capabilities is called with non-None values.""" - def __init__(self, host=None, db_driver=None, service_name="undefined"): - self.last_capabilities = None - self.service_name = service_name - super(SchedulerDependentManager, self).__init__(host, db_driver) - - def update_service_capabilities(self, capabilities): - """Remember these capabilities to send on next periodic update.""" - self.last_capabilities = capabilities - - def periodic_tasks(self, context=None): - """Pass data back to the scheduler at a periodic interval""" - if self.last_capabilities: - logging.debug(_("Notifying Schedulers of capabilities ...")) - api.API.update_service_capabilities(context, self.service_name, - self.host, self.last_capabilities) - - super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/service.py b/nova/service.py index 51c0e935e..b021c1cee 100644 --- a/nova/service.py +++ b/nova/service.py @@ -92,6 +92,9 @@ class Service(object): except exception.NotFound: self._create_service_ref(ctxt) + if 'nova-compute' == self.binary: + self.manager.update_available_resource(ctxt) + conn1 = rpc.Connection.instance(new=True) conn2 = rpc.Connection.instance(new=True) conn3 = rpc.Connection.instance(new=True) diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py index 3f9c7d3cf..74bb8729a 100644 --- a/nova/tests/api/openstack/common.py +++ b/nova/tests/api/openstack/common.py @@ -28,6 +28,7 @@ def webob_factory(url): def web_request(url, method=None, body=None): req = webob.Request.blank("%s%s" % (base_url, url)) if method: + req.content_type = "application/json" req.method = method if body: req.body = json.dumps(body) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..0bbb1c890 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -25,8 +25,8 @@ import webob.dec from paste import urlmap from glance import client as glance_client +from glance.common import exception as glance_exc -from nova import auth from nova import context from nova import exception as exc from nova import flags @@ -35,6 +35,7 @@ import nova.api.openstack.auth from nova.api import openstack from nova.api.openstack import auth from nova.api.openstack import ratelimiting +from nova.auth.manager import User, Project from nova.image import glance from nova.image import local from nova.image import service @@ -68,8 +69,6 @@ def fake_auth_init(self, application): @webob.dec.wsgify def fake_wsgi(self, req): req.environ['nova.context'] = context.RequestContext(1, 1) - if req.body: - req.environ['inst_dict'] = json.loads(req.body) return self.application @@ -84,10 +83,17 @@ def wsgi_app(inner_application=None): return mapper -def stub_out_key_pair_funcs(stubs): +def stub_out_key_pair_funcs(stubs, have_key_pair=True): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] - stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) + + def no_key_pair(context, user_id): + return [] + + if have_key_pair: + stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) + else: + stubs.Set(nova.db, 'key_pair_get_all_by_user', no_key_pair) def stub_out_image_service(stubs): @@ -149,25 +155,26 @@ def stub_out_glance(stubs, initial_fixtures=None): for f in self.fixtures: if f['id'] == image_id: return f - return None + raise glance_exc.NotFound - def fake_add_image(self, image_meta): + def fake_add_image(self, image_meta, data=None): id = ''.join(random.choice(string.letters) for _ in range(20)) image_meta['id'] = id self.fixtures.append(image_meta) - return id + return image_meta - def fake_update_image(self, image_id, image_meta): + def fake_update_image(self, image_id, image_meta, data=None): f = self.fake_get_image_meta(image_id) if not f: - raise exc.NotFound + raise glance_exc.NotFound f.update(image_meta) + return f def fake_delete_image(self, image_id): f = self.fake_get_image_meta(image_id) if not f: - raise exc.NotFound + raise glance_exc.NotFound self.fixtures.remove(f) @@ -227,21 +234,102 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} + projects = {} + + @classmethod + def clear_fakes(cls): + cls.auth_data = {} + cls.projects = {} + + @classmethod + def reset_fake_data(cls): + cls.auth_data = dict(acc1=User('guy1', 'guy1', 'acc1', + 'fortytwo!', False)) + cls.projects = dict(testacct=Project('testacct', + 'testacct', + 'guy1', + 'test', + [])) def add_user(self, key, user): FakeAuthManager.auth_data[key] = user + def get_users(self): + return FakeAuthManager.auth_data.values() + def get_user(self, uid): for k, v in FakeAuthManager.auth_data.iteritems(): if v.id == uid: return v return None - def get_project(self, pid): + def delete_user(self, uid): + for k, v in FakeAuthManager.auth_data.items(): + if v.id == uid: + del FakeAuthManager.auth_data[k] return None + def create_user(self, name, access=None, secret=None, admin=False): + u = User(name, name, access, secret, admin) + FakeAuthManager.auth_data[access] = u + return u + + def modify_user(self, user_id, access=None, secret=None, admin=None): + user = None + for k, v in FakeAuthManager.auth_data.iteritems(): + if v.id == user_id: + user = v + if user: + user.access = access + user.secret = secret + if admin is not None: + user.admin = admin + + def is_admin(self, user): + return user.admin + + def is_project_member(self, user, project): + return ((user.id in project.member_ids) or + (user.id == project.project_manager_id)) + + def create_project(self, name, manager_user, description=None, + member_users=None): + member_ids = [User.safe_id(m) for m in member_users] \ + if member_users else [] + p = Project(name, name, User.safe_id(manager_user), + description, member_ids) + FakeAuthManager.projects[name] = p + return p + + def delete_project(self, pid): + if pid in FakeAuthManager.projects: + del FakeAuthManager.projects[pid] + + def modify_project(self, project, manager_user=None, description=None): + p = FakeAuthManager.projects.get(project) + p.project_manager_id = User.safe_id(manager_user) + p.description = description + + def get_project(self, pid): + p = FakeAuthManager.projects.get(pid) + if p: + return p + else: + raise exc.NotFound + + def get_projects(self, user=None): + if not user: + return FakeAuthManager.projects.values() + else: + return [p for p in FakeAuthManager.projects.values() + if (user.id in p.member_ids) or + (user.id == p.project_manager_id)] + def get_user_from_access_key(self, key): - return FakeAuthManager.auth_data.get(key, None) + try: + return FakeAuthManager.auth_data[key] + except KeyError: + raise exc.NotFound class FakeRateLimiter(object): diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py new file mode 100644 index 000000000..60edce769 --- /dev/null +++ b/nova/tests/api/openstack/test_accounts.py @@ -0,0 +1,125 @@ +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import json + +import stubout +import webob + +import nova.api +import nova.api.openstack.auth +from nova import context +from nova import flags +from nova import test +from nova.auth.manager import User +from nova.tests.api.openstack import fakes + + +FLAGS = flags.FLAGS +FLAGS.verbose = True + + +def fake_init(self): + self.manager = fakes.FakeAuthManager() + + +def fake_admin_check(self, req): + return True + + +class AccountsTest(test.TestCase): + def setUp(self): + super(AccountsTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.openstack.accounts.Controller, '__init__', + fake_init) + self.stubs.Set(nova.api.openstack.accounts.Controller, '_check_admin', + fake_admin_check) + fakes.FakeAuthManager.clear_fakes() + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + + self.allow_admin = FLAGS.allow_admin_api + FLAGS.allow_admin_api = True + fakemgr = fakes.FakeAuthManager() + joeuser = User('guy1', 'guy1', 'acc1', 'fortytwo!', False) + superuser = User('guy2', 'guy2', 'acc2', 'swordfish', True) + fakemgr.add_user(joeuser.access, joeuser) + fakemgr.add_user(superuser.access, superuser) + fakemgr.create_project('test1', joeuser) + fakemgr.create_project('test2', superuser) + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(AccountsTest, self).tearDown() + + def test_get_account(self): + req = webob.Request.blank('/v1.0/accounts/test1') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['account']['id'], 'test1') + self.assertEqual(res_dict['account']['name'], 'test1') + self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertEqual(res.status_int, 200) + + def test_account_delete(self): + req = webob.Request.blank('/v1.0/accounts/test1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('test1' not in fakes.FakeAuthManager.projects) + self.assertEqual(res.status_int, 200) + + def test_account_create(self): + body = dict(account=dict(description='test account', + manager='guy1')) + req = webob.Request.blank('/v1.0/accounts/newacct') + req.headers["Content-Type"] = "application/json" + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['account']['id'], 'newacct') + self.assertEqual(res_dict['account']['name'], 'newacct') + self.assertEqual(res_dict['account']['description'], 'test account') + self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertTrue('newacct' in + fakes.FakeAuthManager.projects) + self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 3) + + def test_account_update(self): + body = dict(account=dict(description='test account', + manager='guy2')) + req = webob.Request.blank('/v1.0/accounts/test1') + req.headers["Content-Type"] = "application/json" + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['account']['id'], 'test1') + self.assertEqual(res_dict['account']['name'], 'test1') + self.assertEqual(res_dict['account']['description'], 'test account') + self.assertEqual(res_dict['account']['manager'], 'guy2') + self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 2) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index dfce1b127..4568cb9f5 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -35,7 +35,7 @@ class AdminAPITest(test.TestCase): def setUp(self): super(AdminAPITest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index ff8d42a14..0448ed701 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,11 +51,12 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + f.add_user('user1_key', + nova.auth.manager.User(1, 'user1', None, None, None)) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) @@ -65,11 +66,13 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + f.create_project('user1_project', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) @@ -90,7 +93,7 @@ class Test(test.TestCase): def test_token_expiry(self): self.destroy_called = False - token_hash = 'bacon' + token_hash = 'token_hash' def destroy_token_mock(meh, context, token): self.destroy_called = True @@ -107,15 +110,26 @@ class Test(test.TestCase): bad_token) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'bacon' + req.headers['X-Auth-Token'] = 'token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) - def test_bad_user(self): + def test_bad_user_bad_key(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'unknown_user' + req.headers['X-Auth-Key'] = 'unknown_user_key' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_bad_user_good_key(self): + f = fakes.FakeAuthManager() + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'unknown_user' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -126,7 +140,7 @@ class Test(test.TestCase): def test_bad_token(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'baconbaconbacon' + req.headers['X-Auth-Token'] = 'unknown_token' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -135,11 +149,11 @@ class TestFunctional(test.TestCase): def test_token_expiry(self): ctx = context.get_admin_context() tok = db.auth_token_create(ctx, dict( - token_hash='bacon', + token_hash='test_token_hash', cdn_management_url='', server_management_url='', storage_url='', - user_id='ham', + user_id='user1', )) db.auth_token_update(ctx, tok.token_hash, dict( @@ -147,13 +161,13 @@ class TestFunctional(test.TestCase): )) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'bacon' + req.headers['X-Auth-Token'] = 'test_token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_token_doesnotexist(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'ham' + req.headers['X-Auth-Token'] = 'nonexistant_token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -176,11 +190,13 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(len(result.headers['X-Auth-Token']), 40) diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 92023362c..8f57c5b67 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -79,20 +79,14 @@ class LimiterTest(test.TestCase): Test offset key works with a blank offset. """ req = Request.blank('/?offset=') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_offset_bad(self): """ Test offset key works with a BAD offset. """ req = Request.blank(u'/?offset=\u0020aa') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_nothing(self): """ @@ -166,18 +160,12 @@ class LimiterTest(test.TestCase): """ Test a negative limit. """ - def _limit_large(): - limited(self.large, req, max_limit=2000) - req = Request.blank('/?limit=-3000') - self.assertRaises(webob.exc.HTTPBadRequest, _limit_large) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_negative_offset(self): """ Test a negative offset. """ - def _limit_large(): - limited(self.large, req, max_limit=2000) - req = Request.blank('/?offset=-30') - self.assertRaises(webob.exc.HTTPBadRequest, _limit_large) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 319767bb5..8280a505f 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -30,7 +30,7 @@ class FlavorsTest(test.TestCase): def setUp(self): super(FlavorsTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index e232bc3d5..76f758929 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,6 +22,8 @@ and as a WSGI layer import json import datetime +import shutil +import tempfile import stubout import webob @@ -54,7 +56,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index(self.context)) - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] self.assertNotEquals(None, id) self.assertEquals(num_images + 1, @@ -71,7 +73,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index(self.context)) - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] self.assertNotEquals(None, id) @@ -89,7 +91,7 @@ class BaseImageServiceTests(object): 'instance_id': None, 'progress': None} - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] fixture['status'] = 'in progress' @@ -118,7 +120,7 @@ class BaseImageServiceTests(object): ids = [] for fixture in fixtures: - new_id = self.service.create(self.context, fixture) + new_id = self.service.create(self.context, fixture)['id'] ids.append(new_id) num_images = len(self.service.index(self.context)) @@ -137,14 +139,15 @@ class LocalImageServiceTest(test.TestCase, def setUp(self): super(LocalImageServiceTest, self).setUp() + self.tempdir = tempfile.mkdtemp() + self.flags(images_path=self.tempdir) self.stubs = stubout.StubOutForTesting() service_class = 'nova.image.local.LocalImageService' self.service = utils.import_object(service_class) self.context = context.RequestContext(None, None) def tearDown(self): - self.service.delete_all() - self.service.delete_imagedir() + shutil.rmtree(self.tempdir) self.stubs.UnsetAll() super(LocalImageServiceTest, self).tearDown() @@ -202,7 +205,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): self.orig_image_service = FLAGS.image_service FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index c9566c7e6..03e00af2a 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -15,8 +15,11 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 import datetime import json +import unittest +from xml.dom import minidom import stubout import webob @@ -120,7 +123,7 @@ class ServersTest(test.TestCase): def setUp(self): super(ServersTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -188,9 +191,38 @@ class ServersTest(test.TestCase): self.assertEqual(s.get('imageId', None), None) i += 1 - def test_create_instance(self): + def test_get_servers_with_limit(self): + req = webob.Request.blank('/v1.0/servers?limit=3') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [0, 1, 2]) + + req = webob.Request.blank('/v1.0/servers?limit=aaa') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + self.assertTrue('limit' in res.body) + + def test_get_servers_with_offset(self): + req = webob.Request.blank('/v1.0/servers?offset=2') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [2, 3, 4]) + + req = webob.Request.blank('/v1.0/servers?offset=aaa') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + self.assertTrue('offset' in res.body) + + def test_get_servers_with_limit_and_offset(self): + req = webob.Request.blank('/v1.0/servers?limit=2&offset=1') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [1, 2]) + + def _test_create_instance_helper(self): + """Shared implementation for tests below that create instance""" def instance_create(context, inst): - return {'id': '1', 'display_name': ''} + return {'id': '1', 'display_name': 'server_test'} def server_update(context, id, params): return instance_create(context, id) @@ -231,11 +263,25 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) + server = json.loads(res.body)['server'] + self.assertEqual('serv', server['adminPass'][:4]) + self.assertEqual(16, len(server['adminPass'])) + self.assertEqual('server_test', server['name']) + self.assertEqual('1', server['id']) + self.assertEqual(res.status_int, 200) + def test_create_instance(self): + self._test_create_instance_helper() + + def test_create_instance_no_key_pair(self): + fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) + self._test_create_instance_helper() + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' @@ -405,7 +451,8 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/inject_network_info') + req = webob.Request.blank( + '/v1.0/servers/1/inject_network_info') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -563,5 +610,529 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 400) + +class TestServerCreateRequestXMLDeserializer(unittest.TestCase): + + def setUp(self): + self.deserializer = servers.ServerCreateRequestXMLDeserializer() + + def test_minimal_request(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"/>""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata/> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + }} + self.assertEquals(request, expected) + + def test_request_with_empty_personality(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <personality/> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata_and_personality(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata/> + <personality/> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata_and_personality_reversed(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <personality/> + <metadata/> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_one_personality(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <personality> + <file path="/etc/conf">aabbccdd</file> + </personality> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_two_personalities(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> +<personality><file path="/etc/conf">aabbccdd</file> +<file path="/etc/sudoers">abcd</file></personality></server>""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}, + {"path": "/etc/sudoers", "contents": "abcd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_second_personality_node_ignored(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <personality> + <file path="/etc/conf">aabbccdd</file> + </personality> + <personality> + <file path="/etc/ignoreme">anything</file> + </personality> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_missing_path(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> +<personality><file>aabbccdd</file></personality></server>""" + request = self.deserializer.deserialize(serial_request) + expected = [{"contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_empty_contents(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> +<personality><file path="/etc/conf"></file></personality></server>""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": ""}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_empty_contents_variation(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> +<personality><file path="/etc/conf"/></personality></server>""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": ""}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_metadata(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta key="alpha">beta</meta> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "beta"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_two_metadata(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta key="alpha">beta</meta> + <meta key="foo">bar</meta> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "beta", "foo": "bar"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_missing_value(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta key="alpha"></meta> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": ""} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_two_metadata_missing_value(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta key="alpha"/> + <meta key="delta"/> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "", "delta": ""} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_missing_key(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta>beta</meta> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"": "beta"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_two_metadata_missing_key(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta>beta</meta> + <meta>gamma</meta> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"": "gamma"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_duplicate_key(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta key="foo">bar</meta> + <meta key="foo">baz</meta> + </metadata> +</server>""" + request = self.deserializer.deserialize(serial_request) + expected = {"foo": "baz"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_canonical_request_from_docs(self): + serial_request = """ +<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" + name="new-server-test" imageId="1" flavorId="1"> + <metadata> + <meta key="My Server Name">Apache1</meta> + </metadata> + <personality> + <file path="/etc/banner.txt">\ +ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ +dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ +IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ +c2lvbi4uLnRoaXMgaXMgdGhlIHBsYWNlIHRvIGdvIG5vdy4g\ +QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ +ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ +dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ +c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ +b25zLiINCg0KLVJpY2hhcmQgQmFjaA==</file> + </personality> +</server>""" + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": { + "My Server Name": "Apache1", + }, + "personality": [ + { + "path": "/etc/banner.txt", + "contents": """\ +ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ +dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ +IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ +c2lvbi4uLnRoaXMgaXMgdGhlIHBsYWNlIHRvIGdvIG5vdy4g\ +QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ +ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ +dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ +c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ +b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", + }, + ], + }} + request = self.deserializer.deserialize(serial_request) + self.assertEqual(request, expected) + + +class TestServerInstanceCreation(test.TestCase): + + def setUp(self): + super(TestServerInstanceCreation, self).setUp() + self.stubs = stubout.StubOutForTesting() + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_auth(self.stubs) + fakes.stub_out_key_pair_funcs(self.stubs) + self.allow_admin = FLAGS.allow_admin_api + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(TestServerInstanceCreation, self).tearDown() + + def _setup_mock_compute_api_for_personality(self): + + class MockComputeAPI(object): + + def __init__(self): + self.injected_files = None + + def create(self, *args, **kwargs): + if 'injected_files' in kwargs: + self.injected_files = kwargs['injected_files'] + else: + self.injected_files = None + return [{'id': '1234', 'display_name': 'fakeinstance'}] + + def set_admin_password(self, *args, **kwargs): + pass + + def make_stub_method(canned_return): + def stub_method(*args, **kwargs): + return canned_return + return stub_method + + compute_api = MockComputeAPI() + self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) + self.stubs.Set(nova.api.openstack.servers.Controller, + '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) + self.stubs.Set(nova.api.openstack.common, + 'get_image_id_from_image_hash', make_stub_method(2)) + return compute_api + + def _create_personality_request_dict(self, personality_files): + server = {} + server['name'] = 'new-server-test' + server['imageId'] = 1 + server['flavorId'] = 1 + if personality_files is not None: + personalities = [] + for path, contents in personality_files: + personalities.append({'path': path, 'contents': contents}) + server['personality'] = personalities + return {'server': server} + + def _get_create_request_json(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/json' + req.method = 'POST' + req.body = json.dumps(body_dict) + return req + + def _run_create_instance_with_mock_compute_api(self, request): + compute_api = self._setup_mock_compute_api_for_personality() + response = request.get_response(fakes.wsgi_app()) + return compute_api, response + + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '<?xml version="1.0" encoding="UTF-8"?>', + '<server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0"', + ' name="%s" imageId="%s" flavorId="%s">' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('<metadata>') + for item in metadata.iteritems(): + body_parts.append('<meta key="%s">%s</meta>' % item) + body_parts.append('</metadata>') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('<personality>') + for file in personalities: + item = (file['path'], file['contents']) + body_parts.append('<file path="%s">%s</file>' % item) + body_parts.append('</personality>') + body_parts.append('</server>') + return ''.join(body_parts) + + def _get_create_request_xml(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.accept = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + + def _create_instance_with_personality_json(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.injected_files + + def _create_instance_with_personality_xml(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.injected_files + + def test_create_instance_with_no_personality(self): + request, response, injected_files = \ + self._create_instance_with_personality_json(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(injected_files, []) + + def test_create_instance_with_no_personality_xml(self): + request, response, injected_files = \ + self._create_instance_with_personality_xml(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(injected_files, []) + + def test_create_instance_with_personality(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, injected_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(injected_files, [(path, contents)]) + + def test_create_instance_with_personality_xml(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, injected_files = \ + self._create_instance_with_personality_xml(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(injected_files, [(path, contents)]) + + def test_create_instance_with_personality_no_path(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['path'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.injected_files, None) + + def _test_create_instance_with_personality_no_path_xml(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + request.body = request.body.replace(' path="/remove/this/path"', '') + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.injected_files, None) + + def test_create_instance_with_personality_no_contents(self): + personality = [('/test/path', + base64.b64encode('remove\nthese\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['contents'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.injected_files, None) + + def test_create_instance_with_personality_not_a_list(self): + personality = [('/test/path', base64.b64encode('test\ncontents\n'))] + body_dict = self._create_personality_request_dict(personality) + body_dict['server']['personality'] = \ + body_dict['server']['personality'][0] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.injected_files, None) + + def test_create_instance_with_personality_with_non_b64_content(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Oh no!"\n' + personality = [(path, contents)] + request, response, injected_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(injected_files, None) + + def test_create_instance_with_three_personalities(self): + files = [ + ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), + ('/etc/motd', 'Enjoy your root access!\n'), + ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), + ] + personality = [] + for path, content in files: + personality.append((path, base64.b64encode(content))) + request, response, injected_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(injected_files, files) + + def test_create_instance_personality_empty_content(self): + path = '/my/file/path' + contents = '' + personality = [(path, contents)] + request, response, injected_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(injected_files, [(path, contents)]) + + def test_create_instance_admin_pass_json(self): + request, response, dummy = \ + self._create_instance_with_personality_json(None) + self.assertEquals(response.status_int, 200) + response = json.loads(response.body) + self.assertTrue('adminPass' in response['server']) + self.assertTrue(response['server']['adminPass'].startswith('fake')) + + def test_create_instance_admin_pass_xml(self): + request, response, dummy = \ + self._create_instance_with_personality_xml(None) + self.assertEquals(response.status_int, 200) + dom = minidom.parseString(response.body) + server = dom.childNodes[0] + self.assertEquals(server.nodeName, 'server') + self.assertTrue(server.getAttribute('adminPass').startswith('fake')) + + if __name__ == "__main__": unittest.main() diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py new file mode 100644 index 000000000..2dda4319b --- /dev/null +++ b/nova/tests/api/openstack/test_users.py @@ -0,0 +1,141 @@ +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import stubout +import webob + +import nova.api +import nova.api.openstack.auth +from nova import context +from nova import flags +from nova import test +from nova.auth.manager import User, Project +from nova.tests.api.openstack import fakes + + +FLAGS = flags.FLAGS +FLAGS.verbose = True + + +def fake_init(self): + self.manager = fakes.FakeAuthManager() + + +def fake_admin_check(self, req): + return True + + +class UsersTest(test.TestCase): + def setUp(self): + super(UsersTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.openstack.users.Controller, '__init__', + fake_init) + self.stubs.Set(nova.api.openstack.users.Controller, '_check_admin', + fake_admin_check) + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.projects = dict(testacct=Project('testacct', + 'testacct', + 'guy1', + 'test', + [])) + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + + self.allow_admin = FLAGS.allow_admin_api + FLAGS.allow_admin_api = True + fakemgr = fakes.FakeAuthManager() + fakemgr.add_user('acc1', User('guy1', 'guy1', 'acc1', + 'fortytwo!', False)) + fakemgr.add_user('acc2', User('guy2', 'guy2', 'acc2', + 'swordfish', True)) + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(UsersTest, self).tearDown() + + def test_get_user_list(self): + req = webob.Request.blank('/v1.0/users') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(len(res_dict['users']), 2) + + def test_get_user_by_id(self): + req = webob.Request.blank('/v1.0/users/guy2') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['name'], 'guy2') + self.assertEqual(res_dict['user']['secret'], 'swordfish') + self.assertEqual(res_dict['user']['admin'], True) + self.assertEqual(res.status_int, 200) + + def test_user_delete(self): + req = webob.Request.blank('/v1.0/users/guy1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('guy1' not in [u.id for u in + fakes.FakeAuthManager.auth_data.values()]) + self.assertEqual(res.status_int, 200) + + def test_user_create(self): + body = dict(user=dict(name='test_guy', + access='acc3', + secret='invasionIsInNormandy', + admin=True)) + req = webob.Request.blank('/v1.0/users') + req.headers["Content-Type"] = "application/json" + req.method = 'POST' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['user']['id'], 'test_guy') + self.assertEqual(res_dict['user']['name'], 'test_guy') + self.assertEqual(res_dict['user']['access'], 'acc3') + self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['admin'], True) + self.assertTrue('test_guy' in [u.id for u in + fakes.FakeAuthManager.auth_data.values()]) + self.assertEqual(len(fakes.FakeAuthManager.auth_data.values()), 3) + + def test_user_update(self): + body = dict(user=dict(name='guy2', + access='acc2', + secret='invasionIsInNormandy')) + req = webob.Request.blank('/v1.0/users/guy2') + req.headers["Content-Type"] = "application/json" + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['name'], 'guy2') + self.assertEqual(res_dict['user']['access'], 'acc2') + self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['admin'], True) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index a40d46749..12d39fd29 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -58,7 +58,7 @@ def zone_get_all_scheduler(*args): dict(id=1, api_url='http://example.com', username='bob', password='xxx'), dict(id=2, api_url='http://example.org', username='alice', - password='qwerty') + password='qwerty'), ] @@ -71,7 +71,7 @@ def zone_get_all_db(context): dict(id=1, api_url='http://example.com', username='bob', password='xxx'), dict(id=2, api_url='http://example.org', username='alice', - password='qwerty') + password='qwerty'), ] @@ -83,7 +83,7 @@ class ZonesTest(test.TestCase): def setUp(self): super(ZonesTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -120,24 +120,27 @@ class ZonesTest(test.TestCase): self.stubs.Set(api, '_call_scheduler', zone_get_all_scheduler_empty) self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('password' in res_dict['zone']) - self.assertEqual(res.status_int, 200) def test_zone_delete(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -146,13 +149,14 @@ class ZonesTest(test.TestCase): body = dict(zone=dict(api_url='http://example.com', username='fred', password='fubar')) req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" req.method = 'POST' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('username' in res_dict['zone']) @@ -160,20 +164,21 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" req.method = 'PUT' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('username' in res_dict['zone']) def test_zone_info(self): FLAGS.zone_name = 'darksecret' - FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' + FLAGS.zone_capabilities = ['cap1=a;b', 'cap2=c;d'] self.stubs.Set(api, '_call_scheduler', zone_caps) body = dict(zone=dict(username='zeb', password='sneaky')) @@ -183,5 +188,5 @@ class ZonesTest(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) self.assertEqual(res_dict['zone']['name'], 'darksecret') - self.assertEqual(res_dict['zone']['cap1'], 'a,b') - self.assertEqual(res_dict['zone']['cap2'], 'c,d') + self.assertEqual(res_dict['zone']['cap1'], 'a;b') + self.assertEqual(res_dict['zone']['cap2'], 'c;d') diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 2c7852214..b1a849cf9 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -21,11 +21,13 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ +import json from nova import test import routes import webob +from nova import exception from nova import wsgi @@ -66,63 +68,164 @@ class Test(test.TestCase): result = webob.Request.blank('/bad').get_response(Router()) self.assertNotEqual(result.body, "Router result") - def test_controller(self): - class Controller(wsgi.Controller): - """Test controller to call from router.""" - test = self +class ControllerTest(test.TestCase): - def show(self, req, id): # pylint: disable-msg=W0622,C0103 - """Default action called for requests with an ID.""" - self.test.assertEqual(req.path_info, '/tests/123') - self.test.assertEqual(id, '123') - return id + class TestRouter(wsgi.Router): - class Router(wsgi.Router): - """Test router.""" + class TestController(wsgi.Controller): - def __init__(self): - mapper = routes.Mapper() - mapper.resource("test", "tests", controller=Controller()) - super(Router, self).__init__(mapper) + _serialization_metadata = { + 'application/xml': { + "attributes": { + "test": ["id"]}}} - result = webob.Request.blank('/tests/123').get_response(Router()) - self.assertEqual(result.body, "123") - result = webob.Request.blank('/test/123').get_response(Router()) - self.assertNotEqual(result.body, "123") + def show(self, req, id): # pylint: disable-msg=W0622,C0103 + return {"test": {"id": id}} + + def __init__(self): + mapper = routes.Mapper() + mapper.resource("test", "tests", controller=self.TestController()) + wsgi.Router.__init__(self, mapper) + + def test_show(self): + request = wsgi.Request.blank('/tests/123') + result = request.get_response(self.TestRouter()) + self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) + + def test_response_content_type_from_accept_xml(self): + request = webob.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_response_content_type_from_accept_json(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_response_content_type_from_query_extension_xml(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_response_content_type_from_query_extension_json(self): + request = wsgi.Request.blank('/tests/123.json') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_response_content_type_default_when_unsupported(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.status_int, 200) + self.assertEqual(result.headers["Content-Type"], "application/json") + + +class RequestTest(test.TestCase): + + def test_request_content_type_missing(self): + request = wsgi.Request.blank('/tests/123') + request.body = "<body />" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + + def test_request_content_type_unsupported(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Content-Type"] = "text/html" + request.body = "asdf<br />" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + + def test_content_type_from_accept_xml(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml, application/json" + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = \ + "application/json; q=0.3, application/xml; q=0.9" + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + def test_content_type_from_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123.json') + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123.invalid') + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + def test_content_type_accept_and_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + request.headers["Accept"] = "application/json" + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + def test_content_type_accept_default(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.best_match_content_type() + self.assertEqual(result, "application/json") class SerializerTest(test.TestCase): - def match(self, url, accept, expect): + def test_xml(self): input_dict = dict(servers=dict(a=(2, 3))) expected_xml = '<servers><a>(2,3)</a></servers>' + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/xml") + result = result.replace('\n', '').replace(' ', '') + self.assertEqual(result, expected_xml) + + def test_json(self): + input_dict = dict(servers=dict(a=(2, 3))) expected_json = '{"servers":{"a":[2,3]}}' - req = webob.Request.blank(url, headers=dict(Accept=accept)) - result = wsgi.Serializer(req.environ).to_content_type(input_dict) + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/json") result = result.replace('\n', '').replace(' ', '') - if expect == 'xml': - self.assertEqual(result, expected_xml) - elif expect == 'json': - self.assertEqual(result, expected_json) - else: - raise "Bad expect value" - - def test_basic(self): - self.match('/servers/4.json', None, expect='json') - self.match('/servers/4', 'application/json', expect='json') - self.match('/servers/4', 'application/xml', expect='xml') - self.match('/servers/4.xml', None, expect='xml') - - def test_defaults_to_json(self): - self.match('/servers/4', None, expect='json') - self.match('/servers/4', 'text/html', expect='json') - - def test_suffix_takes_precedence_over_accept_header(self): - self.match('/servers/4.xml', 'application/json', expect='xml') - self.match('/servers/4.xml.', 'application/json', expect='json') - - def test_deserialize(self): + self.assertEqual(result, expected_json) + + def test_unsupported_content_type(self): + serializer = wsgi.Serializer() + self.assertRaises(exception.InvalidContentType, serializer.serialize, + {}, "text/null") + + def test_deserialize_json(self): + data = """{"a": { + "a1": "1", + "a2": "2", + "bs": ["1", "2", "3", {"c": {"c1": "1"}}], + "d": {"e": "1"}, + "f": "1"}}""" + as_dict = dict(a={ + 'a1': '1', + 'a2': '2', + 'bs': ['1', '2', '3', {'c': dict(c1='1')}], + 'd': {'e': '1'}, + 'f': '1'}) + metadata = {} + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(data, "application/json"), + as_dict) + + def test_deserialize_xml(self): xml = """ <a a1="1" a2="2"> <bs><b>1</b><b>2</b><b>3</b><b><c c1="1"/></b></bs> @@ -137,11 +240,13 @@ class SerializerTest(test.TestCase): 'd': {'e': '1'}, 'f': '1'}) metadata = {'application/xml': dict(plurals={'bs': 'b', 'ts': 't'})} - serializer = wsgi.Serializer({}, metadata) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) def test_deserialize_empty_xml(self): xml = """<a></a>""" as_dict = {"a": {}} - serializer = wsgi.Serializer({}) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer() + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py index d760dc456..5e9a3aa3b 100644 --- a/nova/tests/db/fakes.py +++ b/nova/tests/db/fakes.py @@ -77,7 +77,8 @@ def stub_out_db_instance_api(stubs): 'mac_address': values['mac_address'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], - } + 'os_type': values['os_type']} + return FakeModel(base_options) def fake_network_get_by_instance(context, instance_id): diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index cbd949477..5d7ca98b5 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -32,6 +32,7 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 8 FLAGS.num_networks = 2 FLAGS.fake_network = True +FLAGS.image_service = 'nova.image.local.LocalImageService' flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') diff --git a/nova/tests/integrated/__init__.py b/nova/tests/integrated/__init__.py new file mode 100644 index 000000000..10e0a91d7 --- /dev/null +++ b/nova/tests/integrated/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`integrated` -- Tests whole systems, using mock services where needed +================================= +""" diff --git a/nova/tests/integrated/api/__init__.py b/nova/tests/integrated/api/__init__.py new file mode 100644 index 000000000..5798ab3d1 --- /dev/null +++ b/nova/tests/integrated/api/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`api` -- OpenStack API client, for testing rather than production +================================= +""" diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py new file mode 100644 index 000000000..245eb8c69 --- /dev/null +++ b/nova/tests/integrated/api/client.py @@ -0,0 +1,212 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import httplib +import urlparse + +from nova import log as logging + + +LOG = logging.getLogger('nova.tests.api') + + +class OpenStackApiException(Exception): + def __init__(self, message=None, response=None): + self.response = response + if not message: + message = 'Unspecified error' + + if response: + _status = response.status + _body = response.read() + + message = _('%(message)s\nStatus Code: %(_status)s\n' + 'Body: %(_body)s') % locals() + + super(OpenStackApiException, self).__init__(message) + + +class OpenStackApiAuthenticationException(OpenStackApiException): + def __init__(self, response=None, message=None): + if not message: + message = _("Authentication error") + super(OpenStackApiAuthenticationException, self).__init__(message, + response) + + +class OpenStackApiNotFoundException(OpenStackApiException): + def __init__(self, response=None, message=None): + if not message: + message = _("Item not found") + super(OpenStackApiNotFoundException, self).__init__(message, response) + + +class TestOpenStackClient(object): + """ A really basic OpenStack API client that is under our control, + so we can make changes / insert hooks for testing""" + + def __init__(self, auth_user, auth_key, auth_uri): + super(TestOpenStackClient, self).__init__() + self.auth_result = None + self.auth_user = auth_user + self.auth_key = auth_key + self.auth_uri = auth_uri + + def request(self, url, method='GET', body=None, headers=None): + if headers is None: + headers = {} + + parsed_url = urlparse.urlparse(url) + port = parsed_url.port + hostname = parsed_url.hostname + scheme = parsed_url.scheme + + if scheme == 'http': + conn = httplib.HTTPConnection(hostname, + port=port) + elif scheme == 'https': + conn = httplib.HTTPSConnection(hostname, + port=port) + else: + raise OpenStackApiException("Unknown scheme: %s" % url) + + relative_url = parsed_url.path + if parsed_url.query: + relative_url = relative_url + parsed_url.query + LOG.info(_("Doing %(method)s on %(relative_url)s") % locals()) + if body: + LOG.info(_("Body: %s") % body) + + conn.request(method, relative_url, body, headers) + response = conn.getresponse() + return response + + def _authenticate(self): + if self.auth_result: + return self.auth_result + + auth_uri = self.auth_uri + headers = {'X-Auth-User': self.auth_user, + 'X-Auth-Key': self.auth_key} + response = self.request(auth_uri, + headers=headers) + + http_status = response.status + LOG.debug(_("%(auth_uri)s => code %(http_status)s") % locals()) + + # Until bug732866 is fixed, we can't check this properly... + #if http_status == 401: + if http_status != 204: + raise OpenStackApiAuthenticationException(response=response) + + auth_headers = {} + for k, v in response.getheaders(): + auth_headers[k] = v + + self.auth_result = auth_headers + return self.auth_result + + def api_request(self, relative_uri, check_response_status=None, **kwargs): + auth_result = self._authenticate() + + #NOTE(justinsb): httplib 'helpfully' converts headers to lower case + base_uri = auth_result['x-server-management-url'] + full_uri = base_uri + relative_uri + + headers = kwargs.setdefault('headers', {}) + headers['X-Auth-Token'] = auth_result['x-auth-token'] + + response = self.request(full_uri, **kwargs) + + http_status = response.status + LOG.debug(_("%(relative_uri)s => code %(http_status)s") % locals()) + + if check_response_status: + if not http_status in check_response_status: + if http_status == 404: + raise OpenStackApiNotFoundException(response=response) + else: + raise OpenStackApiException( + message=_("Unexpected status code"), + response=response) + + return response + + def _decode_json(self, response): + body = response.read() + LOG.debug(_("Decoding JSON: %s") % (body)) + return json.loads(body) + + def api_get(self, relative_uri, **kwargs): + kwargs.setdefault('check_response_status', [200]) + response = self.api_request(relative_uri, **kwargs) + return self._decode_json(response) + + def api_post(self, relative_uri, body, **kwargs): + kwargs['method'] = 'POST' + if body: + headers = kwargs.setdefault('headers', {}) + headers['Content-Type'] = 'application/json' + kwargs['body'] = json.dumps(body) + + kwargs.setdefault('check_response_status', [200]) + response = self.api_request(relative_uri, **kwargs) + return self._decode_json(response) + + def api_delete(self, relative_uri, **kwargs): + kwargs['method'] = 'DELETE' + kwargs.setdefault('check_response_status', [200, 202]) + return self.api_request(relative_uri, **kwargs) + + def get_server(self, server_id): + return self.api_get('/servers/%s' % server_id)['server'] + + def get_servers(self, detail=True): + rel_url = '/servers/detail' if detail else '/servers' + return self.api_get(rel_url)['servers'] + + def post_server(self, server): + return self.api_post('/servers', server)['server'] + + def delete_server(self, server_id): + return self.api_delete('/servers/%s' % server_id) + + def get_image(self, image_id): + return self.api_get('/images/%s' % image_id)['image'] + + def get_images(self, detail=True): + rel_url = '/images/detail' if detail else '/images' + return self.api_get(rel_url)['images'] + + def post_image(self, image): + return self.api_post('/images', image)['image'] + + def delete_image(self, image_id): + return self.api_delete('/images/%s' % image_id) + + def get_flavor(self, flavor_id): + return self.api_get('/flavors/%s' % flavor_id)['flavor'] + + def get_flavors(self, detail=True): + rel_url = '/flavors/detail' if detail else '/flavors' + return self.api_get(rel_url)['flavors'] + + def post_flavor(self, flavor): + return self.api_post('/flavors', flavor)['flavor'] + + def delete_flavor(self, flavor_id): + return self.api_delete('/flavors/%s' % flavor_id) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index b195fa520..cf8ee7eff 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -38,6 +38,8 @@ from nova import test from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils +from nova.image import local from nova.objectstore import image @@ -76,6 +78,12 @@ class CloudTestCase(test.TestCase): project=self.project) host = self.network.get_network_host(self.context.elevated()) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} + + self.stubs.Set(local.LocalImageService, 'show', fake_show) + self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) + def tearDown(self): network_ref = db.project_get_network(self.context, self.project.id) @@ -122,7 +130,7 @@ class CloudTestCase(test.TestCase): self.cloud.allocate_address(self.context) inst = db.instance_create(self.context, {'host': self.compute.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.associate_address(self.context, instance_id=ec2_id, public_ip=address) @@ -158,12 +166,12 @@ class CloudTestCase(test.TestCase): vol2 = db.volume_create(self.context, {}) result = self.cloud.describe_volumes(self.context) self.assertEqual(len(result['volumeSet']), 2) - volume_id = cloud.id_to_ec2_id(vol2['id'], 'vol-%08x') + volume_id = ec2utils.id_to_ec2_id(vol2['id'], 'vol-%08x') result = self.cloud.describe_volumes(self.context, volume_id=[volume_id]) self.assertEqual(len(result['volumeSet']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['volumeSet'][0]['volumeId']), + ec2utils.ec2_id_to_id(result['volumeSet'][0]['volumeId']), vol2['id']) db.volume_destroy(self.context, vol1['id']) db.volume_destroy(self.context, vol2['id']) @@ -188,8 +196,10 @@ class CloudTestCase(test.TestCase): def test_describe_instances(self): """Makes sure describe_instances works and filters results.""" inst1 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host1'}) inst2 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host2'}) comp1 = db.service_create(self.context, {'host': 'host1', 'availability_zone': 'zone1', @@ -200,7 +210,7 @@ class CloudTestCase(test.TestCase): result = self.cloud.describe_instances(self.context) result = result['reservationSet'][0] self.assertEqual(len(result['instancesSet']), 2) - instance_id = cloud.id_to_ec2_id(inst2['id']) + instance_id = ec2utils.id_to_ec2_id(inst2['id']) result = self.cloud.describe_instances(self.context, instance_id=[instance_id]) result = result['reservationSet'][0] @@ -215,10 +225,9 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, comp2['id']) def test_console_output(self): - image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type max_count = 1 - kwargs = {'image_id': image_id, + kwargs = {'image_id': 'ami-1', 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) @@ -234,8 +243,7 @@ class CloudTestCase(test.TestCase): greenthread.sleep(0.3) def test_ajax_console(self): - image_id = FLAGS.default_image - kwargs = {'image_id': image_id} + kwargs = {'image_id': 'ami-1'} rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] greenthread.sleep(0.3) @@ -347,7 +355,7 @@ class CloudTestCase(test.TestCase): def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.update_instance(self.context, ec2_id, display_name='c00l 1m4g3') inst = db.instance_get(self.context, inst['id']) @@ -365,7 +373,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_display_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), display_name='c00l v0lum3') vol = db.volume_get(self.context, vol['id']) self.assertEqual('c00l v0lum3', vol['display_name']) @@ -374,7 +382,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_wont_update_private_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), mountpoint='/not/here') vol = db.volume_get(self.context, vol['id']) self.assertEqual(None, vol['mountpoint']) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 58493d7ac..3651f4cef 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -20,6 +20,7 @@ Tests For Compute """ import datetime +import mox from nova import compute from nova import context @@ -27,15 +28,20 @@ from nova import db from nova import exception from nova import flags from nova import log as logging +from nova import rpc from nova import test from nova import utils from nova.auth import manager from nova.compute import instance_types - +from nova.compute import manager as compute_manager +from nova.compute import power_state +from nova.db.sqlalchemy import models +from nova.image import local LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS flags.DECLARE('stub_network', 'nova.compute.manager') +flags.DECLARE('live_migration_retry_count', 'nova.compute.manager') class ComputeTestCase(test.TestCase): @@ -52,6 +58,11 @@ class ComputeTestCase(test.TestCase): self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext('fake', 'fake', False) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} + + self.stubs.Set(local.LocalImageService, 'show', fake_show) + def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) @@ -60,7 +71,7 @@ class ComputeTestCase(test.TestCase): def _create_instance(self, params={}): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id @@ -78,6 +89,21 @@ class ComputeTestCase(test.TestCase): 'project_id': self.project.id} return db.security_group_create(self.context, values) + def _get_dummy_instance(self): + """Get mock-return-value instance object + Use this when any testcase executed later than test_run_terminate + """ + vol1 = models.Volume() + vol1['id'] = 1 + vol2 = models.Volume() + vol2['id'] = 2 + instance_ref = models.Instance() + instance_ref['id'] = 1 + instance_ref['volumes'] = [vol1, vol2] + instance_ref['hostname'] = 'i-00000001' + instance_ref['host'] = 'dummy' + return instance_ref + def test_create_instance_defaults_display_name(self): """Verify that an instance cannot be created without a display_name.""" cases = [dict(), dict(display_name=None)] @@ -296,3 +322,256 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(self.context, instance_id) type = instance_types.get_by_flavor_id("1") self.assertEqual(type, 'm1.tiny') + + def _setup_other_managers(self): + self.volume_manager = utils.import_object(FLAGS.volume_manager) + self.network_manager = utils.import_object(FLAGS.network_manager) + self.compute_driver = utils.import_object(FLAGS.compute_driver) + + def test_pre_live_migration_instance_has_no_fixed_ip(self): + """Confirm raising exception if instance doesn't have fixed_ip.""" + instance_ref = self._get_dummy_instance() + c = context.get_admin_context() + i_id = instance_ref['id'] + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn(None) + + self.compute.db = dbmock + self.mox.ReplayAll() + self.assertRaises(exception.NotFound, + self.compute.pre_live_migration, + c, instance_ref['id']) + + def test_pre_live_migration_instance_has_volume(self): + """Confirm setup_compute_volume is called when volume is mounted.""" + i_ref = self._get_dummy_instance() + c = context.get_admin_context() + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + volmock = self.mox.CreateMock(self.volume_manager) + netmock = self.mox.CreateMock(self.network_manager) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + dbmock.instance_get_fixed_address(c, i_ref['id']).AndReturn('dummy') + for i in range(len(i_ref['volumes'])): + vid = i_ref['volumes'][i]['id'] + volmock.setup_compute_volume(c, vid).InAnyOrder('g1') + netmock.setup_compute_network(c, i_ref['id']) + drivermock.ensure_filtering_rules_for_instance(i_ref) + + self.compute.db = dbmock + self.compute.volume_manager = volmock + self.compute.network_manager = netmock + self.compute.driver = drivermock + + self.mox.ReplayAll() + ret = self.compute.pre_live_migration(c, i_ref['id']) + self.assertEqual(ret, None) + + def test_pre_live_migration_instance_has_no_volume(self): + """Confirm log meg when instance doesn't mount any volumes.""" + i_ref = self._get_dummy_instance() + i_ref['volumes'] = [] + c = context.get_admin_context() + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + netmock = self.mox.CreateMock(self.network_manager) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + dbmock.instance_get_fixed_address(c, i_ref['id']).AndReturn('dummy') + self.mox.StubOutWithMock(compute_manager.LOG, 'info') + compute_manager.LOG.info(_("%s has no volume."), i_ref['hostname']) + netmock.setup_compute_network(c, i_ref['id']) + drivermock.ensure_filtering_rules_for_instance(i_ref) + + self.compute.db = dbmock + self.compute.network_manager = netmock + self.compute.driver = drivermock + + self.mox.ReplayAll() + ret = self.compute.pre_live_migration(c, i_ref['id']) + self.assertEqual(ret, None) + + def test_pre_live_migration_setup_compute_node_fail(self): + """Confirm operation setup_compute_network() fails. + + It retries and raise exception when timeout exceeded. + + """ + + i_ref = self._get_dummy_instance() + c = context.get_admin_context() + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + netmock = self.mox.CreateMock(self.network_manager) + volmock = self.mox.CreateMock(self.volume_manager) + + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + dbmock.instance_get_fixed_address(c, i_ref['id']).AndReturn('dummy') + for i in range(len(i_ref['volumes'])): + volmock.setup_compute_volume(c, i_ref['volumes'][i]['id']) + for i in range(FLAGS.live_migration_retry_count): + netmock.setup_compute_network(c, i_ref['id']).\ + AndRaise(exception.ProcessExecutionError()) + + self.compute.db = dbmock + self.compute.network_manager = netmock + self.compute.volume_manager = volmock + + self.mox.ReplayAll() + self.assertRaises(exception.ProcessExecutionError, + self.compute.pre_live_migration, + c, i_ref['id']) + + def test_live_migration_works_correctly_with_volume(self): + """Confirm check_for_export to confirm volume health check.""" + i_ref = self._get_dummy_instance() + c = context.get_admin_context() + topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, {"method": "check_for_export", + "args": {'instance_id': i_ref['id']}}) + dbmock.queue_get_for(c, FLAGS.compute_topic, i_ref['host']).\ + AndReturn(topic) + rpc.call(c, topic, {"method": "pre_live_migration", + "args": {'instance_id': i_ref['id']}}) + self.mox.StubOutWithMock(self.compute.driver, 'live_migration') + self.compute.driver.live_migration(c, i_ref, i_ref['host'], + self.compute.post_live_migration, + self.compute.recover_live_migration) + + self.compute.db = dbmock + self.mox.ReplayAll() + ret = self.compute.live_migration(c, i_ref['id'], i_ref['host']) + self.assertEqual(ret, None) + + def test_live_migration_dest_raises_exception(self): + """Confirm exception when pre_live_migration fails.""" + i_ref = self._get_dummy_instance() + c = context.get_admin_context() + topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, {"method": "check_for_export", + "args": {'instance_id': i_ref['id']}}) + dbmock.queue_get_for(c, FLAGS.compute_topic, i_ref['host']).\ + AndReturn(topic) + rpc.call(c, topic, {"method": "pre_live_migration", + "args": {'instance_id': i_ref['id']}}).\ + AndRaise(rpc.RemoteError('', '', '')) + dbmock.instance_update(c, i_ref['id'], {'state_description': 'running', + 'state': power_state.RUNNING, + 'host': i_ref['host']}) + for v in i_ref['volumes']: + dbmock.volume_update(c, v['id'], {'status': 'in-use'}) + + self.compute.db = dbmock + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + self.compute.live_migration, + c, i_ref['id'], i_ref['host']) + + def test_live_migration_dest_raises_exception_no_volume(self): + """Same as above test(input pattern is different) """ + i_ref = self._get_dummy_instance() + i_ref['volumes'] = [] + c = context.get_admin_context() + topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + dbmock.queue_get_for(c, FLAGS.compute_topic, i_ref['host']).\ + AndReturn(topic) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, topic, {"method": "pre_live_migration", + "args": {'instance_id': i_ref['id']}}).\ + AndRaise(rpc.RemoteError('', '', '')) + dbmock.instance_update(c, i_ref['id'], {'state_description': 'running', + 'state': power_state.RUNNING, + 'host': i_ref['host']}) + + self.compute.db = dbmock + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + self.compute.live_migration, + c, i_ref['id'], i_ref['host']) + + def test_live_migration_works_correctly_no_volume(self): + """Confirm live_migration() works as expected correctly.""" + i_ref = self._get_dummy_instance() + i_ref['volumes'] = [] + c = context.get_admin_context() + topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_ref['id']).AndReturn(i_ref) + self.mox.StubOutWithMock(rpc, 'call') + dbmock.queue_get_for(c, FLAGS.compute_topic, i_ref['host']).\ + AndReturn(topic) + rpc.call(c, topic, {"method": "pre_live_migration", + "args": {'instance_id': i_ref['id']}}) + self.mox.StubOutWithMock(self.compute.driver, 'live_migration') + self.compute.driver.live_migration(c, i_ref, i_ref['host'], + self.compute.post_live_migration, + self.compute.recover_live_migration) + + self.compute.db = dbmock + self.mox.ReplayAll() + ret = self.compute.live_migration(c, i_ref['id'], i_ref['host']) + self.assertEqual(ret, None) + + def test_post_live_migration_working_correctly(self): + """Confirm post_live_migration() works as expected correctly.""" + dest = 'desthost' + flo_addr = '1.2.1.2' + + # Preparing datas + c = context.get_admin_context() + instance_id = self._create_instance() + i_ref = db.instance_get(c, instance_id) + db.instance_update(c, i_ref['id'], {'state_description': 'migrating', + 'state': power_state.PAUSED}) + v_ref = db.volume_create(c, {'size': 1, 'instance_id': instance_id}) + fix_addr = db.fixed_ip_create(c, {'address': '1.1.1.1', + 'instance_id': instance_id}) + fix_ref = db.fixed_ip_get_by_address(c, fix_addr) + flo_ref = db.floating_ip_create(c, {'address': flo_addr, + 'fixed_ip_id': fix_ref['id']}) + # reload is necessary before setting mocks + i_ref = db.instance_get(c, instance_id) + + # Preparing mocks + self.mox.StubOutWithMock(self.compute.volume_manager, + 'remove_compute_volume') + for v in i_ref['volumes']: + self.compute.volume_manager.remove_compute_volume(c, v['id']) + self.mox.StubOutWithMock(self.compute.driver, 'unfilter_instance') + self.compute.driver.unfilter_instance(i_ref) + + # executing + self.mox.ReplayAll() + ret = self.compute.post_live_migration(c, i_ref, dest) + + # make sure every data is rewritten to dest + i_ref = db.instance_get(c, i_ref['id']) + c1 = (i_ref['host'] == dest) + flo_refs = db.floating_ip_get_all_by_host(c, dest) + c2 = (len(flo_refs) != 0 and flo_refs[0]['address'] == flo_addr) + + # post operaton + self.assertTrue(c1 and c2) + db.instance_destroy(c, instance_id) + db.volume_destroy(c, v_ref['id']) + db.floating_ip_destroy(c, flo_addr) diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 49ff24413..d47c70d88 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -57,7 +57,7 @@ class ConsoleTestCase(test.TestCase): inst = {} #inst['host'] = self.host #inst['name'] = 'instance-1234' - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..80e4d2e1f 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -59,6 +59,7 @@ class DirectTestCase(test.TestCase): req.headers['X-OpenStack-User'] = 'user1' req.headers['X-OpenStack-Project'] = 'proj1' resp = req.get_response(self.auth_router) + self.assertEqual(resp.status_int, 200) data = json.loads(resp.body) self.assertEqual(data['user'], 'user1') self.assertEqual(data['project'], 'proj1') @@ -69,6 +70,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'json=%s' % json.dumps({'data': 'foo'}) resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') @@ -78,6 +80,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'data=foo' resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') @@ -90,8 +93,7 @@ class DirectTestCase(test.TestCase): class DirectCloudTestCase(test_cloud.CloudTestCase): def setUp(self): super(DirectCloudTestCase, self).setUp() - compute_handle = compute.API(image_service=self.cloud.image_service, - network_api=self.cloud.network_api, + compute_handle = compute.API(network_api=self.cloud.network_api, volume_api=self.cloud.volume_api) direct.register_service('compute', compute_handle) self.router = direct.JsonParamsMiddleware(direct.Router()) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index e6da6112a..1fbaf304f 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,26 +14,29 @@ # License for the specific language governing permissions and limitations # under the License. +import errno import os +import select from nova import test -from nova.utils import parse_mailmap, str_dict_replace +from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): def test_authors_up_to_date(self): - if os.path.exists('.bzr'): + topdir = os.path.normpath(os.path.dirname(__file__) + '/../../') + if os.path.exists(os.path.join(topdir, '.bzr')): contributors = set() - mailmap = parse_mailmap('.mailmap') + mailmap = parse_mailmap(os.path.join(topdir, '.mailmap')) import bzrlib.workingtree - tree = bzrlib.workingtree.WorkingTree.open('.') + tree = bzrlib.workingtree.WorkingTree.open(topdir) tree.lock_read() try: parents = tree.get_parent_ids() g = tree.branch.repository.get_graph() - for p in parents[1:]: + for p in parents: rev_ids = [r for r, _ in g.iter_ancestry(parents) if r != "null:"] revs = tree.branch.repository.get_revisions(rev_ids) @@ -42,7 +45,8 @@ class ProjectTestCase(test.TestCase): email = author.split(' ')[-1] contributors.add(str_dict_replace(email, mailmap)) - authors_file = open('Authors', 'r').read() + authors_file = open(os.path.join(topdir, 'Authors'), + 'r').read() missing = set() for contributor in contributors: @@ -55,3 +59,47 @@ class ProjectTestCase(test.TestCase): '%r not listed in Authors' % missing) finally: tree.unlock() + + +class LockTestCase(test.TestCase): + def test_synchronized_wrapped_function_metadata(self): + @synchronized('whatever') + def foo(): + """Bar""" + pass + self.assertEquals(foo.__doc__, 'Bar', "Wrapped function's docstring " + "got lost") + self.assertEquals(foo.__name__, 'foo', "Wrapped function's name " + "got mangled") + + def test_synchronized(self): + rpipe1, wpipe1 = os.pipe() + rpipe2, wpipe2 = os.pipe() + + @synchronized('testlock') + def f(rpipe, wpipe): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + + pid = os.fork() + if pid > 0: + os.close(wpipe1) + os.close(rpipe2) + + f(rpipe1, wpipe2) + else: + os.close(rpipe1) + os.close(wpipe2) + + f(rpipe2, wpipe1) + os._exit(0) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index ce1c77210..1e634b388 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -20,6 +20,7 @@ Unit Tests for network code """ import IPy import os +import time from nova import context from nova import db @@ -29,11 +30,153 @@ from nova import log as logging from nova import test from nova import utils from nova.auth import manager +from nova.network import linux_net FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.network') +class IptablesManagerTestCase(test.TestCase): + sample_filter = ['#Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*filter', + ':INPUT ACCEPT [2223527:305688874]', + ':FORWARD ACCEPT [0:0]', + ':OUTPUT ACCEPT [2172501:140856656]', + ':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-filter-top - [0:0]', + '-A FORWARD -j nova-filter-top ', + '-A OUTPUT -j nova-filter-top ', + '-A nova-filter-top -j nova-compute-local ', + '-A INPUT -j nova-compute-INPUT ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A FORWARD -j nova-compute-FORWARD ', + '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', + '-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT ', + '-A FORWARD -i virbr0 -o virbr0 -j ACCEPT ', + '-A FORWARD -o virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + '-A FORWARD -i virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] + + sample_nat = ['# Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*nat', + ':PREROUTING ACCEPT [3936:762355]', + ':INPUT ACCEPT [2447:225266]', + ':OUTPUT ACCEPT [63491:4191863]', + ':POSTROUTING ACCEPT [63112:4108641]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]', + ':nova-postrouting-bottom - [0:0]', + '-A PREROUTING -j nova-compute-PREROUTING ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A POSTROUTING -j nova-compute-POSTROUTING ', + '-A POSTROUTING -j nova-postrouting-bottom ', + '-A nova-postrouting-bottom -j nova-compute-SNATTING ', + '-A nova-compute-SNATTING -j nova-compute-floating-ip-snat ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] + + def setUp(self): + super(IptablesManagerTestCase, self).setUp() + self.manager = linux_net.IptablesManager() + + def test_filter_rules_are_wrapped(self): + current_lines = self.sample_filter + + table = self.manager.ipv4['filter'] + table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager._modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' in new_lines) + + table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager._modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' not in new_lines) + + def test_nat_rules(self): + current_lines = self.sample_nat + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['nat']) + + for line in [':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains " + "went missing.") + + seen_lines = set() + for line in new_lines: + line = line.strip() + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) + + last_postrouting_line = '' + + for line in new_lines: + if line.startswith('-A POSTROUTING'): + last_postrouting_line = line + + self.assertTrue('-j nova-postrouting-bottom' in last_postrouting_line, + "Last POSTROUTING rule does not jump to " + "nova-postouting-bottom: %s" % last_postrouting_line) + + for chain in ['POSTROUTING', 'PREROUTING', 'OUTPUT']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) + + def test_filter_rules(self): + current_lines = self.sample_filter + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['filter']) + + for line in [':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains" + " went missing.") + + seen_lines = set() + for line in new_lines: + line = line.strip() + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) + + for chain in ['FORWARD', 'OUTPUT']: + for line in new_lines: + if line.startswith('-A %s' % chain): + self.assertTrue('-j nova-filter-top' in line, + "First %s rule does not " + "jump to nova-filter-top" % chain) + break + + self.assertTrue('-A nova-filter-top ' + '-j run_tests.py-local' in new_lines, + "nova-filter-top does not jump to wrapped local chain") + + for chain in ['INPUT', 'OUTPUT', 'FORWARD']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) + + class NetworkTestCase(test.TestCase): """Test cases for network code""" def setUp(self): @@ -321,6 +464,31 @@ class NetworkTestCase(test.TestCase): network['id']) self.assertEqual(ip_count, num_available_ips) + def test_dhcp_lease_output(self): + admin_ctxt = context.get_admin_context() + address = self._create_address(0, self.instance_id) + lease_ip(address) + network_ref = db.network_get_by_instance(admin_ctxt, self.instance_id) + leases = linux_net.get_dhcp_leases(context.get_admin_context(), + network_ref['id']) + for line in leases.split('\n'): + seconds, mac, ip, hostname, client_id = line.split(' ') + self.assertTrue(int(seconds) > time.time(), 'Lease expires in ' + 'the past') + octets = mac.split(':') + self.assertEqual(len(octets), 6, "Wrong number of octets " + "in %s" % (max,)) + for octet in octets: + self.assertEqual(len(octet), 2, "Oddly sized octet: %s" + % (octet,)) + # This will throw an exception if the octet is invalid + int(octet, 16) + + # And this will raise an exception in case of an invalid IP + IPy.IP(ip) + + release_ip(address) + def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" @@ -343,13 +511,13 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'add', + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("ISSUE_IP: %s, %s ", out, err) @@ -359,11 +527,11 @@ def release_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'del', + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("RELEASE_IP: %s, %s ", out, err) diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 4ecb36b54..c65bc459d 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -20,11 +20,12 @@ from nova import compute from nova import context from nova import db from nova import flags +from nova import network from nova import quota from nova import test from nova import utils +from nova import volume from nova.auth import manager -from nova.api.ec2 import cloud from nova.compute import instance_types @@ -32,6 +33,12 @@ FLAGS = flags.FLAGS class QuotaTestCase(test.TestCase): + + class StubImageService(object): + + def show(self, *args, **kwargs): + return {"properties": {}} + def setUp(self): super(QuotaTestCase, self).setUp() self.flags(connection_type='fake', @@ -41,7 +48,6 @@ class QuotaTestCase(test.TestCase): quota_gigabytes=20, quota_floating_ips=1) - self.cloud = cloud.CloudController() self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('admin', 'admin', 'admin') @@ -57,7 +63,7 @@ class QuotaTestCase(test.TestCase): def _create_instance(self, cores=2): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -118,12 +124,12 @@ class QuotaTestCase(test.TestCase): for i in range(FLAGS.quota_instances): instance_id = self._create_instance() instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, instance_type='m1.small', - image_id='fake') + image_id=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -131,12 +137,12 @@ class QuotaTestCase(test.TestCase): instance_ids = [] instance_id = self._create_instance(cores=4) instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, instance_type='m1.small', - image_id='fake') + image_id=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -145,9 +151,12 @@ class QuotaTestCase(test.TestCase): for i in range(FLAGS.quota_volumes): volume_id = self._create_volume() volume_ids.append(volume_id) - self.assertRaises(quota.QuotaError, self.cloud.create_volume, - self.context, - size=10) + self.assertRaises(quota.QuotaError, + volume.API().create, + self.context, + size=10, + name='', + description='') for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -156,9 +165,11 @@ class QuotaTestCase(test.TestCase): volume_id = self._create_volume(size=20) volume_ids.append(volume_id) self.assertRaises(quota.QuotaError, - self.cloud.create_volume, + volume.API().create, self.context, - size=10) + size=10, + name='', + description='') for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -172,7 +183,8 @@ class QuotaTestCase(test.TestCase): # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertRaises(quota.QuotaError, self.cloud.allocate_address, + self.assertRaises(quota.QuotaError, + network.API().allocate_floating_ip, self.context) db.floating_ip_destroy(context.get_admin_context(), address) @@ -187,3 +199,67 @@ class QuotaTestCase(test.TestCase): instance_type='m1.small', image_id='fake', metadata=metadata) + + def test_allowed_injected_files(self): + self.assertEqual( + quota.allowed_injected_files(self.context), + FLAGS.quota_max_injected_files) + + def _create_with_injected_files(self, files): + api = compute.API(image_service=self.StubImageService()) + api.create(self.context, min_count=1, max_count=1, + instance_type='m1.small', image_id='fake', + injected_files=files) + + def test_no_injected_files(self): + api = compute.API(image_service=self.StubImageService()) + api.create(self.context, instance_type='m1.small', image_id='fake') + + def test_max_injected_files(self): + files = [] + for i in xrange(FLAGS.quota_max_injected_files): + files.append(('/my/path%d' % i, 'config = test\n')) + self._create_with_injected_files(files) # no QuotaError + + def test_too_many_injected_files(self): + files = [] + for i in xrange(FLAGS.quota_max_injected_files + 1): + files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) + self.assertRaises(quota.QuotaError, + self._create_with_injected_files, files) + + def test_allowed_injected_file_content_bytes(self): + self.assertEqual( + quota.allowed_injected_file_content_bytes(self.context), + FLAGS.quota_max_injected_file_content_bytes) + + def test_max_injected_file_content_bytes(self): + max = FLAGS.quota_max_injected_file_content_bytes + content = ''.join(['a' for i in xrange(max)]) + files = [('/test/path', content)] + self._create_with_injected_files(files) # no QuotaError + + def test_too_many_injected_file_content_bytes(self): + max = FLAGS.quota_max_injected_file_content_bytes + content = ''.join(['a' for i in xrange(max + 1)]) + files = [('/test/path', content)] + self.assertRaises(quota.QuotaError, + self._create_with_injected_files, files) + + def test_allowed_injected_file_path_bytes(self): + self.assertEqual( + quota.allowed_injected_file_path_bytes(self.context), + FLAGS.quota_max_injected_file_path_bytes) + + def test_max_injected_file_path_bytes(self): + max = FLAGS.quota_max_injected_file_path_bytes + path = ''.join(['a' for i in xrange(max)]) + files = [(path, 'config = quotatest')] + self._create_with_injected_files(files) # no QuotaError + + def test_too_many_injected_file_path_bytes(self): + max = FLAGS.quota_max_injected_file_path_bytes + path = ''.join(['a' for i in xrange(max + 1)]) + files = [(path, 'config = quotatest')] + self.assertRaises(quota.QuotaError, + self._create_with_injected_files, files) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index b6888c4d2..244e43bd9 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -20,10 +20,12 @@ Tests For Scheduler """ import datetime +import mox from mox import IgnoreArg from nova import context from nova import db +from nova import exception from nova import flags from nova import service from nova import test @@ -32,11 +34,14 @@ from nova import utils from nova.auth import manager as auth_manager from nova.scheduler import manager from nova.scheduler import driver +from nova.compute import power_state +from nova.db.sqlalchemy import models FLAGS = flags.FLAGS flags.DECLARE('max_cores', 'nova.scheduler.simple') flags.DECLARE('stub_network', 'nova.compute.manager') +flags.DECLARE('instances_path', 'nova.compute.manager') class TestDriver(driver.Scheduler): @@ -54,6 +59,34 @@ class SchedulerTestCase(test.TestCase): super(SchedulerTestCase, self).setUp() self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') + def _create_compute_service(self): + """Create compute-manager(ComputeNode and Service record).""" + ctxt = context.get_admin_context() + dic = {'host': 'dummy', 'binary': 'nova-compute', 'topic': 'compute', + 'report_count': 0, 'availability_zone': 'dummyzone'} + s_ref = db.service_create(ctxt, dic) + + dic = {'service_id': s_ref['id'], + 'vcpus': 16, 'memory_mb': 32, 'local_gb': 100, + 'vcpus_used': 16, 'memory_mb_used': 32, 'local_gb_used': 10, + 'hypervisor_type': 'qemu', 'hypervisor_version': 12003, + 'cpu_info': ''} + db.compute_node_create(ctxt, dic) + + return db.service_get(ctxt, s_ref['id']) + + def _create_instance(self, **kwargs): + """Create a test instance""" + ctxt = context.get_admin_context() + inst = {} + inst['user_id'] = 'admin' + inst['project_id'] = kwargs.get('project_id', 'fake') + inst['host'] = kwargs.get('host', 'dummy') + inst['vcpus'] = kwargs.get('vcpus', 1) + inst['memory_mb'] = kwargs.get('memory_mb', 10) + inst['local_gb'] = kwargs.get('local_gb', 20) + return db.instance_create(ctxt, inst) + def test_fallback(self): scheduler = manager.SchedulerManager() self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) @@ -76,6 +109,73 @@ class SchedulerTestCase(test.TestCase): self.mox.ReplayAll() scheduler.named_method(ctxt, 'topic', num=7) + def test_show_host_resources_host_not_exit(self): + """A host given as an argument does not exists.""" + + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + + try: + scheduler.show_host_resources(ctxt, dest) + except exception.NotFound, e: + c1 = (e.message.find(_("does not exist or is not a " + "compute node.")) >= 0) + self.assertTrue(c1) + + def _dic_is_equal(self, dic1, dic2, keys=None): + """Compares 2 dictionary contents(Helper method)""" + if not keys: + keys = ['vcpus', 'memory_mb', 'local_gb', + 'vcpus_used', 'memory_mb_used', 'local_gb_used'] + + for key in keys: + if not (dic1[key] == dic2[key]): + return False + return True + + def test_show_host_resources_no_project(self): + """No instance are running on the given host.""" + + scheduler = manager.SchedulerManager() + ctxt = context.get_admin_context() + s_ref = self._create_compute_service() + + result = scheduler.show_host_resources(ctxt, s_ref['host']) + + # result checking + c1 = ('resource' in result and 'usage' in result) + compute_node = s_ref['compute_node'][0] + c2 = self._dic_is_equal(result['resource'], compute_node) + c3 = result['usage'] == {} + self.assertTrue(c1 and c2 and c3) + db.service_destroy(ctxt, s_ref['id']) + + def test_show_host_resources_works_correctly(self): + """Show_host_resources() works correctly as expected.""" + + scheduler = manager.SchedulerManager() + ctxt = context.get_admin_context() + s_ref = self._create_compute_service() + i_ref1 = self._create_instance(project_id='p-01', host=s_ref['host']) + i_ref2 = self._create_instance(project_id='p-02', vcpus=3, + host=s_ref['host']) + + result = scheduler.show_host_resources(ctxt, s_ref['host']) + + c1 = ('resource' in result and 'usage' in result) + compute_node = s_ref['compute_node'][0] + c2 = self._dic_is_equal(result['resource'], compute_node) + c3 = result['usage'].keys() == ['p-01', 'p-02'] + keys = ['vcpus', 'memory_mb', 'local_gb'] + c4 = self._dic_is_equal(result['usage']['p-01'], i_ref1, keys) + c5 = self._dic_is_equal(result['usage']['p-02'], i_ref2, keys) + self.assertTrue(c1 and c2 and c3 and c4 and c5) + + db.service_destroy(ctxt, s_ref['id']) + db.instance_destroy(ctxt, i_ref1['id']) + db.instance_destroy(ctxt, i_ref2['id']) + class ZoneSchedulerTestCase(test.TestCase): """Test case for zone scheduler""" @@ -155,26 +255,235 @@ class SimpleDriverTestCase(test.TestCase): def _create_instance(self, **kwargs): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() + inst['vcpus'] = kwargs.get('vcpus', 1) inst['ami_launch_index'] = 0 - inst['vcpus'] = 1 inst['availability_zone'] = kwargs.get('availability_zone', None) + inst['host'] = kwargs.get('host', 'dummy') + inst['memory_mb'] = kwargs.get('memory_mb', 20) + inst['local_gb'] = kwargs.get('local_gb', 30) + inst['launched_on'] = kwargs.get('launghed_on', 'dummy') + inst['state_description'] = kwargs.get('state_description', 'running') + inst['state'] = kwargs.get('state', power_state.RUNNING) return db.instance_create(self.context, inst)['id'] def _create_volume(self): """Create a test volume""" vol = {} - vol['image_id'] = 'ami-test' - vol['reservation_id'] = 'r-fakeres' vol['size'] = 1 vol['availability_zone'] = 'test' return db.volume_create(self.context, vol)['id'] + def _create_compute_service(self, **kwargs): + """Create a compute service.""" + + dic = {'binary': 'nova-compute', 'topic': 'compute', + 'report_count': 0, 'availability_zone': 'dummyzone'} + dic['host'] = kwargs.get('host', 'dummy') + s_ref = db.service_create(self.context, dic) + if 'created_at' in kwargs.keys() or 'updated_at' in kwargs.keys(): + t = datetime.datetime.utcnow() - datetime.timedelta(0) + dic['created_at'] = kwargs.get('created_at', t) + dic['updated_at'] = kwargs.get('updated_at', t) + db.service_update(self.context, s_ref['id'], dic) + + dic = {'service_id': s_ref['id'], + 'vcpus': 16, 'memory_mb': 32, 'local_gb': 100, + 'vcpus_used': 16, 'local_gb_used': 10, + 'hypervisor_type': 'qemu', 'hypervisor_version': 12003, + 'cpu_info': ''} + dic['memory_mb_used'] = kwargs.get('memory_mb_used', 32) + dic['hypervisor_type'] = kwargs.get('hypervisor_type', 'qemu') + dic['hypervisor_version'] = kwargs.get('hypervisor_version', 12003) + db.compute_node_create(self.context, dic) + return db.service_get(self.context, s_ref['id']) + + def test_doesnt_report_disabled_hosts_as_up(self): + """Ensures driver doesn't find hosts before they are enabled""" + # NOTE(vish): constructing service without create method + # because we are going to use it without queue + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + s2 = db.service_get_by_args(self.context, 'host2', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': True}) + db.service_update(self.context, s2['id'], {'disabled': True}) + hosts = self.scheduler.driver.hosts_up(self.context, 'compute') + self.assertEqual(0, len(hosts)) + compute1.kill() + compute2.kill() + + def test_reports_enabled_hosts_as_up(self): + """Ensures driver can find the hosts that are up""" + # NOTE(vish): constructing service without create method + # because we are going to use it without queue + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + hosts = self.scheduler.driver.hosts_up(self.context, 'compute') + self.assertEqual(2, len(hosts)) + compute1.kill() + compute2.kill() + + def test_least_busy_host_gets_instance(self): + """Ensures the host with less cores gets the next one""" + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + instance_id1 = self._create_instance() + compute1.run_instance(self.context, instance_id1) + instance_id2 = self._create_instance() + host = self.scheduler.driver.schedule_run_instance(self.context, + instance_id2) + self.assertEqual(host, 'host2') + compute1.terminate_instance(self.context, instance_id1) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + compute2.kill() + + def test_specific_host_gets_instance(self): + """Ensures if you set availability_zone it launches on that zone""" + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + instance_id1 = self._create_instance() + compute1.run_instance(self.context, instance_id1) + instance_id2 = self._create_instance(availability_zone='nova:host1') + host = self.scheduler.driver.schedule_run_instance(self.context, + instance_id2) + self.assertEqual('host1', host) + compute1.terminate_instance(self.context, instance_id1) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + compute2.kill() + + def test_wont_sechedule_if_specified_host_is_down(self): + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + now = datetime.datetime.utcnow() + delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) + past = now - delta + db.service_update(self.context, s1['id'], {'updated_at': past}) + instance_id2 = self._create_instance(availability_zone='nova:host1') + self.assertRaises(driver.WillNotSchedule, + self.scheduler.driver.schedule_run_instance, + self.context, + instance_id2) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + + def test_will_schedule_on_disabled_host_if_specified(self): + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') + db.service_update(self.context, s1['id'], {'disabled': True}) + instance_id2 = self._create_instance(availability_zone='nova:host1') + host = self.scheduler.driver.schedule_run_instance(self.context, + instance_id2) + self.assertEqual('host1', host) + db.instance_destroy(self.context, instance_id2) + compute1.kill() + + def test_too_many_cores(self): + """Ensures we don't go over max cores""" + compute1 = service.Service('host1', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute1.start() + compute2 = service.Service('host2', + 'nova-compute', + 'compute', + FLAGS.compute_manager) + compute2.start() + instance_ids1 = [] + instance_ids2 = [] + for index in xrange(FLAGS.max_cores): + instance_id = self._create_instance() + compute1.run_instance(self.context, instance_id) + instance_ids1.append(instance_id) + instance_id = self._create_instance() + compute2.run_instance(self.context, instance_id) + instance_ids2.append(instance_id) + instance_id = self._create_instance() + self.assertRaises(driver.NoValidHost, + self.scheduler.driver.schedule_run_instance, + self.context, + instance_id) + for instance_id in instance_ids1: + compute1.terminate_instance(self.context, instance_id) + for instance_id in instance_ids2: + compute2.terminate_instance(self.context, instance_id) + compute1.kill() + compute2.kill() + + def test_least_busy_host_gets_volume(self): + """Ensures the host with less gigabytes gets the next one""" + volume1 = service.Service('host1', + 'nova-volume', + 'volume', + FLAGS.volume_manager) + volume1.start() + volume2 = service.Service('host2', + 'nova-volume', + 'volume', + FLAGS.volume_manager) + volume2.start() + volume_id1 = self._create_volume() + volume1.create_volume(self.context, volume_id1) + volume_id2 = self._create_volume() + host = self.scheduler.driver.schedule_create_volume(self.context, + volume_id2) + self.assertEqual(host, 'host2') + volume1.delete_volume(self.context, volume_id1) + db.volume_destroy(self.context, volume_id2) + dic = {'service_id': s_ref['id'], + 'vcpus': 16, 'memory_mb': 32, 'local_gb': 100, + 'vcpus_used': 16, 'memory_mb_used': 12, 'local_gb_used': 10, + 'hypervisor_type': 'qemu', 'hypervisor_version': 12003, + 'cpu_info': ''} + def test_doesnt_report_disabled_hosts_as_up(self): """Ensures driver doesn't find hosts before they are enabled""" compute1 = self.start_service('compute', host='host1') @@ -318,3 +627,313 @@ class SimpleDriverTestCase(test.TestCase): volume2.delete_volume(self.context, volume_id) volume1.kill() volume2.kill() + + def test_scheduler_live_migration_with_volume(self): + """scheduler_live_migration() works correctly as expected. + + Also, checks instance state is changed from 'running' -> 'migrating'. + + """ + + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + dic = {'instance_id': instance_id, 'size': 1} + v_ref = db.volume_create(self.context, dic) + + # cannot check 2nd argument b/c the addresses of instance object + # is different. + driver_i = self.scheduler.driver + nocare = mox.IgnoreArg() + self.mox.StubOutWithMock(driver_i, '_live_migration_src_check') + self.mox.StubOutWithMock(driver_i, '_live_migration_dest_check') + self.mox.StubOutWithMock(driver_i, '_live_migration_common_check') + driver_i._live_migration_src_check(nocare, nocare) + driver_i._live_migration_dest_check(nocare, nocare, i_ref['host']) + driver_i._live_migration_common_check(nocare, nocare, i_ref['host']) + self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + kwargs = {'instance_id': instance_id, 'dest': i_ref['host']} + rpc.cast(self.context, + db.queue_get_for(nocare, FLAGS.compute_topic, i_ref['host']), + {"method": 'live_migration', "args": kwargs}) + + self.mox.ReplayAll() + self.scheduler.live_migration(self.context, FLAGS.compute_topic, + instance_id=instance_id, + dest=i_ref['host']) + + i_ref = db.instance_get(self.context, instance_id) + self.assertTrue(i_ref['state_description'] == 'migrating') + db.instance_destroy(self.context, instance_id) + db.volume_destroy(self.context, v_ref['id']) + + def test_live_migration_src_check_instance_not_running(self): + """The instance given by instance_id is not running.""" + + instance_id = self._create_instance(state_description='migrating') + i_ref = db.instance_get(self.context, instance_id) + + try: + self.scheduler.driver._live_migration_src_check(self.context, + i_ref) + except exception.Invalid, e: + c = (e.message.find('is not running') > 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + + def test_live_migration_src_check_volume_node_not_alive(self): + """Raise exception when volume node is not alive.""" + + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + dic = {'instance_id': instance_id, 'size': 1} + v_ref = db.volume_create(self.context, {'instance_id': instance_id, + 'size': 1}) + t1 = datetime.datetime.utcnow() - datetime.timedelta(1) + dic = {'created_at': t1, 'updated_at': t1, 'binary': 'nova-volume', + 'topic': 'volume', 'report_count': 0} + s_ref = db.service_create(self.context, dic) + + try: + self.scheduler.driver.schedule_live_migration(self.context, + instance_id, + i_ref['host']) + except exception.Invalid, e: + c = (e.message.find('volume node is not alive') >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + db.volume_destroy(self.context, v_ref['id']) + + def test_live_migration_src_check_compute_node_not_alive(self): + """Confirms src-compute node is alive.""" + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + t = datetime.datetime.utcnow() - datetime.timedelta(10) + s_ref = self._create_compute_service(created_at=t, updated_at=t, + host=i_ref['host']) + + try: + self.scheduler.driver._live_migration_src_check(self.context, + i_ref) + except exception.Invalid, e: + c = (e.message.find('is not alive') >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_src_check_works_correctly(self): + """Confirms this method finishes with no error.""" + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + s_ref = self._create_compute_service(host=i_ref['host']) + + ret = self.scheduler.driver._live_migration_src_check(self.context, + i_ref) + + self.assertTrue(ret == None) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_dest_check_not_alive(self): + """Confirms exception raises in case dest host does not exist.""" + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + t = datetime.datetime.utcnow() - datetime.timedelta(10) + s_ref = self._create_compute_service(created_at=t, updated_at=t, + host=i_ref['host']) + + try: + self.scheduler.driver._live_migration_dest_check(self.context, + i_ref, + i_ref['host']) + except exception.Invalid, e: + c = (e.message.find('is not alive') >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_dest_check_service_same_host(self): + """Confirms exceptioin raises in case dest and src is same host.""" + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + s_ref = self._create_compute_service(host=i_ref['host']) + + try: + self.scheduler.driver._live_migration_dest_check(self.context, + i_ref, + i_ref['host']) + except exception.Invalid, e: + c = (e.message.find('choose other host') >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_dest_check_service_lack_memory(self): + """Confirms exception raises when dest doesn't have enough memory.""" + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + s_ref = self._create_compute_service(host='somewhere', + memory_mb_used=12) + + try: + self.scheduler.driver._live_migration_dest_check(self.context, + i_ref, + 'somewhere') + except exception.NotEmpty, e: + c = (e.message.find('Unable to migrate') >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_dest_check_service_works_correctly(self): + """Confirms method finishes with no error.""" + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + s_ref = self._create_compute_service(host='somewhere', + memory_mb_used=5) + + ret = self.scheduler.driver._live_migration_dest_check(self.context, + i_ref, + 'somewhere') + self.assertTrue(ret == None) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_common_check_service_orig_not_exists(self): + """Destination host does not exist.""" + + dest = 'dummydest' + # mocks for live_migration_common_check() + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + t1 = datetime.datetime.utcnow() - datetime.timedelta(10) + s_ref = self._create_compute_service(created_at=t1, updated_at=t1, + host=dest) + + # mocks for mounted_on_same_shared_storage() + fpath = '/test/20110127120000' + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + topic = FLAGS.compute_topic + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(self.context, topic, dest), + {"method": 'create_shared_storage_test_file'}).AndReturn(fpath) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), topic, i_ref['host']), + {"method": 'check_shared_storage_test_file', + "args": {'filename': fpath}}) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), topic, dest), + {"method": 'cleanup_shared_storage_test_file', + "args": {'filename': fpath}}) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(self.context, + i_ref, + dest) + except exception.Invalid, e: + c = (e.message.find('does not exist') >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + + def test_live_migration_common_check_service_different_hypervisor(self): + """Original host and dest host has different hypervisor type.""" + dest = 'dummydest' + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + + # compute service for destination + s_ref = self._create_compute_service(host=i_ref['host']) + # compute service for original host + s_ref2 = self._create_compute_service(host=dest, hypervisor_type='xen') + + # mocks + driver = self.scheduler.driver + self.mox.StubOutWithMock(driver, 'mounted_on_same_shared_storage') + driver.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(self.context, + i_ref, + dest) + except exception.Invalid, e: + c = (e.message.find(_('Different hypervisor type')) >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + db.service_destroy(self.context, s_ref2['id']) + + def test_live_migration_common_check_service_different_version(self): + """Original host and dest host has different hypervisor version.""" + dest = 'dummydest' + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + + # compute service for destination + s_ref = self._create_compute_service(host=i_ref['host']) + # compute service for original host + s_ref2 = self._create_compute_service(host=dest, + hypervisor_version=12002) + + # mocks + driver = self.scheduler.driver + self.mox.StubOutWithMock(driver, 'mounted_on_same_shared_storage') + driver.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(self.context, + i_ref, + dest) + except exception.Invalid, e: + c = (e.message.find(_('Older hypervisor version')) >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + db.service_destroy(self.context, s_ref2['id']) + + def test_live_migration_common_check_checking_cpuinfo_fail(self): + """Raise excetion when original host doen't have compatible cpu.""" + + dest = 'dummydest' + instance_id = self._create_instance() + i_ref = db.instance_get(self.context, instance_id) + + # compute service for destination + s_ref = self._create_compute_service(host=i_ref['host']) + # compute service for original host + s_ref2 = self._create_compute_service(host=dest) + + # mocks + driver = self.scheduler.driver + self.mox.StubOutWithMock(driver, 'mounted_on_same_shared_storage') + driver.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) + self.mox.StubOutWithMock(rpc, 'call', use_mock_anything=True) + rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), + {"method": 'compare_cpu', + "args": {'cpu_info': s_ref2['compute_node'][0]['cpu_info']}}).\ + AndRaise(rpc.RemoteError("doesn't have compatibility to", "", "")) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(self.context, + i_ref, + dest) + except rpc.RemoteError, e: + c = (e.message.find(_("doesn't have compatibility to")) >= 0) + + self.assertTrue(c) + db.instance_destroy(self.context, instance_id) + db.service_destroy(self.context, s_ref['id']) + db.service_destroy(self.context, s_ref2['id']) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index b006caadd..d48de2057 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -30,6 +30,7 @@ from nova import rpc from nova import test from nova import service from nova import manager +from nova.compute import manager as compute_manager FLAGS = flags.FLAGS flags.DEFINE_string("fake_manager", "nova.tests.test_service.FakeManager", @@ -260,3 +261,44 @@ class ServiceTestCase(test.TestCase): serv.report_state() self.assert_(not serv.model_disconnected) + + def test_compute_can_update_available_resource(self): + """Confirm compute updates their record of compute-service table.""" + host = 'foo' + binary = 'nova-compute' + topic = 'compute' + + # Any mocks are not working without UnsetStubs() here. + self.mox.UnsetStubs() + ctxt = context.get_admin_context() + service_ref = db.service_create(ctxt, {'host': host, + 'binary': binary, + 'topic': topic}) + serv = service.Service(host, + binary, + topic, + 'nova.compute.manager.ComputeManager') + + # This testcase want to test calling update_available_resource. + # No need to call periodic call, then below variable must be set 0. + serv.report_interval = 0 + serv.periodic_interval = 0 + + # Creating mocks + self.mox.StubOutWithMock(service.rpc.Connection, 'instance') + service.rpc.Connection.instance(new=mox.IgnoreArg()) + service.rpc.Connection.instance(new=mox.IgnoreArg()) + service.rpc.Connection.instance(new=mox.IgnoreArg()) + self.mox.StubOutWithMock(serv.manager.driver, + 'update_available_resource') + serv.manager.driver.update_available_resource(mox.IgnoreArg(), host) + + # Just doing start()-stop(), not confirm new db record is created, + # because update_available_resource() works only in + # libvirt environment. This testcase confirms + # update_available_resource() is called. Otherwise, mox complains. + self.mox.ReplayAll() + serv.start() + serv.stop() + + db.service_destroy(ctxt, service_ref['id']) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..b214f5ce7 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,33 +14,125 @@ # License for the specific language governing permissions and limitations # under the License. +import eventlet +import mox +import os +import re +import sys + from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db +from nova import exception from nova import flags from nova import test from nova import utils from nova.api.ec2 import cloud from nova.auth import manager +from nova.compute import manager as compute_manager +from nova.compute import power_state +from nova.db.sqlalchemy import models from nova.virt import libvirt_conn +libvirt = None FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +def _concurrency(wait, done, target): + wait.wait() + done.send() + + +class CacheConcurrencyTestCase(test.TestCase): + def setUp(self): + super(CacheConcurrencyTestCase, self).setUp() + + def fake_exists(fname): + basedir = os.path.join(FLAGS.instances_path, '_base') + if fname == basedir: + return True + return False + + def fake_execute(*args, **kwargs): + pass + + self.stubs.Set(os.path, 'exists', fake_exists) + self.stubs.Set(utils, 'execute', fake_execute) + + def test_same_fname_concurrency(self): + """Ensures that the same fname cache runs at a sequentially""" + conn = libvirt_conn.LibvirtConnection + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertFalse(done2.ready()) + self.assertTrue('fname' in conn._image_sems) + finally: + wait1.send() + done1.wait() + eventlet.sleep(0) + self.assertTrue(done2.ready()) + self.assertFalse('fname' in conn._image_sems) + + def test_different_fname_concurrency(self): + """Ensures that two different fname caches are concurrent""" + conn = libvirt_conn.LibvirtConnection + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname2', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname1', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertTrue(done2.ready()) + finally: + wait1.send() + eventlet.sleep(0) + + class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() libvirt_conn._late_load_cheetah() self.flags(fake_call=True) self.manager = manager.AuthManager() + + try: + pjs = self.manager.get_projects() + pjs = [p for p in pjs if p.name == 'fake'] + if 0 != len(pjs): + self.manager.delete_project(pjs[0]) + + users = self.manager.get_users() + users = [u for u in users if u.name == 'fake'] + if 0 != len(users): + self.manager.delete_user(users[0]) + except Exception, e: + pass + + users = self.manager.get_users() self.user = self.manager.create_user('fake', 'fake', 'fake', admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) + self.context = context.get_admin_context() FLAGS.instances_path = '' + self.call_libvirt_dependant_setup = False test_ip = '10.11.12.13' test_instance = {'memory_kb': '1024000', @@ -52,6 +144,58 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} + def lazy_load_library_exists(self): + """check if libvirt is available.""" + # try to connect libvirt. if fail, skip test. + try: + import libvirt + import libxml2 + except ImportError: + return False + global libvirt + libvirt = __import__('libvirt') + libvirt_conn.libvirt = __import__('libvirt') + libvirt_conn.libxml2 = __import__('libxml2') + return True + + def create_fake_libvirt_mock(self, **kwargs): + """Defining mocks for LibvirtConnection(libvirt is not used).""" + + # A fake libvirt.virConnect + class FakeLibvirtConnection(object): + pass + + # A fake libvirt_conn.IptablesFirewallDriver + class FakeIptablesFirewallDriver(object): + + def __init__(self, **kwargs): + pass + + def setattr(self, key, val): + self.__setattr__(key, val) + + # Creating mocks + fake = FakeLibvirtConnection() + fakeip = FakeIptablesFirewallDriver + # Customizing above fake if necessary + for key, val in kwargs.items(): + fake.__setattr__(key, val) + + # Inevitable mocks for libvirt_conn.LibvirtConnection + self.mox.StubOutWithMock(libvirt_conn.utils, 'import_class') + libvirt_conn.utils.import_class(mox.IgnoreArg()).AndReturn(fakeip) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') + libvirt_conn.LibvirtConnection._conn = fake + + def create_service(self, **kwargs): + service_ref = {'host': kwargs.get('host', 'dummy'), + 'binary': 'nova-compute', + 'topic': 'compute', + 'report_count': 0, + 'availability_zone': 'zone'} + + return db.service_create(context.get_admin_context(), service_ref) + def test_xml_and_uri_no_ramdisk_no_kernel(self): instance_data = dict(self.test_instance) self._check_xml_and_uri(instance_data, @@ -191,8 +335,8 @@ class LibvirtConnTestCase(test.TestCase): expected_result, '%s failed common check %d' % (xml, i)) - # This test is supposed to make sure we don't override a specifically - # set uri + # This test is supposed to make sure we don't + # override a specifically set uri # # Deliberately not just assigning this string to FLAGS.libvirt_uri and # checking against that later on. This way we make sure the @@ -206,6 +350,150 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) + def test_update_available_resource_works_correctly(self): + """Confirm compute_node table is updated successfully.""" + org_path = FLAGS.instances_path = '' + FLAGS.instances_path = '.' + + # Prepare mocks + def getVersion(): + return 12003 + + def getType(): + return 'qemu' + + def listDomainsID(): + return [] + + service_ref = self.create_service(host='dummy') + self.create_fake_libvirt_mock(getVersion=getVersion, + getType=getType, + listDomainsID=listDomainsID) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, + 'get_cpu_info') + libvirt_conn.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') + + # Start test + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn.update_available_resource(self.context, 'dummy') + service_ref = db.service_get(self.context, service_ref['id']) + compute_node = service_ref['compute_node'][0] + + if sys.platform.upper() == 'LINUX2': + self.assertTrue(compute_node['vcpus'] >= 0) + self.assertTrue(compute_node['memory_mb'] > 0) + self.assertTrue(compute_node['local_gb'] > 0) + self.assertTrue(compute_node['vcpus_used'] == 0) + self.assertTrue(compute_node['memory_mb_used'] > 0) + self.assertTrue(compute_node['local_gb_used'] > 0) + self.assertTrue(len(compute_node['hypervisor_type']) > 0) + self.assertTrue(compute_node['hypervisor_version'] > 0) + else: + self.assertTrue(compute_node['vcpus'] >= 0) + self.assertTrue(compute_node['memory_mb'] == 0) + self.assertTrue(compute_node['local_gb'] > 0) + self.assertTrue(compute_node['vcpus_used'] == 0) + self.assertTrue(compute_node['memory_mb_used'] == 0) + self.assertTrue(compute_node['local_gb_used'] > 0) + self.assertTrue(len(compute_node['hypervisor_type']) > 0) + self.assertTrue(compute_node['hypervisor_version'] > 0) + + db.service_destroy(self.context, service_ref['id']) + FLAGS.instances_path = org_path + + def test_update_resource_info_no_compute_record_found(self): + """Raise exception if no recorde found on services table.""" + org_path = FLAGS.instances_path = '' + FLAGS.instances_path = '.' + self.create_fake_libvirt_mock() + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(exception.Invalid, + conn.update_available_resource, + self.context, 'dummy') + + FLAGS.instances_path = org_path + + def test_ensure_filtering_rules_for_instance_timeout(self): + """ensure_filtering_fules_for_instance() finishes with timeout.""" + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + # Preparing mocks + def fake_none(self): + return + + def fake_raise(self): + raise libvirt.libvirtError('ERR') + + self.create_fake_libvirt_mock(nwfilterLookupByName=fake_raise) + instance_ref = db.instance_create(self.context, self.test_instance) + + # Start test + self.mox.ReplayAll() + try: + conn = libvirt_conn.LibvirtConnection(False) + conn.firewall_driver.setattr('setup_basic_filtering', fake_none) + conn.firewall_driver.setattr('prepare_instance_filter', fake_none) + conn.ensure_filtering_rules_for_instance(instance_ref) + except exception.Error, e: + c1 = (0 <= e.message.find('Timeout migrating for')) + self.assertTrue(c1) + + db.instance_destroy(self.context, instance_ref['id']) + + def test_live_migration_raises_exception(self): + """Confirms recover method is called when exceptions are raised.""" + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + # Preparing data + self.compute = utils.import_object(FLAGS.compute_manager) + instance_dict = {'host': 'fake', 'state': power_state.RUNNING, + 'state_description': 'running'} + instance_ref = db.instance_create(self.context, self.test_instance) + instance_ref = db.instance_update(self.context, instance_ref['id'], + instance_dict) + vol_dict = {'status': 'migrating', 'size': 1} + volume_ref = db.volume_create(self.context, vol_dict) + db.volume_attached(self.context, volume_ref['id'], instance_ref['id'], + '/dev/fake') + + # Preparing mocks + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI") + vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', + mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndRaise(libvirt.libvirtError('ERR')) + + def fake_lookup(instance_name): + if instance_name == instance_ref.name: + return vdmock + + self.create_fake_libvirt_mock(lookupByName=fake_lookup) + + # Start test + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, + conn._live_migration, + self.context, instance_ref, 'dest', '', + self.compute.recover_live_migration) + + instance_ref = db.instance_get(self.context, instance_ref['id']) + self.assertTrue(instance_ref['state_description'] == 'running') + self.assertTrue(instance_ref['state'] == power_state.RUNNING) + volume_ref = db.volume_get(self.context, volume_ref['id']) + self.assertTrue(volume_ref['status'] == 'in-use') + + db.volume_destroy(self.context, volume_ref['id']) + db.instance_destroy(self.context, instance_ref['id']) + def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) @@ -234,16 +522,22 @@ class IptablesFirewallTestCase(test.TestCase): self.manager.delete_user(self.user) super(IptablesFirewallTestCase, self).tearDown() - in_rules = [ + in_nat_rules = [ + '# Generated by iptables-save v1.4.10 on Sat Feb 19 00:03:19 2011', + '*nat', + ':PREROUTING ACCEPT [1170:189210]', + ':INPUT ACCEPT [844:71028]', + ':OUTPUT ACCEPT [5149:405186]', + ':POSTROUTING ACCEPT [5063:386098]', + ] + + in_filter_rules = [ '# Generated by iptables-save v1.4.4 on Mon Dec 6 11:54:13 2010', '*filter', ':INPUT ACCEPT [969615:281627771]', ':FORWARD ACCEPT [0:0]', ':OUTPUT ACCEPT [915599:63811649]', ':nova-block-ipv4 - [0:0]', - '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', '-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED' ',ESTABLISHED -j ACCEPT ', @@ -255,7 +549,7 @@ class IptablesFirewallTestCase(test.TestCase): '# Completed on Mon Dec 6 11:54:13 2010', ] - in6_rules = [ + in6_filter_rules = [ '# Generated by ip6tables-save v1.4.4 on Tue Jan 18 23:47:56 2011', '*filter', ':INPUT ACCEPT [349155:75810423]', @@ -315,23 +609,34 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): - if cmd == 'sudo ip6tables-save -t filter': - return '\n'.join(self.in6_rules), None - if cmd == 'sudo iptables-save -t filter': - return '\n'.join(self.in_rules), None - if cmd == 'sudo iptables-restore': - self.out_rules = process_input.split('\n') + def fake_iptables_execute(*cmd, **kwargs): + process_input = kwargs.get('process_input', None) + if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): + return '\n'.join(self.in6_filter_rules), None + if cmd == ('sudo', 'iptables-save', '-t', 'filter'): + return '\n'.join(self.in_filter_rules), None + if cmd == ('sudo', 'iptables-save', '-t', 'nat'): + return '\n'.join(self.in_nat_rules), None + if cmd == ('sudo', 'iptables-restore'): + lines = process_input.split('\n') + if '*filter' in lines: + self.out_rules = lines return '', '' - if cmd == 'sudo ip6tables-restore': - self.out6_rules = process_input.split('\n') + if cmd == ('sudo', 'ip6tables-restore'): + lines = process_input.split('\n') + if '*filter' in lines: + self.out6_rules = lines return '', '' - self.fw.execute = fake_iptables_execute + print cmd, kwargs + + from nova.network import linux_net + linux_net.iptables_manager.execute = fake_iptables_execute self.fw.prepare_instance_filter(instance_ref) self.fw.apply_instance_filter(instance_ref) - in_rules = filter(lambda l: not l.startswith('#'), self.in_rules) + in_rules = filter(lambda l: not l.startswith('#'), + self.in_filter_rules) for rule in in_rules: if not 'nova' in rule: self.assertTrue(rule in self.out_rules, @@ -354,17 +659,18 @@ class IptablesFirewallTestCase(test.TestCase): self.assertTrue(security_group_chain, "The security group chain wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -j ACCEPT' % \ - security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP acceptance rule wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type ' - '8 -j ACCEPT' % security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -m icmp ' + '--icmp-type 8 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP Echo Request acceptance rule wasn't added") - self.assertTrue('-A %s -p tcp -s 192.168.10.0/24 -m multiport ' - '--dports 80:81 -j ACCEPT' % security_group_chain \ - in self.out_rules, + regex = re.compile('-A .* -p tcp -s 192.168.10.0/24 -m multiport ' + '--dports 80:81 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "TCP port 80/81 acceptance rule wasn't added") db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index b40ca004b..1b1d72092 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -20,6 +20,8 @@ Tests for Volume Code. """ +import cStringIO + from nova import context from nova import exception from nova import db @@ -99,7 +101,7 @@ class VolumeTestCase(test.TestCase): def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance.""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = 'fake' @@ -173,3 +175,196 @@ class VolumeTestCase(test.TestCase): # each of them having a different FLAG for storage_node # This will allow us to test cross-node interactions pass + + +class DriverTestCase(test.TestCase): + """Base Test class for Drivers.""" + driver_name = "nova.volume.driver.FakeAOEDriver" + + def setUp(self): + super(DriverTestCase, self).setUp() + self.flags(volume_driver=self.driver_name, + logging_default_format_string="%(message)s") + self.volume = utils.import_object(FLAGS.volume_manager) + self.context = context.get_admin_context() + self.output = "" + + def _fake_execute(_command, *_args, **_kwargs): + """Fake _execute.""" + return self.output, None + self.volume.driver._execute = _fake_execute + self.volume.driver._sync_execute = _fake_execute + + log = logging.getLogger() + self.stream = cStringIO.StringIO() + log.addHandler(logging.StreamHandler(self.stream)) + + inst = {} + self.instance_id = db.instance_create(self.context, inst)['id'] + + def tearDown(self): + super(DriverTestCase, self).tearDown() + + def _attach_volume(self): + """Attach volumes to an instance. This function also sets + a fake log message.""" + return [] + + def _detach_volume(self, volume_id_list): + """Detach volumes from an instance.""" + for volume_id in volume_id_list: + db.volume_detached(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) + + +class AOETestCase(DriverTestCase): + """Test Case for AOEDriver""" + driver_name = "nova.volume.driver.AOEDriver" + + def setUp(self): + super(AOETestCase, self).setUp() + + def tearDown(self): + super(AOETestCase, self).tearDown() + + def _attach_volume(self): + """Attach volumes to an instance. This function also sets + a fake log message.""" + volume_id_list = [] + for index in xrange(3): + vol = {} + vol['size'] = 0 + volume_id = db.volume_create(self.context, + vol)['id'] + self.volume.create_volume(self.context, volume_id) + + # each volume has a different mountpoint + mountpoint = "/dev/sd" + chr((ord('b') + index)) + db.volume_attached(self.context, volume_id, self.instance_id, + mountpoint) + + (shelf_id, blade_id) = db.volume_get_shelf_and_blade(self.context, + volume_id) + self.output += "%s %s eth0 /dev/nova-volumes/vol-foo auto run\n" \ + % (shelf_id, blade_id) + + volume_id_list.append(volume_id) + + return volume_id_list + + def test_check_for_export_with_no_volume(self): + """No log message when no volume is attached to an instance.""" + self.stream.truncate(0) + self.volume.check_for_export(self.context, self.instance_id) + self.assertEqual(self.stream.getvalue(), '') + + def test_check_for_export_with_all_vblade_processes(self): + """No log message when all the vblade processes are running.""" + volume_id_list = self._attach_volume() + + self.stream.truncate(0) + self.volume.check_for_export(self.context, self.instance_id) + self.assertEqual(self.stream.getvalue(), '') + + self._detach_volume(volume_id_list) + + def test_check_for_export_with_vblade_process_missing(self): + """Output a warning message when some vblade processes aren't + running.""" + volume_id_list = self._attach_volume() + + # the first vblade process isn't running + self.output = self.output.replace("run", "down", 1) + (shelf_id, blade_id) = db.volume_get_shelf_and_blade(self.context, + volume_id_list[0]) + + msg_is_match = False + self.stream.truncate(0) + try: + self.volume.check_for_export(self.context, self.instance_id) + except exception.ProcessExecutionError, e: + volume_id = volume_id_list[0] + msg = _("Cannot confirm exported volume id:%(volume_id)s. " + "vblade process for e%(shelf_id)s.%(blade_id)s " + "isn't running.") % locals() + + msg_is_match = (0 <= e.message.find(msg)) + + self.assertTrue(msg_is_match) + self._detach_volume(volume_id_list) + + +class ISCSITestCase(DriverTestCase): + """Test Case for ISCSIDriver""" + driver_name = "nova.volume.driver.ISCSIDriver" + + def setUp(self): + super(ISCSITestCase, self).setUp() + + def tearDown(self): + super(ISCSITestCase, self).tearDown() + + def _attach_volume(self): + """Attach volumes to an instance. This function also sets + a fake log message.""" + volume_id_list = [] + for index in xrange(3): + vol = {} + vol['size'] = 0 + vol_ref = db.volume_create(self.context, vol) + self.volume.create_volume(self.context, vol_ref['id']) + vol_ref = db.volume_get(self.context, vol_ref['id']) + + # each volume has a different mountpoint + mountpoint = "/dev/sd" + chr((ord('b') + index)) + db.volume_attached(self.context, vol_ref['id'], self.instance_id, + mountpoint) + volume_id_list.append(vol_ref['id']) + + return volume_id_list + + def test_check_for_export_with_no_volume(self): + """No log message when no volume is attached to an instance.""" + self.stream.truncate(0) + self.volume.check_for_export(self.context, self.instance_id) + self.assertEqual(self.stream.getvalue(), '') + + def test_check_for_export_with_all_volume_exported(self): + """No log message when all the vblade processes are running.""" + volume_id_list = self._attach_volume() + + self.mox.StubOutWithMock(self.volume.driver, '_execute') + for i in volume_id_list: + tid = db.volume_get_iscsi_target_num(self.context, i) + self.volume.driver._execute("sudo ietadm --op show --tid=%(tid)d" + % locals()) + + self.stream.truncate(0) + self.mox.ReplayAll() + self.volume.check_for_export(self.context, self.instance_id) + self.assertEqual(self.stream.getvalue(), '') + self.mox.UnsetStubs() + + self._detach_volume(volume_id_list) + + def test_check_for_export_with_some_volume_missing(self): + """Output a warning message when some volumes are not recognied + by ietd.""" + volume_id_list = self._attach_volume() + + # the first vblade process isn't running + tid = db.volume_get_iscsi_target_num(self.context, volume_id_list[0]) + self.mox.StubOutWithMock(self.volume.driver, '_execute') + self.volume.driver._execute("sudo ietadm --op show --tid=%(tid)d" + % locals()).AndRaise(exception.ProcessExecutionError()) + + self.mox.ReplayAll() + self.assertRaises(exception.ProcessExecutionError, + self.volume.check_for_export, + self.context, + self.instance_id) + msg = _("Cannot confirm exported volume id:%s.") % volume_id_list[0] + self.assertTrue(0 <= self.stream.getvalue().find(msg)) + self.mox.UnsetStubs() + + self._detach_volume(volume_id_list) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index c26dc8639..8b0affd5c 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -18,6 +18,7 @@ Test suite for XenAPI """ +import functools import stubout from nova import db @@ -41,6 +42,21 @@ from nova.tests.glance import stubs as glance_stubs FLAGS = flags.FLAGS +def stub_vm_utils_with_vdi_attached_here(function, should_return=True): + """ + vm_utils.with_vdi_attached_here needs to be stubbed out because it + calls down to the filesystem to attach a vdi. This provides a + decorator to handle that. + """ + @functools.wraps(function) + def decorated_function(self, *args, **kwargs): + orig_with_vdi_attached_here = vm_utils.with_vdi_attached_here + vm_utils.with_vdi_attached_here = lambda *x: should_return + function(self, *args, **kwargs) + vm_utils.with_vdi_attached_here = orig_with_vdi_attached_here + return decorated_function + + class XenAPIVolumeTestCase(test.TestCase): """ Unit tests for Volume operations @@ -62,7 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', - } + 'os_type': 'linux'} def _create_volume(self, size='0'): """Create a volume object.""" @@ -219,7 +235,7 @@ class XenAPIVMTestCase(test.TestCase): check() - def check_vm_record(self, conn): + def create_vm_record(self, conn, os_type): instances = conn.list_instances() self.assertEquals(instances, [1]) @@ -231,28 +247,63 @@ class XenAPIVMTestCase(test.TestCase): in xenapi_fake.get_all_records('VM').iteritems() if not rec['is_control_domain']] vm = vms[0] + self.vm_info = vm_info + self.vm = vm + def check_vm_record(self, conn): # Check that m1.large above turned into the right thing. instance_type = db.instance_type_get_by_name(conn, 'm1.large') mem_kib = long(instance_type['memory_mb']) << 10 mem_bytes = str(mem_kib << 10) vcpus = instance_type['vcpus'] - self.assertEquals(vm_info['max_mem'], mem_kib) - self.assertEquals(vm_info['mem'], mem_kib) - self.assertEquals(vm['memory_static_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_min'], mem_bytes) - self.assertEquals(vm['VCPUs_max'], str(vcpus)) - self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + self.assertEquals(self.vm_info['max_mem'], mem_kib) + self.assertEquals(self.vm_info['mem'], mem_kib) + self.assertEquals(self.vm['memory_static_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(self.vm['VCPUs_max'], str(vcpus)) + self.assertEquals(self.vm['VCPUs_at_startup'], str(vcpus)) # Check that the VM is running according to Nova - self.assertEquals(vm_info['state'], power_state.RUNNING) + self.assertEquals(self.vm_info['state'], power_state.RUNNING) # Check that the VM is running according to XenAPI. - self.assertEquals(vm['power_state'], 'Running') + self.assertEquals(self.vm['power_state'], 'Running') + + def check_vm_params_for_windows(self): + self.assertEquals(self.vm['platform']['nx'], 'true') + self.assertEquals(self.vm['HVM_boot_params'], {'order': 'dc'}) + self.assertEquals(self.vm['HVM_boot_policy'], 'BIOS order') + + # check that these are not set + self.assertEquals(self.vm['PV_args'], '') + self.assertEquals(self.vm['PV_bootloader'], '') + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + + def check_vm_params_for_linux(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies') + self.assertEquals(self.vm['PV_bootloader'], 'pygrub') + + # check that these are not set + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') + + def check_vm_params_for_linux_with_external_kernel(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'root=/dev/xvda1') + self.assertNotEquals(self.vm['PV_kernel'], '') + self.assertNotEquals(self.vm['PV_ramdisk'], '') + + # check that these are not set + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') def _test_spawn(self, image_id, kernel_id, ramdisk_id, - instance_type="m1.large"): + instance_type="m1.large", os_type="linux"): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) values = {'name': 1, 'id': 1, @@ -263,10 +314,12 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type': instance_type, 'mac_address': 'aa:bb:cc:dd:ee:ff', - } + 'os_type': os_type} + conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) conn.spawn(instance) + self.create_vm_record(conn, os_type) self.check_vm_record(conn) def test_spawn_not_enough_memory(self): @@ -283,24 +336,37 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_image_service = 'objectstore' self._test_spawn(1, 2, 3) + @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) + self.check_vm_params_for_linux() - def test_spawn_vhd_glance(self): + def test_spawn_vhd_glance_linux(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="linux") + self.check_vm_params_for_linux() + + def test_spawn_vhd_glance_windows(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="windows") + self.check_vm_params_for_windows() def test_spawn_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK) + self.check_vm_params_for_linux_with_external_kernel() def tearDown(self): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + self.vm_info = None + self.vm = None self.stubs.UnsetAll() def _create_instance(self): @@ -314,7 +380,8 @@ class XenAPIVMTestCase(test.TestCase): 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff'} + 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux'} instance = db.instance_create(values) self.conn.spawn(instance) return instance @@ -372,7 +439,8 @@ class XenAPIMigrateInstance(test.TestCase): 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', - } + 'os_type': 'linux'} + stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) @@ -410,6 +478,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance = FakeInstance() self.fake_instance.id = 42 + self.fake_instance.os_type = 'linux' def assert_disk_type(self, disk_type): dt = vm_utils.VMHelper.determine_disk_image_type( diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..24b8da9ea 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -23,10 +23,14 @@ System-level utilities and helper functions. import base64 import datetime +import functools import inspect import json +import lockfile +import netaddr import os import random +import re import socket import string import struct @@ -34,20 +38,20 @@ import sys import time import types from xml.sax import saxutils -import re -import netaddr from eventlet import event from eventlet import greenthread from eventlet.green import subprocess - +None from nova import exception from nova.exception import ProcessExecutionError +from nova import flags from nova import log as logging LOG = logging.getLogger("nova.utils") TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +FLAGS = flags.FLAGS def import_class(import_str): @@ -125,40 +129,59 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl --fail %s -o %s" % (url, target)) - - -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) - env = os.environ.copy() - if addl_env: - env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) - result = None - if process_input != None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=obj.returncode, - stdout=stdout, - stderr=stderr, - cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one - greenthread.sleep(0) - return result + execute("curl", "--fail", url, "-o", target) + + +def execute(*cmd, **kwargs): + process_input = kwargs.get('process_input', None) + addl_env = kwargs.get('addl_env', None) + check_exit_code = kwargs.get('check_exit_code', 0) + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + attempts = kwargs.get('attempts', 1) + cmd = map(str, cmd) + + while attempts > 0: + attempts -= 1 + try: + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) + env = os.environ.copy() + if addl_env: + env.update(addl_env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) + result = None + if process_input != None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() + if obj.returncode: + LOG.debug(_("Result was %s") % obj.returncode) + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.debug(_("%r failed. Retrying."), cmd) + greenthread.sleep(random.randint(20, 200) / 100.0) def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (SSH): %s"), cmd) + LOG.debug(_("Running cmd (SSH): %s"), ' '.join(cmd)) if addl_env: raise exception.Error("Environment not supported over SSH") @@ -187,7 +210,7 @@ def ssh_execute(ssh, cmd, process_input=None, raise exception.ProcessExecutionError(exit_code=exit_status, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) return (stdout, stderr) @@ -220,9 +243,9 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code=True): - LOG.debug(_("Running %s"), (cmd)) - rv, err = execute(cmd, check_exit_code=check_exit_code) +def runthis(prompt, *cmd, **kwargs): + LOG.debug(_("Running %s"), (" ".join(cmd))) + rv, err = execute(*cmd, **kwargs) def generate_uid(topic, size=8): @@ -239,13 +262,25 @@ def generate_mac(): return ':'.join(map(lambda x: "%02x" % x, mac)) -def generate_password(length=20): - """Generate a random sequence of letters and digits - to be used as a password. Note that this is not intended - to represent the ultimate in security. +# Default symbols to use for passwords. Avoids visually confusing characters. +# ~6 bits per symbol +DEFAULT_PASSWORD_SYMBOLS = ("23456789" # Removed: 0,1 + "ABCDEFGHJKLMNPQRSTUVWXYZ" # Removed: I, O + "abcdefghijkmnopqrstuvwxyz") # Removed: l + + +# ~5 bits per symbol +EASIER_PASSWORD_SYMBOLS = ("23456789" # Removed: 0, 1 + "ABCDEFGHJKLMNPQRSTUVWXYZ") # Removed: I, O + + +def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS): + """Generate a random password from the supplied symbols. + + Believed to be reasonably secure (with a reasonable password length!) """ - chrs = string.letters + string.digits - return "".join([random.choice(chrs) for i in xrange(length)]) + r = random.SystemRandom() + return "".join([r.choice(symbols) for _i in xrange(length)]) def last_octet(address): @@ -254,7 +289,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip -f inet6 -o addr show %s" % interface) + if_str = execute("ip", "-f", "inet6", "-o", "addr", "show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] @@ -491,16 +526,19 @@ def loads(s): return json.loads(s) -def ensure_b64_encoding(val): - """Safety method to ensure that values expected to be base64-encoded - actually are. If they are, the value is returned unchanged. Otherwise, - the encoded value is returned. - """ - try: - dummy = base64.decode(val) - return val - except TypeError: - return base64.b64encode(val) +def synchronized(name): + def wrap(f): + @functools.wraps(f) + def inner(*args, **kwargs): + LOG.debug(_("Attempting to grab %(lock)s for method " + "%(method)s..." % {"lock": name, + "method": f.__name__})) + lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name)) + with lock: + return f(*args, **kwargs) + return inner + return wrap def get_from_path(items, path): diff --git a/nova/virt/cpuinfo.xml.template b/nova/virt/cpuinfo.xml.template new file mode 100644 index 000000000..48842b29d --- /dev/null +++ b/nova/virt/cpuinfo.xml.template @@ -0,0 +1,9 @@ +<cpu> + <arch>$arch</arch> + <model>$model</model> + <vendor>$vendor</vendor> + <topology sockets="$topology.sockets" cores="$topology.cores" threads="$topology.threads"/> +#for $var in $features + <features name="$var" /> +#end for +</cpu> diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 2bded07a4..9abe44cc3 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -49,10 +49,10 @@ def extend(image, size): file_size = os.path.getsize(image) if file_size >= size: return - utils.execute('truncate -s %s %s' % (size, image)) + utils.execute('truncate', '-s', size, image) # NOTE(vish): attempts to resize filesystem - utils.execute('e2fsck -fp %s' % image, check_exit_code=False) - utils.execute('resize2fs %s' % image, check_exit_code=False) + utils.execute('e2fsck', '-fp', image, check_exit_code=False) + utils.execute('resize2fs', image, check_exit_code=False) def inject_data(image, key=None, net=None, partition=None, nbd=False): @@ -68,7 +68,7 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): try: if not partition is None: # create partition - out, err = utils.execute('sudo kpartx -a %s' % device) + out, err = utils.execute('sudo', 'kpartx', '-a', device) if err: raise exception.Error(_('Failed to load partition: %s') % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], @@ -84,13 +84,14 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): mapped_device) # Configure ext2fs so that it doesn't auto-check every N boots - out, err = utils.execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + out, err = utils.execute('sudo', 'tune2fs', + '-c', 0, '-i', 0, mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir out, err = utils.execute( - 'sudo mount %s %s' % (mapped_device, tmpdir)) + 'sudo', 'mount', mapped_device, tmpdir) if err: raise exception.Error(_('Failed to mount filesystem: %s') % err) @@ -103,13 +104,13 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): _inject_net_into_fs(net, tmpdir) finally: # unmount device - utils.execute('sudo umount %s' % mapped_device) + utils.execute('sudo', 'umount', mapped_device) finally: # remove temporary directory - utils.execute('rmdir %s' % tmpdir) + utils.execute('rmdir', tmpdir) if not partition is None: # remove partitions - utils.execute('sudo kpartx -d %s' % device) + utils.execute('sudo', 'kpartx', '-d', device) finally: _unlink_device(device, nbd) @@ -118,7 +119,7 @@ def _link_device(image, nbd): """Link image to device using loopback or nbd""" if nbd: device = _allocate_device() - utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) + utils.execute('sudo', 'qemu-nbd', '-c', device, image) # NOTE(vish): this forks into another process, so give it a chance # to set up before continuuing for i in xrange(FLAGS.timeout_nbd): @@ -127,7 +128,7 @@ def _link_device(image, nbd): time.sleep(1) raise exception.Error(_('nbd device %s did not show up') % device) else: - out, err = utils.execute('sudo losetup --find --show %s' % image) + out, err = utils.execute('sudo', 'losetup', '--find', '--show', image) if err: raise exception.Error(_('Could not attach image to loopback: %s') % err) @@ -137,10 +138,10 @@ def _link_device(image, nbd): def _unlink_device(device, nbd): """Unlink image from device using loopback or nbd""" if nbd: - utils.execute('sudo qemu-nbd -d %s' % device) + utils.execute('sudo', 'qemu-nbd', '-d', device) _free_device(device) else: - utils.execute('sudo losetup --detach %s' % device) + utils.execute('sudo', 'losetup', '--detach', device) _DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.max_nbd_devices)] @@ -170,11 +171,12 @@ def _inject_key_into_fs(key, fs): fs is the path to the base of the filesystem into which to inject the key. """ sshdir = os.path.join(fs, 'root', '.ssh') - utils.execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - utils.execute('sudo chown root %s' % sshdir) - utils.execute('sudo chmod 700 %s' % sshdir) + utils.execute('sudo', 'mkdir', '-p', sshdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root', sshdir) + utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - utils.execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + utils.execute('sudo', 'tee', '-a', keyfile, + process_input='\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): @@ -183,8 +185,8 @@ def _inject_net_into_fs(net, fs): net is the contents of /etc/network/interfaces. """ netdir = os.path.join(os.path.join(fs, 'etc'), 'network') - utils.execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter - utils.execute('sudo chown root:root %s' % netdir) - utils.execute('sudo chmod 755 %s' % netdir) + utils.execute('sudo', 'mkdir', '-p', netdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root:root', netdir) + utils.execute('sudo', 'chmod', 755, netdir) netfile = os.path.join(netdir, 'interfaces') - utils.execute('sudo tee %s' % netfile, net) + utils.execute('sudo', 'tee', netfile, process_input=net) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index c744acf91..3a06284a1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -407,6 +407,27 @@ class FakeConnection(object): """ return True + def update_available_resource(self, ctxt, host): + """This method is supported only by libvirt.""" + return + + def compare_cpu(self, xml): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + + def ensure_filtering_rules_for_instance(self, instance_ref): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + + def live_migration(self, context, instance_ref, dest, + post_method, recover_method): + """This method is supported only by libvirt.""" + return + + def unfilter_instance(self, instance_ref): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + class FakeInstance(object): diff --git a/nova/virt/images.py b/nova/virt/images.py index 7a6fef330..2e3f2ee4d 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -28,29 +28,32 @@ import time import urllib2 import urlparse +from nova import context from nova import flags from nova import log as logging from nova import utils from nova.auth import manager from nova.auth import signer -from nova.objectstore import image FLAGS = flags.FLAGS -flags.DEFINE_bool('use_s3', True, - 'whether to get images from s3 or use local copy') - LOG = logging.getLogger('nova.virt.images') -def fetch(image, path, user, project): - if FLAGS.use_s3: - f = _fetch_s3_image - else: - f = _fetch_local_image - return f(image, path, user, project) +def fetch(image_id, path, _user, _project): + # TODO(vish): Improve context handling and add owner and auth data + # when it is added to glance. Right now there is no + # auth checking in glance, so we assume that access was + # checked before we got here. + image_service = utils.import_object(FLAGS.image_service) + with open(path, "wb") as image_file: + elevated = context.get_admin_context() + metadata = image_service.get(elevated, image_id, image_file) + return metadata +# NOTE(vish): The methods below should be unnecessary, but I'm leaving +# them in case the glance client does not work on windows. def _fetch_image_no_curl(url, path, headers): request = urllib2.Request(url) for (k, v) in headers.iteritems(): @@ -94,8 +97,7 @@ def _fetch_s3_image(image, path, user, project): cmd += ['-H', '\'%s: %s\'' % (k, v)] cmd += ['-o', path] - cmd_out = ' '.join(cmd) - return utils.execute(cmd_out) + return utils.execute(*cmd) def _fetch_local_image(image, path, user, project): @@ -103,13 +105,15 @@ def _fetch_local_image(image, path, user, project): if sys.platform.startswith('win'): return shutil.copy(source, path) else: - return utils.execute('cp %s %s' % (source, path)) + return utils.execute('cp', source, path) def _image_path(path): return os.path.join(FLAGS.images_path, path) +# TODO(vish): xenapi should use the glance client code directly instead +# of retrieving the image using this method. def image_url(image): if FLAGS.image_service == "nova.image.glance.GlanceImageService": return "http://%s:%s/images/%s" % (FLAGS.glance_host, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9f7315c17..2559c2b81 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -36,17 +36,19 @@ Supports KVM, QEMU, UML, and XEN. """ +import multiprocessing import os import shutil +import sys import random import subprocess +import time import uuid from xml.dom import minidom -from eventlet import greenthread -from eventlet import event from eventlet import tpool +from eventlet import semaphore import IPy @@ -57,7 +59,6 @@ from nova import flags from nova import log as logging #from nova import test from nova import utils -#from nova.api import context from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state @@ -71,6 +72,7 @@ Template = None LOG = logging.getLogger('nova.virt.libvirt_conn') FLAGS = flags.FLAGS +flags.DECLARE('live_migration_retry_count', 'nova.compute.manager') # TODO(vish): These flags should probably go into a shared location flags.DEFINE_string('rescue_image_id', 'ami-rescue', 'Rescue ami image') flags.DEFINE_string('rescue_kernel_id', 'aki-rescue', 'Rescue aki image') @@ -101,6 +103,17 @@ flags.DEFINE_string('ajaxterm_portrange', flags.DEFINE_string('firewall_driver', 'nova.virt.libvirt_conn.IptablesFirewallDriver', 'Firewall driver (defaults to iptables)') +flags.DEFINE_string('cpuinfo_xml_template', + utils.abspath('virt/cpuinfo.xml.template'), + 'CpuInfo XML Template (Used only live migration now)') +flags.DEFINE_string('live_migration_uri', + "qemu+tcp://%s/system", + 'Define protocol used by live_migration feature') +flags.DEFINE_string('live_migration_flag', + "VIR_MIGRATE_UNDEFINE_SOURCE, VIR_MIGRATE_PEER2PEER", + 'Define live migration behavior.') +flags.DEFINE_integer('live_migration_bandwidth', 0, + 'Define live migration behavior') def get_connection(read_only): @@ -147,6 +160,7 @@ class LibvirtConnection(object): self.libvirt_uri = self.get_uri() self.libvirt_xml = open(FLAGS.libvirt_xml_template).read() + self.cpuinfo_xml = open(FLAGS.cpuinfo_xml_template).read() self._wrapped_conn = None self.read_only = read_only @@ -348,19 +362,19 @@ class LibvirtConnection(object): @exception.wrap_exception def pause(self, instance, callback): - raise exception.APIError("pause not supported for libvirt.") + raise exception.ApiError("pause not supported for libvirt.") @exception.wrap_exception def unpause(self, instance, callback): - raise exception.APIError("unpause not supported for libvirt.") + raise exception.ApiError("unpause not supported for libvirt.") @exception.wrap_exception def suspend(self, instance, callback): - raise exception.APIError("suspend not supported for libvirt") + raise exception.ApiError("suspend not supported for libvirt") @exception.wrap_exception def resume(self, instance, callback): - raise exception.APIError("resume not supported for libvirt") + raise exception.ApiError("resume not supported for libvirt") @exception.wrap_exception def rescue(self, instance, callback=None): @@ -439,8 +453,10 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): LOG.info(_("cool, it's a device")) - out, err = utils.execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) + out, err = utils.execute('sudo', 'dd', + "if=%s" % virsh_output, + 'iflag=nonblock', + check_exit_code=False) return out else: return '' @@ -462,11 +478,11 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) + utils.execute('sudo', 'chown', os.getuid(), console_log) if FLAGS.libvirt_type == 'xen': # Xen is special - virsh_output = utils.execute("virsh ttyconsole %s" % + virsh_output = utils.execute('virsh', 'ttyconsole', instance['name']) data = self._flush_xen_console(virsh_output) fpath = self._append_to_file(data, console_log) @@ -483,9 +499,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - cmd = 'netcat 0.0.0.0 %s -w 1 </dev/null || echo free' % (port) - stdout, stderr = utils.execute(cmd) - if stdout.strip() == 'free': + cmd = 'netcat', '0.0.0.0', port, '-w', '1' + try: + stdout, stderr = utils.execute(*cmd, process_input='') + except exception.ProcessExecutionError: return port raise Exception(_('Unable to find an open port')) @@ -512,7 +529,10 @@ class LibvirtConnection(object): subprocess.Popen(cmd, shell=True) return {'token': token, 'host': host, 'port': port} - def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): + _image_sems = {} + + @staticmethod + def _cache_image(fn, target, fname, cow=False, *args, **kwargs): """Wrapper for a method that creates an image that caches the image. This wrapper will save the image into a common store and create a @@ -531,14 +551,21 @@ class LibvirtConnection(object): if not os.path.exists(base_dir): os.mkdir(base_dir) base = os.path.join(base_dir, fname) - if not os.path.exists(base): - fn(target=base, *args, **kwargs) + + if fname not in LibvirtConnection._image_sems: + LibvirtConnection._image_sems[fname] = semaphore.Semaphore() + with LibvirtConnection._image_sems[fname]: + if not os.path.exists(base): + fn(target=base, *args, **kwargs) + if not LibvirtConnection._image_sems[fname].locked(): + del LibvirtConnection._image_sems[fname] + if cow: - utils.execute('qemu-img create -f qcow2 -o ' - 'cluster_size=2M,backing_file=%s %s' - % (base, target)) + utils.execute('qemu-img', 'create', '-f', 'qcow2', '-o', + 'cluster_size=2M,backing_file=%s' % base, + target) else: - utils.execute('cp %s %s' % (base, target)) + utils.execute('cp', base, target) def _fetch_image(self, target, image_id, user, project, size=None): """Grab image and optionally attempt to resize it""" @@ -548,7 +575,7 @@ class LibvirtConnection(object): def _create_local(self, target, local_gb): """Create a blank image of specified size""" - utils.execute('truncate %s -s %dG' % (target, local_gb)) + utils.execute('truncate', target, '-s', "%dG" % local_gb) # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): @@ -559,7 +586,7 @@ class LibvirtConnection(object): fname + suffix) # ensure directories exist and are writable - utils.execute('mkdir -p %s' % basepath(suffix='')) + utils.execute('mkdir', '-p', basepath(suffix='')) LOG.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') @@ -579,21 +606,23 @@ class LibvirtConnection(object): 'ramdisk_id': inst['ramdisk_id']} if disk_images['kernel_id']: + fname = '%08x' % int(disk_images['kernel_id']) self._cache_image(fn=self._fetch_image, target=basepath('kernel'), - fname=disk_images['kernel_id'], + fname=fname, image_id=disk_images['kernel_id'], user=user, project=project) if disk_images['ramdisk_id']: + fname = '%08x' % int(disk_images['ramdisk_id']) self._cache_image(fn=self._fetch_image, target=basepath('ramdisk'), - fname=disk_images['ramdisk_id'], + fname=fname, image_id=disk_images['ramdisk_id'], user=user, project=project) - root_fname = disk_images['image_id'] + root_fname = '%08x' % int(disk_images['image_id']) size = FLAGS.minimum_root_size if inst['instance_type'] == 'm1.tiny' or suffix == '.rescue': size = None @@ -659,7 +688,7 @@ class LibvirtConnection(object): ' data into image %(img_id)s (%(e)s)') % locals()) if FLAGS.libvirt_type == 'uml': - utils.execute('sudo chown root %s' % basepath('disk')) + utils.execute('sudo', 'chown', 'root', basepath('disk')) def to_xml(self, instance, rescue=False): # TODO(termie): cache? @@ -750,7 +779,7 @@ class LibvirtConnection(object): 'cpu_time': cpu_time} def get_diagnostics(self, instance_name): - raise exception.APIError(_("diagnostics are not supported " + raise exception.ApiError(_("diagnostics are not supported " "for libvirt")) def get_disks(self, instance_name): @@ -837,6 +866,159 @@ class LibvirtConnection(object): return interfaces + def get_vcpu_total(self): + """Get vcpu number of physical computer. + + :returns: the number of cpu core. + + """ + + # On certain platforms, this will raise a NotImplementedError. + try: + return multiprocessing.cpu_count() + except NotImplementedError: + LOG.warn(_("Cannot get the number of cpu, because this " + "function is not implemented for this platform. " + "This error can be safely ignored for now.")) + return 0 + + def get_memory_mb_total(self): + """Get the total memory size(MB) of physical computer. + + :returns: the total amount of memory(MB). + + """ + + if sys.platform.upper() != 'LINUX2': + return 0 + + meminfo = open('/proc/meminfo').read().split() + idx = meminfo.index('MemTotal:') + # transforming kb to mb. + return int(meminfo[idx + 1]) / 1024 + + def get_local_gb_total(self): + """Get the total hdd size(GB) of physical computer. + + :returns: + The total amount of HDD(GB). + Note that this value shows a partition where + NOVA-INST-DIR/instances mounts. + + """ + + hddinfo = os.statvfs(FLAGS.instances_path) + return hddinfo.f_frsize * hddinfo.f_blocks / 1024 / 1024 / 1024 + + def get_vcpu_used(self): + """ Get vcpu usage number of physical computer. + + :returns: The total number of vcpu that currently used. + + """ + + total = 0 + for dom_id in self._conn.listDomainsID(): + dom = self._conn.lookupByID(dom_id) + total += len(dom.vcpus()[1]) + return total + + def get_memory_mb_used(self): + """Get the free memory size(MB) of physical computer. + + :returns: the total usage of memory(MB). + + """ + + if sys.platform.upper() != 'LINUX2': + return 0 + + m = open('/proc/meminfo').read().split() + idx1 = m.index('MemFree:') + idx2 = m.index('Buffers:') + idx3 = m.index('Cached:') + avail = (int(m[idx1 + 1]) + int(m[idx2 + 1]) + int(m[idx3 + 1])) / 1024 + return self.get_memory_mb_total() - avail + + def get_local_gb_used(self): + """Get the free hdd size(GB) of physical computer. + + :returns: + The total usage of HDD(GB). + Note that this value shows a partition where + NOVA-INST-DIR/instances mounts. + + """ + + hddinfo = os.statvfs(FLAGS.instances_path) + avail = hddinfo.f_frsize * hddinfo.f_bavail / 1024 / 1024 / 1024 + return self.get_local_gb_total() - avail + + def get_hypervisor_type(self): + """Get hypervisor type. + + :returns: hypervisor type (ex. qemu) + + """ + + return self._conn.getType() + + def get_hypervisor_version(self): + """Get hypervisor version. + + :returns: hypervisor version (ex. 12003) + + """ + + return self._conn.getVersion() + + def get_cpu_info(self): + """Get cpuinfo information. + + Obtains cpu feature from virConnect.getCapabilities, + and returns as a json string. + + :return: see above description + + """ + + xml = self._conn.getCapabilities() + xml = libxml2.parseDoc(xml) + nodes = xml.xpathEval('//host/cpu') + if len(nodes) != 1: + raise exception.Invalid(_("Invalid xml. '<cpu>' must be 1," + "but %d\n") % len(nodes) + + xml.serialize()) + + cpu_info = dict() + cpu_info['arch'] = xml.xpathEval('//host/cpu/arch')[0].getContent() + cpu_info['model'] = xml.xpathEval('//host/cpu/model')[0].getContent() + cpu_info['vendor'] = xml.xpathEval('//host/cpu/vendor')[0].getContent() + + topology_node = xml.xpathEval('//host/cpu/topology')[0]\ + .get_properties() + topology = dict() + while topology_node != None: + name = topology_node.get_name() + topology[name] = topology_node.getContent() + topology_node = topology_node.get_next() + + keys = ['cores', 'sockets', 'threads'] + tkeys = topology.keys() + if list(set(tkeys)) != list(set(keys)): + ks = ', '.join(keys) + raise exception.Invalid(_("Invalid xml: topology(%(topology)s) " + "must have %(ks)s") % locals()) + + feature_nodes = xml.xpathEval('//host/cpu/feature') + features = list() + for nodes in feature_nodes: + features.append(nodes.get_properties().getContent()) + + cpu_info['topology'] = topology + cpu_info['features'] = features + return utils.dumps(cpu_info) + def block_stats(self, instance_name, disk): """ Note that this function takes an instance name, not an Instance, so @@ -867,6 +1049,207 @@ class LibvirtConnection(object): def refresh_security_group_members(self, security_group_id): self.firewall_driver.refresh_security_group_members(security_group_id) + def update_available_resource(self, ctxt, host): + """Updates compute manager resource info on ComputeNode table. + + This method is called when nova-coompute launches, and + whenever admin executes "nova-manage service update_resource". + + :param ctxt: security context + :param host: hostname that compute manager is currently running + + """ + + try: + service_ref = db.service_get_all_compute_by_host(ctxt, host)[0] + except exception.NotFound: + raise exception.Invalid(_("Cannot update compute manager " + "specific info, because no service " + "record was found.")) + + # Updating host information + dic = {'vcpus': self.get_vcpu_total(), + 'memory_mb': self.get_memory_mb_total(), + 'local_gb': self.get_local_gb_total(), + 'vcpus_used': self.get_vcpu_used(), + 'memory_mb_used': self.get_memory_mb_used(), + 'local_gb_used': self.get_local_gb_used(), + 'hypervisor_type': self.get_hypervisor_type(), + 'hypervisor_version': self.get_hypervisor_version(), + 'cpu_info': self.get_cpu_info()} + + compute_node_ref = service_ref['compute_node'] + if not compute_node_ref: + LOG.info(_('Compute_service record created for %s ') % host) + dic['service_id'] = service_ref['id'] + db.compute_node_create(ctxt, dic) + else: + LOG.info(_('Compute_service record updated for %s ') % host) + db.compute_node_update(ctxt, compute_node_ref[0]['id'], dic) + + def compare_cpu(self, cpu_info): + """Checks the host cpu is compatible to a cpu given by xml. + + "xml" must be a part of libvirt.openReadonly().getCapabilities(). + return values follows by virCPUCompareResult. + if 0 > return value, do live migration. + 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' + + :param cpu_info: json string that shows cpu feature(see get_cpu_info()) + :returns: + None. if given cpu info is not compatible to this server, + raise exception. + + """ + + LOG.info(_('Instance launched has CPU info:\n%s') % cpu_info) + dic = utils.loads(cpu_info) + xml = str(Template(self.cpuinfo_xml, searchList=dic)) + LOG.info(_('to xml...\n:%s ' % xml)) + + u = "http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult" + m = _("CPU doesn't have compatibility.\n\n%(ret)s\n\nRefer to %(u)s") + # unknown character exists in xml, then libvirt complains + try: + ret = self._conn.compareCPU(xml, 0) + except libvirt.libvirtError, e: + ret = e.message + LOG.error(m % locals()) + raise + + if ret <= 0: + raise exception.Invalid(m % locals()) + + return + + def ensure_filtering_rules_for_instance(self, instance_ref): + """Setting up filtering rules and waiting for its completion. + + To migrate an instance, filtering rules to hypervisors + and firewalls are inevitable on destination host. + ( Waiting only for filterling rules to hypervisor, + since filtering rules to firewall rules can be set faster). + + Concretely, the below method must be called. + - setup_basic_filtering (for nova-basic, etc.) + - prepare_instance_filter(for nova-instance-instance-xxx, etc.) + + to_xml may have to be called since it defines PROJNET, PROJMASK. + but libvirt migrates those value through migrateToURI(), + so , no need to be called. + + Don't use thread for this method since migration should + not be started when setting-up filtering rules operations + are not completed. + + :params instance_ref: nova.db.sqlalchemy.models.Instance object + + """ + + # If any instances never launch at destination host, + # basic-filtering must be set here. + self.firewall_driver.setup_basic_filtering(instance_ref) + # setting up n)ova-instance-instance-xx mainly. + self.firewall_driver.prepare_instance_filter(instance_ref) + + # wait for completion + timeout_count = range(FLAGS.live_migration_retry_count) + while timeout_count: + try: + filter_name = 'nova-instance-%s' % instance_ref.name + self._conn.nwfilterLookupByName(filter_name) + break + except libvirt.libvirtError: + timeout_count.pop() + if len(timeout_count) == 0: + ec2_id = instance_ref['hostname'] + iname = instance_ref.name + msg = _('Timeout migrating for %(ec2_id)s(%(iname)s)') + raise exception.Error(msg % locals()) + time.sleep(1) + + def live_migration(self, ctxt, instance_ref, dest, + post_method, recover_method): + """Spawning live_migration operation for distributing high-load. + + :params ctxt: security context + :params instance_ref: + nova.db.sqlalchemy.models.Instance object + instance object that is migrated. + :params dest: destination host + :params post_method: + post operation method. + expected nova.compute.manager.post_live_migration. + :params recover_method: + recovery method when any exception occurs. + expected nova.compute.manager.recover_live_migration. + + """ + + greenthread.spawn(self._live_migration, ctxt, instance_ref, dest, + post_method, recover_method) + + def _live_migration(self, ctxt, instance_ref, dest, + post_method, recover_method): + """Do live migration. + + :params ctxt: security context + :params instance_ref: + nova.db.sqlalchemy.models.Instance object + instance object that is migrated. + :params dest: destination host + :params post_method: + post operation method. + expected nova.compute.manager.post_live_migration. + :params recover_method: + recovery method when any exception occurs. + expected nova.compute.manager.recover_live_migration. + + """ + + # Do live migration. + try: + flaglist = FLAGS.live_migration_flag.split(',') + flagvals = [getattr(libvirt, x.strip()) for x in flaglist] + logical_sum = reduce(lambda x, y: x | y, flagvals) + + if self.read_only: + tmpconn = self._connect(self.libvirt_uri, False) + dom = tmpconn.lookupByName(instance_ref.name) + dom.migrateToURI(FLAGS.live_migration_uri % dest, + logical_sum, + None, + FLAGS.live_migration_bandwidth) + tmpconn.close() + else: + dom = self._conn.lookupByName(instance_ref.name) + dom.migrateToURI(FLAGS.live_migration_uri % dest, + logical_sum, + None, + FLAGS.live_migration_bandwidth) + + except Exception: + recover_method(ctxt, instance_ref) + raise + + # Waiting for completion of live_migration. + timer = utils.LoopingCall(f=None) + + def wait_for_live_migration(): + """waiting for live migration completion""" + try: + self.get_info(instance_ref.name)['state'] + except exception.NotFound: + timer.stop() + post_method(ctxt, instance_ref, dest) + + timer.f = wait_for_live_migration + timer.start(interval=0.5, now=True) + + def unfilter_instance(self, instance_ref): + """See comments of same method in firewall_driver.""" + self.firewall_driver.unfilter_instance(instance_ref) + class FirewallDriver(object): def prepare_instance_filter(self, instance): @@ -1208,10 +1591,16 @@ class NWFilterFirewall(FirewallDriver): class IptablesFirewallDriver(FirewallDriver): def __init__(self, execute=None, **kwargs): - self.execute = execute or utils.execute + from nova.network import linux_net + self.iptables = linux_net.iptables_manager self.instances = {} self.nwfilter = NWFilterFirewall(kwargs['get_connection']) + self.iptables.ipv4['filter'].add_chain('sg-fallback') + self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') + self.iptables.ipv6['filter'].add_chain('sg-fallback') + self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') + def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" return self.nwfilter.setup_basic_filtering(instance) @@ -1220,126 +1609,96 @@ class IptablesFirewallDriver(FirewallDriver): """No-op. Everything is done in prepare_instance_filter""" pass - def remove_instance(self, instance): - if instance['id'] in self.instances: - del self.instances[instance['id']] + def unfilter_instance(self, instance): + if self.instances.pop(instance['id'], None): + self.remove_filters_for_instance(instance) + self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' - 'filtered'), instance['id']) + 'filtered'), instance['id']) - def add_instance(self, instance): + def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance + self.add_filters_for_instance(instance) + self.iptables.apply() - def unfilter_instance(self, instance): - self.remove_instance(instance) - self.apply_ruleset() + def add_filters_for_instance(self, instance): + chain_name = self._instance_chain_name(instance) - def prepare_instance_filter(self, instance): - self.add_instance(instance) - self.apply_ruleset() - - def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter') - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter)) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save -t filter') - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', - process_input='\n'.join(new_filter)) + self.iptables.ipv4['filter'].add_chain(chain_name) + ipv4_address = self._ip_for_instance(instance) + self.iptables.ipv4['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv4_address, chain_name)) + + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain(chain_name) + ipv6_address = self._ip_for_instance_v6(instance) + self.iptables.ipv6['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv6_address, + chain_name)) - def modify_rules(self, current_lines, ip_version=4): + ipv4_rules, ipv6_rules = self.instance_rules(instance) + + for rule in ipv4_rules: + self.iptables.ipv4['filter'].add_rule(chain_name, rule) + + if FLAGS.use_ipv6: + for rule in ipv6_rules: + self.iptables.ipv6['filter'].add_rule(chain_name, rule) + + def remove_filters_for_instance(self, instance): + chain_name = self._instance_chain_name(instance) + + self.iptables.ipv4['filter'].remove_chain(chain_name) + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].remove_chain(chain_name) + + def instance_rules(self, instance): ctxt = context.get_admin_context() - # Remove any trace of nova rules. - new_filter = filter(lambda l: 'nova-' not in l, current_lines) - - seen_chains = False - for rules_index in range(len(new_filter)): - if not seen_chains: - if new_filter[rules_index].startswith(':'): - seen_chains = True - elif seen_chains == 1: - if not new_filter[rules_index].startswith(':'): - break - our_chains = [':nova-fallback - [0:0]'] - our_rules = ['-A nova-fallback -j DROP'] - - our_chains += [':nova-local - [0:0]'] - our_rules += ['-A FORWARD -j nova-local'] - our_rules += ['-A OUTPUT -j nova-local'] - - security_groups = {} - # Add our chains - # First, we add instance chains and rules - for instance_id in self.instances: - instance = self.instances[instance_id] - chain_name = self._instance_chain_name(instance) - if(ip_version == 4): - ip_address = self._ip_for_instance(instance) - elif(ip_version == 6): - ip_address = self._ip_for_instance_v6(instance) - - our_chains += [':%s - [0:0]' % chain_name] - - # Jump to the per-instance chain - our_rules += ['-A nova-local -d %s -j %s' % (ip_address, - chain_name)] - - # Always drop invalid packets - our_rules += ['-A %s -m state --state ' - 'INVALID -j DROP' % (chain_name,)] - - # Allow established connections - our_rules += ['-A %s -m state --state ' - 'ESTABLISHED,RELATED -j ACCEPT' % (chain_name,)] - - # Jump to each security group chain in turn - for security_group in \ - db.security_group_get_by_instance(ctxt, - instance['id']): - security_groups[security_group['id']] = security_group - - sg_chain_name = self._security_group_chain_name( - security_group['id']) + ipv4_rules = [] + ipv6_rules = [] - our_rules += ['-A %s -j %s' % (chain_name, sg_chain_name)] - - if(ip_version == 4): - # Allow DHCP responses - dhcp_server = self._dhcp_server_for_instance(instance) - our_rules += ['-A %s -s %s -p udp --sport 67 --dport 68 ' - '-j ACCEPT ' % (chain_name, dhcp_server)] - #Allow project network traffic - if (FLAGS.allow_project_net_traffic): - cidr = self._project_cidr_for_instance(instance) - our_rules += ['-A %s -s %s -j ACCEPT' % (chain_name, cidr)] - elif(ip_version == 6): - # Allow RA responses - ra_server = self._ra_server_for_instance(instance) - if ra_server: - our_rules += ['-A %s -s %s -p icmpv6 -j ACCEPT' % - (chain_name, ra_server + "/128")] - #Allow project network traffic - if (FLAGS.allow_project_net_traffic): - cidrv6 = self._project_cidrv6_for_instance(instance) - our_rules += ['-A %s -s %s -j ACCEPT' % - (chain_name, cidrv6)] - - # If nothing matches, jump to the fallback chain - our_rules += ['-A %s -j nova-fallback' % (chain_name,)] + # Always drop invalid packets + ipv4_rules += ['-m state --state ' 'INVALID -j DROP'] + ipv6_rules += ['-m state --state ' 'INVALID -j DROP'] - # then, security group chains and rules - for security_group_id in security_groups: - chain_name = self._security_group_chain_name(security_group_id) - our_chains += [':%s - [0:0]' % chain_name] + # Allow established connections + ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + + dhcp_server = self._dhcp_server_for_instance(instance) + ipv4_rules += ['-s %s -p udp --sport 67 --dport 68 ' + '-j ACCEPT' % (dhcp_server,)] + + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidr = self._project_cidr_for_instance(instance) + ipv4_rules += ['-s %s -j ACCEPT' % (cidr,)] + + # We wrap these in FLAGS.use_ipv6 because they might cause + # a DB lookup. The other ones are just list operations, so + # they're not worth the clutter. + if FLAGS.use_ipv6: + # Allow RA responses + ra_server = self._ra_server_for_instance(instance) + if ra_server: + ipv6_rules += ['-s %s/128 -p icmpv6 -j ACCEPT' % (ra_server,)] + + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidrv6 = self._project_cidrv6_for_instance(instance) + ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] - rules = \ - db.security_group_rule_get_by_security_group(ctxt, - security_group_id) + security_groups = db.security_group_get_by_instance(ctxt, + instance['id']) + + # then, security group chains and rules + for security_group in security_groups: + rules = db.security_group_rule_get_by_security_group(ctxt, + security_group['id']) for rule in rules: logging.info('%r', rule) @@ -1350,14 +1709,16 @@ class IptablesFirewallDriver(FirewallDriver): continue version = _get_ip_version(rule.cidr) - if version != ip_version: - continue + if version == 4: + rules = ipv4_rules + else: + rules = ipv6_rules protocol = rule.protocol if version == 6 and rule.protocol == 'icmp': protocol = 'icmpv6' - args = ['-A', chain_name, '-p', protocol, '-s', rule.cidr] + args = ['-p', protocol, '-s', rule.cidr] if rule.protocol in ['udp', 'tcp']: if rule.from_port == rule.to_port: @@ -1378,32 +1739,39 @@ class IptablesFirewallDriver(FirewallDriver): icmp_type_arg += '/%s' % icmp_code if icmp_type_arg: - if(ip_version == 4): + if version == 4: args += ['-m', 'icmp', '--icmp-type', icmp_type_arg] - elif(ip_version == 6): + elif version == 6: args += ['-m', 'icmp6', '--icmpv6-type', icmp_type_arg] args += ['-j ACCEPT'] - our_rules += [' '.join(args)] + rules += [' '.join(args)] + + ipv4_rules += ['-j $sg-fallback'] + ipv6_rules += ['-j $sg-fallback'] - new_filter[rules_index:rules_index] = our_rules - new_filter[rules_index:rules_index] = our_chains - logging.info('new_filter: %s', '\n'.join(new_filter)) - return new_filter + return ipv4_rules, ipv6_rules def refresh_security_group_members(self, security_group): pass def refresh_security_group_rules(self, security_group): - self.apply_ruleset() + # We use the semaphore to make sure noone applies the rule set + # after we've yanked the existing rules but before we've put in + # the new ones. + with self.iptables.semaphore: + for instance in self.instances.values(): + self.remove_filters_for_instance(instance) + self.add_filters_for_instance(instance) + self.iptables.apply() def _security_group_chain_name(self, security_group_id): return 'nova-sg-%s' % (security_group_id,) def _instance_chain_name(self, instance): - return 'nova-inst-%s' % (instance['id'],) + return 'inst-%s' % (instance['id'],) def _ip_for_instance(self, instance): return db.instance_get_fixed_address(context.get_admin_context(), diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index ce081a2d6..763c5fe40 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -41,9 +41,11 @@ from nova.virt.xenapi import HelperBase from nova.virt.xenapi.volume_utils import StorageError -FLAGS = flags.FLAGS LOG = logging.getLogger("nova.virt.xenapi.vm_utils") +FLAGS = flags.FLAGS +flags.DEFINE_string('default_os_type', 'linux', 'Default OS type') + XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, 'Running': power_state.RUNNING, @@ -80,10 +82,19 @@ class VMHelper(HelperBase): """ @classmethod - def create_vm(cls, session, instance, kernel, ramdisk, pv_kernel=False): + def create_vm(cls, session, instance, kernel, ramdisk, + use_pv_kernel=False): """Create a VM record. Returns a Deferred that gives the new VM reference. - the pv_kernel flag indicates whether the guest is HVM or PV + the use_pv_kernel flag indicates whether the guest is HVM or PV + + There are 3 scenarios: + + 1. Using paravirtualization, kernel passed in + + 2. Using paravirtualization, kernel within the image + + 3. Using hardware virtualization """ instance_type = instance_types.\ @@ -91,52 +102,61 @@ class VMHelper(HelperBase): mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) rec = { - 'name_label': instance.name, - 'name_description': '', + 'actions_after_crash': 'destroy', + 'actions_after_reboot': 'restart', + 'actions_after_shutdown': 'destroy', + 'affinity': '', + 'blocked_operations': {}, + 'ha_always_run': False, + 'ha_restart_priority': '', + 'HVM_boot_params': {}, + 'HVM_boot_policy': '', 'is_a_template': False, - 'memory_static_min': '0', - 'memory_static_max': mem, 'memory_dynamic_min': mem, 'memory_dynamic_max': mem, - 'VCPUs_at_startup': vcpus, - 'VCPUs_max': vcpus, - 'VCPUs_params': {}, - 'actions_after_shutdown': 'destroy', - 'actions_after_reboot': 'restart', - 'actions_after_crash': 'destroy', - 'PV_bootloader': '', - 'PV_kernel': '', - 'PV_ramdisk': '', + 'memory_static_min': '0', + 'memory_static_max': mem, + 'memory_target': mem, + 'name_description': '', + 'name_label': instance.name, + 'other_config': {'allowvssprovider': False}, + 'other_config': {}, + 'PCI_bus': '', + 'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true', + 'viridian': 'true', 'timeoffset': '0'}, 'PV_args': '', + 'PV_bootloader': '', 'PV_bootloader_args': '', + 'PV_kernel': '', 'PV_legacy_args': '', - 'HVM_boot_policy': '', - 'HVM_boot_params': {}, - 'platform': {}, - 'PCI_bus': '', + 'PV_ramdisk': '', 'recommendations': '', - 'affinity': '', + 'tags': [], 'user_version': '0', - 'other_config': {}, - } - #Complete VM configuration record according to the image type - #non-raw/raw with PV kernel/raw in HVM mode - if instance.kernel_id: - rec['PV_bootloader'] = '' - rec['PV_kernel'] = kernel - rec['PV_ramdisk'] = ramdisk - rec['PV_args'] = 'root=/dev/xvda1' - rec['PV_bootloader_args'] = '' - rec['PV_legacy_args'] = '' - else: - if pv_kernel: - rec['PV_args'] = 'noninteractive' - rec['PV_bootloader'] = 'pygrub' + 'VCPUs_at_startup': vcpus, + 'VCPUs_max': vcpus, + 'VCPUs_params': {}, + 'xenstore_data': {}} + + # Complete VM configuration record according to the image type + # non-raw/raw with PV kernel/raw in HVM mode + if use_pv_kernel: + rec['platform']['nx'] = 'false' + if instance.kernel_id: + # 1. Kernel explicitly passed in, use that + rec['PV_args'] = 'root=/dev/xvda1' + rec['PV_kernel'] = kernel + rec['PV_ramdisk'] = ramdisk else: - rec['HVM_boot_policy'] = 'BIOS order' - rec['HVM_boot_params'] = {'order': 'dc'} - rec['platform'] = {'acpi': 'true', 'apic': 'true', - 'pae': 'true', 'viridian': 'true'} + # 2. Use kernel within the image + rec['PV_args'] = 'clocksource=jiffies' + rec['PV_bootloader'] = 'pygrub' + else: + # 3. Using hardware virtualization + rec['platform']['nx'] = 'true' + rec['HVM_boot_params'] = {'order': 'dc'} + rec['HVM_boot_policy'] = 'BIOS order' + LOG.debug(_('Created VM %s...'), instance.name) vm_ref = session.call_xenapi('VM.create', rec) instance_name = instance.name @@ -181,13 +201,13 @@ class VMHelper(HelperBase): @classmethod def find_vbd_by_number(cls, session, vm_ref, number): """Get the VBD reference from the device number""" - vbds = session.get_xenapi().VM.get_VBDs(vm_ref) - if vbds: - for vbd in vbds: + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + if vbd_refs: + for vbd_ref in vbd_refs: try: - vbd_rec = session.get_xenapi().VBD.get_record(vbd) + vbd_rec = session.get_xenapi().VBD.get_record(vbd_ref) if vbd_rec['userdevice'] == str(number): - return vbd + return vbd_ref except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('VBD not found in instance %s') % vm_ref) @@ -319,7 +339,7 @@ class VMHelper(HelperBase): return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid) @classmethod - def upload_image(cls, session, instance_id, vdi_uuids, image_id): + def upload_image(cls, session, instance, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and push them into Glance using the specified human-friendly name. """ @@ -328,15 +348,18 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) + os_type = instance.os_type or FLAGS.default_os_type + params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, - 'sr_path': cls.get_sr_path(session)} + 'sr_path': cls.get_sr_path(session), + 'os_type': os_type} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) - session.wait_for_task(task, instance_id) + session.wait_for_task(task, instance.id) @classmethod def fetch_image(cls, session, instance_id, image, user, project, @@ -419,29 +442,29 @@ class VMHelper(HelperBase): vdi_size += MBR_SIZE_BYTES name_label = get_name_label_for_image(image) - vdi = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) + vdi_ref = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) - with_vdi_attached_here(session, vdi, False, + with_vdi_attached_here(session, vdi_ref, False, lambda dev: _stream_disk(dev, image_type, virtual_size, image_file)) if image_type == ImageType.KERNEL_RAMDISK: #we need to invoke a plugin for copying VDI's #content into proper path - LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi) + LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi_ref) fn = "copy_kernel_vdi" args = {} - args['vdi-ref'] = vdi + args['vdi-ref'] = vdi_ref #let the plugin copy the correct number of bytes args['image-size'] = str(vdi_size) task = session.async_call_plugin('glance', fn, args) filename = session.wait_for_task(task, instance_id) #remove the VDI as it is not needed anymore - session.get_xenapi().VDI.destroy(vdi) - LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi) + session.get_xenapi().VDI.destroy(vdi_ref) + LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi_ref) return filename else: - return session.get_xenapi().VDI.get_uuid(vdi) + return session.get_xenapi().VDI.get_uuid(vdi_ref) @classmethod def determine_disk_image_type(cls, instance): @@ -533,17 +556,33 @@ class VMHelper(HelperBase): return uuid @classmethod - def lookup_image(cls, session, instance_id, vdi_ref): + def determine_is_pv(cls, session, instance_id, vdi_ref, disk_image_type, + os_type): """ - Determine if VDI is using a PV kernel + Determine whether the VM will use a paravirtualized kernel or if it + will use hardware virtualization. + + 1. Objectstore (any image type): + We use plugin to figure out whether the VDI uses PV + + 2. Glance (VHD): then we use `os_type`, raise if not set + + 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is + available + + 4. Glance (DISK): pv is assumed """ if FLAGS.xenapi_image_service == 'glance': - return cls._lookup_image_glance(session, vdi_ref) + # 2, 3, 4: Glance + return cls._determine_is_pv_glance( + session, vdi_ref, disk_image_type, os_type) else: - return cls._lookup_image_objectstore(session, instance_id, vdi_ref) + # 1. Objecstore + return cls._determine_is_pv_objectstore(session, instance_id, + vdi_ref) @classmethod - def _lookup_image_objectstore(cls, session, instance_id, vdi_ref): + def _determine_is_pv_objectstore(cls, session, instance_id, vdi_ref): LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) fn = "is_vdi_pv" args = {} @@ -559,42 +598,72 @@ class VMHelper(HelperBase): return pv @classmethod - def _lookup_image_glance(cls, session, vdi_ref): + def _determine_is_pv_glance(cls, session, vdi_ref, disk_image_type, + os_type): + """ + For a Glance image, determine if we need paravirtualization. + + The relevant scenarios are: + 2. Glance (VHD): then we use `os_type`, raise if not set + + 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is + available + + 4. Glance (DISK): pv is assumed + """ + LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) - return with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + if disk_image_type == ImageType.DISK_VHD: + # 2. VHD + if os_type == 'windows': + is_pv = False + else: + is_pv = True + elif disk_image_type == ImageType.DISK_RAW: + # 3. RAW + is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + elif disk_image_type == ImageType.DISK: + # 4. Disk + is_pv = True + else: + raise exception.Error(_("Unknown image format %(disk_image_type)s") + % locals()) + + return is_pv @classmethod - def lookup(cls, session, i): + def lookup(cls, session, name_label): """Look the instance i up, and returns it if available""" - vms = session.get_xenapi().VM.get_by_name_label(i) - n = len(vms) + vm_refs = session.get_xenapi().VM.get_by_name_label(name_label) + n = len(vm_refs) if n == 0: return None elif n > 1: - raise exception.Duplicate(_('duplicate name found: %s') % i) + raise exception.Duplicate(_('duplicate name found: %s') % + name_label) else: - return vms[0] + return vm_refs[0] @classmethod - def lookup_vm_vdis(cls, session, vm): + def lookup_vm_vdis(cls, session, vm_ref): """Look for the VDIs that are attached to the VM""" # Firstly we get the VBDs, then the VDIs. # TODO(Armando): do we leave the read-only devices? - vbds = session.get_xenapi().VM.get_VBDs(vm) - vdis = [] - if vbds: - for vbd in vbds: + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + vdi_refs = [] + if vbd_refs: + for vbd_ref in vbd_refs: try: - vdi = session.get_xenapi().VBD.get_VDI(vbd) + vdi_ref = session.get_xenapi().VBD.get_VDI(vbd_ref) # Test valid VDI - record = session.get_xenapi().VDI.get_record(vdi) + record = session.get_xenapi().VDI.get_record(vdi_ref) LOG.debug(_('VDI %s is still available'), record['uuid']) except cls.XenAPI.Failure, exc: LOG.exception(exc) else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis + vdi_refs.append(vdi_ref) + if len(vdi_refs) > 0: + return vdi_refs else: return None @@ -770,16 +839,16 @@ def safe_find_sr(session): def find_sr(session): """Return the storage repository to hold VM images""" host = session.get_xenapi_host() - srs = session.get_xenapi().SR.get_all() - for sr in srs: - sr_rec = session.get_xenapi().SR.get_record(sr) + sr_refs = session.get_xenapi().SR.get_all() + for sr_ref in sr_refs: + sr_rec = session.get_xenapi().SR.get_record(sr_ref) if not ('i18n-key' in sr_rec['other_config'] and sr_rec['other_config']['i18n-key'] == 'local-storage'): continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.get_xenapi().PBD.get_record(pbd) + for pbd_ref in sr_rec['PBDs']: + pbd_rec = session.get_xenapi().PBD.get_record(pbd_ref) if pbd_rec['host'] == host: - return sr + return sr_ref return None @@ -804,11 +873,11 @@ def remap_vbd_dev(dev): return remapped_dev -def with_vdi_attached_here(session, vdi, read_only, f): +def with_vdi_attached_here(session, vdi_ref, read_only, f): this_vm_ref = get_this_vm_ref(session) vbd_rec = {} vbd_rec['VM'] = this_vm_ref - vbd_rec['VDI'] = vdi + vbd_rec['VDI'] = vdi_ref vbd_rec['userdevice'] = 'autodetect' vbd_rec['bootable'] = False vbd_rec['mode'] = read_only and 'RO' or 'RW' @@ -819,28 +888,28 @@ def with_vdi_attached_here(session, vdi, read_only, f): vbd_rec['qos_algorithm_type'] = '' vbd_rec['qos_algorithm_params'] = {} vbd_rec['qos_supported_algorithms'] = [] - LOG.debug(_('Creating VBD for VDI %s ... '), vdi) - vbd = session.get_xenapi().VBD.create(vbd_rec) - LOG.debug(_('Creating VBD for VDI %s done.'), vdi) + LOG.debug(_('Creating VBD for VDI %s ... '), vdi_ref) + vbd_ref = session.get_xenapi().VBD.create(vbd_rec) + LOG.debug(_('Creating VBD for VDI %s done.'), vdi_ref) try: - LOG.debug(_('Plugging VBD %s ... '), vbd) - session.get_xenapi().VBD.plug(vbd) - LOG.debug(_('Plugging VBD %s done.'), vbd) - orig_dev = session.get_xenapi().VBD.get_device(vbd) - LOG.debug(_('VBD %(vbd)s plugged as %(orig_dev)s') % locals()) + LOG.debug(_('Plugging VBD %s ... '), vbd_ref) + session.get_xenapi().VBD.plug(vbd_ref) + LOG.debug(_('Plugging VBD %s done.'), vbd_ref) + orig_dev = session.get_xenapi().VBD.get_device(vbd_ref) + LOG.debug(_('VBD %(vbd_ref)s plugged as %(orig_dev)s') % locals()) dev = remap_vbd_dev(orig_dev) if dev != orig_dev: - LOG.debug(_('VBD %(vbd)s plugged into wrong dev, ' + LOG.debug(_('VBD %(vbd_ref)s plugged into wrong dev, ' 'remapping to %(dev)s') % locals()) return f(dev) finally: - LOG.debug(_('Destroying VBD for VDI %s ... '), vdi) - vbd_unplug_with_retry(session, vbd) - ignore_failure(session.get_xenapi().VBD.destroy, vbd) - LOG.debug(_('Destroying VBD for VDI %s done.'), vdi) + LOG.debug(_('Destroying VBD for VDI %s ... '), vdi_ref) + vbd_unplug_with_retry(session, vbd_ref) + ignore_failure(session.get_xenapi().VBD.destroy, vbd_ref) + LOG.debug(_('Destroying VBD for VDI %s done.'), vdi_ref) -def vbd_unplug_with_retry(session, vbd): +def vbd_unplug_with_retry(session, vbd_ref): """Call VBD.unplug on the given VBD, with a retry if we get DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're seeing the device still in use, even when all processes using the device @@ -848,7 +917,7 @@ def vbd_unplug_with_retry(session, vbd): # FIXME(sirp): We can use LoopingCall here w/o blocking sleep() while True: try: - session.get_xenapi().VBD.unplug(vbd) + session.get_xenapi().VBD.unplug(vbd_ref) LOG.debug(_('VBD.unplug successful first time.')) return except VMHelper.XenAPI.Failure, e: @@ -917,14 +986,13 @@ def _write_partition(virtual_size, dev): LOG.debug(_('Writing partition table %(primary_first)d %(primary_last)d' ' to %(dest)s...') % locals()) - def execute(cmd, process_input=None, check_exit_code=True): - return utils.execute(cmd=cmd, - process_input=process_input, - check_exit_code=check_exit_code) + def execute(*cmd, **kwargs): + return utils.execute(*cmd, **kwargs) - execute('parted --script %s mklabel msdos' % dest) - execute('parted --script %s mkpart primary %ds %ds' % - (dest, primary_first, primary_last)) + execute('parted', '--script', dest, 'mklabel', 'msdos') + execute('parted', '--script', dest, 'mkpart', 'primary', + '%ds' % primary_first, + '%ds' % primary_last) LOG.debug(_('Writing partition table %s done.'), dest) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 562ecd4d5..488a61e8e 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -19,6 +19,7 @@ Management class for VM-related functions (spawn, reboot, etc). """ +import base64 import json import M2Crypto import os @@ -55,12 +56,12 @@ class VMOps(object): def list_instances(self): """List VM instances""" - vms = [] - for vm in self._session.get_xenapi().VM.get_all(): - rec = self._session.get_xenapi().VM.get_record(vm) - if not rec["is_a_template"] and not rec["is_control_domain"]: - vms.append(rec["name_label"]) - return vms + vm_refs = [] + for vm_ref in self._session.get_xenapi().VM.get_all(): + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + if not vm_rec["is_a_template"] and not vm_rec["is_control_domain"]: + vm_refs.append(vm_rec["name_label"]) + return vm_refs def _start(self, instance, vm_ref=None): """Power on a VM instance""" @@ -87,8 +88,8 @@ class VMOps(object): def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is not None: + vm_ref = VMHelper.lookup(self._session, instance_name) + if vm_ref is not None: raise exception.Duplicate(_('Attempted to create' ' non-unique name %s') % instance_name) @@ -104,31 +105,26 @@ class VMOps(object): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - kernel = ramdisk = pv_kernel = None - # Are we building from a pre-existing disk? vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) disk_image_type = VMHelper.determine_disk_image_type(instance) - if disk_image_type == ImageType.DISK_RAW: - # Have a look at the VDI and see if it has a PV kernel - pv_kernel = VMHelper.lookup_image(self._session, instance.id, - vdi_ref) - elif disk_image_type == ImageType.DISK_VHD: - # TODO(sirp): Assuming PV for now; this will need to be - # configurable as Windows will use HVM. - pv_kernel = True + kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) + ramdisk = None if instance.ramdisk_id: ramdisk = VMHelper.fetch_image(self._session, instance.id, instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) - vm_ref = VMHelper.create_vm(self._session, - instance, kernel, ramdisk, pv_kernel) + use_pv_kernel = VMHelper.determine_is_pv(self._session, instance.id, + vdi_ref, disk_image_type, instance.os_type) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, + use_pv_kernel) + VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, vdi_ref=vdi_ref, userdevice=0, bootable=True) @@ -141,19 +137,20 @@ class VMOps(object): LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) - def _inject_onset_files(): - onset_files = instance.onset_files - if onset_files: + def _inject_files(): + injected_files = instance.injected_files + if injected_files: # Check if this is a JSON-encoded string and convert if needed. - if isinstance(onset_files, basestring): + if isinstance(injected_files, basestring): try: - onset_files = json.loads(onset_files) + injected_files = json.loads(injected_files) except ValueError: - LOG.exception(_("Invalid value for onset_files: '%s'") - % onset_files) - onset_files = [] + LOG.exception( + _("Invalid value for injected_files: '%s'") + % injected_files) + injected_files = [] # Inject any files, if specified - for path, contents in instance.onset_files: + for path, contents in instance.injected_files: LOG.debug(_("Injecting file path: '%s'") % path) self.inject_file(instance, path, contents) # NOTE(armando): Do we really need to do this in virt? @@ -169,7 +166,7 @@ class VMOps(object): if state == power_state.RUNNING: LOG.debug(_('Instance %s: booted'), instance_name) timer.stop() - _inject_onset_files() + _inject_files() return True except Exception, exc: LOG.warn(exc) @@ -266,7 +263,7 @@ class VMOps(object): template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, image_id) + self._session, instance, template_vdi_uuids, image_id) finally: if template_vm_ref: self._destroy(instance, template_vm_ref, @@ -371,8 +368,8 @@ class VMOps(object): def reboot(self, instance): """Reboot VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.clean_reboot', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.clean_reboot', vm_ref) self._session.wait_for_task(task, instance.id) def set_admin_password(self, instance, new_pass): @@ -413,17 +410,16 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def inject_file(self, instance, b64_path, b64_contents): + def inject_file(self, instance, path, contents): """Write a file to the VM instance. The path to which it is to be - written and the contents of the file need to be supplied; both should + written and the contents of the file need to be supplied; both will be base64-encoded to prevent errors with non-ASCII characters being transmitted. If the agent does not support file injection, or the user has disabled it, a NotImplementedError will be raised. """ - # Files/paths *should* be base64-encoded at this point, but - # double-check to make sure. - b64_path = utils.ensure_b64_encoding(b64_path) - b64_contents = utils.ensure_b64_encoding(b64_contents) + # Files/paths must be base64-encoded for transmission to agent + b64_path = base64.b64encode(path) + b64_contents = base64.b64encode(contents) # Need to uniquely identify this request. transaction_id = str(uuid.uuid4()) @@ -439,7 +435,7 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def _shutdown(self, instance, vm, hard=True): + def _shutdown(self, instance, vm_ref, hard=True): """Shutdown an instance""" state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: @@ -453,31 +449,33 @@ class VMOps(object): try: task = None if hard: - task = self._session.call_xenapi("Async.VM.hard_shutdown", vm) + task = self._session.call_xenapi("Async.VM.hard_shutdown", + vm_ref) else: - task = self._session.call_xenapi('Async.VM.clean_shutdown', vm) + task = self._session.call_xenapi("Async.VM.clean_shutdown", + vm_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_vdis(self, instance, vm): - """Destroys all VDIs associated with a VM """ + def _destroy_vdis(self, instance, vm_ref): + """Destroys all VDIs associated with a VM""" instance_id = instance.id LOG.debug(_("Destroying VDIs for Instance %(instance_id)s") % locals()) - vdis = VMHelper.lookup_vm_vdis(self._session, vm) + vdi_refs = VMHelper.lookup_vm_vdis(self._session, vm_ref) - if not vdis: + if not vdi_refs: return - for vdi in vdis: + for vdi_ref in vdi_refs: try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_kernel_ramdisk(self, instance, vm): + def _destroy_kernel_ramdisk(self, instance, vm_ref): """ Three situations can occur: @@ -504,8 +502,8 @@ class VMOps(object): "both" % locals())) # 3. We have both kernel and ramdisk - (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk( - self._session, vm) + (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk(self._session, + vm_ref) LOG.debug(_("Removing kernel/ramdisk files")) @@ -516,11 +514,11 @@ class VMOps(object): LOG.debug(_("kernel/ramdisk files removed")) - def _destroy_vm(self, instance, vm): - """Destroys a VM record """ + def _destroy_vm(self, instance, vm_ref): + """Destroys a VM record""" instance_id = instance.id try: - task = self._session.call_xenapi('Async.VM.destroy', vm) + task = self._session.call_xenapi('Async.VM.destroy', vm_ref) self._session.wait_for_task(task, instance_id) except self.XenAPI.Failure, exc: LOG.exception(exc) @@ -536,10 +534,10 @@ class VMOps(object): """ instance_id = instance.id LOG.info(_("Destroying VM for Instance %(instance_id)s") % locals()) - vm = VMHelper.lookup(self._session, instance.name) - return self._destroy(instance, vm, shutdown=True) + vm_ref = VMHelper.lookup(self._session, instance.name) + return self._destroy(instance, vm_ref, shutdown=True) - def _destroy(self, instance, vm, shutdown=True, + def _destroy(self, instance, vm_ref, shutdown=True, destroy_kernel_ramdisk=True): """ Destroys VM instance by performing: @@ -549,17 +547,17 @@ class VMOps(object): 3. Destroying kernel and ramdisk files (if necessary) 4. Destroying that actual VM record """ - if vm is None: + if vm_ref is None: LOG.warning(_("VM is not present, skipping destroy...")) return if shutdown: - self._shutdown(instance, vm) + self._shutdown(instance, vm_ref) - self._destroy_vdis(instance, vm) + self._destroy_vdis(instance, vm_ref) if destroy_kernel_ramdisk: - self._destroy_kernel_ramdisk(instance, vm) - self._destroy_vm(instance, vm) + self._destroy_kernel_ramdisk(instance, vm_ref) + self._destroy_vm(instance, vm_ref) def _wait_with_callback(self, instance_id, task, callback): ret = None @@ -571,26 +569,27 @@ class VMOps(object): def pause(self, instance, callback): """Pause VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.pause', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.pause', vm_ref) self._wait_with_callback(instance.id, task, callback) def unpause(self, instance, callback): """Unpause VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.unpause', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.unpause', vm_ref) self._wait_with_callback(instance.id, task, callback) def suspend(self, instance, callback): """suspend the specified instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.suspend', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.suspend', vm_ref) self._wait_with_callback(instance.id, task, callback) def resume(self, instance, callback): """resume the specified instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.resume', vm, False, True) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.resume', vm_ref, False, + True) self._wait_with_callback(instance.id, task, callback) def rescue(self, instance, callback): @@ -600,29 +599,26 @@ class VMOps(object): - spawn a rescue VM (the vm name-label will be instance-N-rescue) """ - rescue_vm = VMHelper.lookup(self._session, instance.name + "-rescue") - if rescue_vm: + rescue_vm_ref = VMHelper.lookup(self._session, + instance.name + "-rescue") + if rescue_vm_ref: raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) - vm = self._get_vm_opaque_ref(instance) - self._shutdown(instance, vm) - self._acquire_bootlock(vm) + vm_ref = self._get_vm_opaque_ref(instance) + self._shutdown(instance, vm_ref) + self._acquire_bootlock(vm_ref) instance._rescue = True self.spawn(instance) - rescue_vm = self._get_vm_opaque_ref(instance) + rescue_vm_ref = self._get_vm_opaque_ref(instance) - vbd = self._session.get_xenapi().VM.get_VBDs(vm)[0] - vdi_ref = self._session.get_xenapi().VBD.get_record(vbd)["VDI"] - vbd_ref = VMHelper.create_vbd( - self._session, - rescue_vm, - vdi_ref, - 1, - False) + vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] + vdi_ref = self._session.get_xenapi().VBD.get_record(vbd_ref)["VDI"] + rescue_vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, + vdi_ref, 1, False) - self._session.call_xenapi("Async.VBD.plug", vbd_ref) + self._session.call_xenapi("Async.VBD.plug", rescue_vbd_ref) def unrescue(self, instance, callback): """Unrescue the specified instance @@ -631,51 +627,53 @@ class VMOps(object): - release the bootlock to allow the instance VM to start """ - rescue_vm = VMHelper.lookup(self._session, instance.name + "-rescue") + rescue_vm_ref = VMHelper.lookup(self._session, + instance.name + "-rescue") - if not rescue_vm: + if not rescue_vm_ref: raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) - original_vm = self._get_vm_opaque_ref(instance) - vbds = self._session.get_xenapi().VM.get_VBDs(rescue_vm) + original_vm_ref = self._get_vm_opaque_ref(instance) + vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) instance._rescue = False - for vbd_ref in vbds: - vbd = self._session.get_xenapi().VBD.get_record(vbd_ref) - if vbd["userdevice"] == "1": + for vbd_ref in vbd_refs: + _vbd_ref = self._session.get_xenapi().VBD.get_record(vbd_ref) + if _vbd_ref["userdevice"] == "1": VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) - task1 = self._session.call_xenapi("Async.VM.hard_shutdown", rescue_vm) + task1 = self._session.call_xenapi("Async.VM.hard_shutdown", + rescue_vm_ref) self._session.wait_for_task(task1, instance.id) - vdis = VMHelper.lookup_vm_vdis(self._session, rescue_vm) - for vdi in vdis: + vdi_refs = VMHelper.lookup_vm_vdis(self._session, rescue_vm_ref) + for vdi_ref in vdi_refs: try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure: continue - task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm) + task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm_ref) self._session.wait_for_task(task2, instance.id) - self._release_bootlock(original_vm) - self._start(instance, original_vm) + self._release_bootlock(original_vm_ref) + self._start(instance, original_vm_ref) def get_info(self, instance): """Return data about VM instance""" - vm = self._get_vm_opaque_ref(instance) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_info(rec) + vm_ref = self._get_vm_opaque_ref(instance) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + return VMHelper.compile_info(vm_rec) def get_diagnostics(self, instance): """Return data about VM diagnostics""" - vm = self._get_vm_opaque_ref(instance) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_diagnostics(self._session, rec) + vm_ref = self._get_vm_opaque_ref(instance) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + return VMHelper.compile_diagnostics(self._session, vm_rec) def get_console_output(self, instance): """Return snapshot of console""" @@ -698,9 +696,9 @@ class VMOps(object): # at this stage even though they aren't implemented because these will # be needed for multi-nic and there was no sense writing it for single # network/single IP and then having to turn around and re-write it - vm_opaque_ref = self._get_vm_opaque_ref(instance.id) + vm_ref = self._get_vm_opaque_ref(instance.id) logging.debug(_("injecting network info to xenstore for vm: |%s|"), - vm_opaque_ref) + vm_ref) admin_context = context.get_admin_context() IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) networks = db.network_get_all_by_instance(admin_context, @@ -731,11 +729,10 @@ class VMOps(object): 'ips': [ip_dict(ip) for ip in network_IPs], 'ip6s': [ip6_dict(ip) for ip in network_IPs]} - self.write_to_param_xenstore(vm_opaque_ref, {location: mapping}) + self.write_to_param_xenstore(vm_ref, {location: mapping}) try: - self.write_to_xenstore(vm_opaque_ref, location, - mapping['location']) + self.write_to_xenstore(vm_ref, location, mapping['location']) except KeyError: # catch KeyError for domid if instance isn't running pass @@ -747,8 +744,8 @@ class VMOps(object): Creates vifs for an instance """ - vm_opaque_ref = self._get_vm_opaque_ref(instance.id) - logging.debug(_("creating vif(s) for vm: |%s|"), vm_opaque_ref) + vm_ref = self._get_vm_opaque_ref(instance.id) + logging.debug(_("creating vif(s) for vm: |%s|"), vm_ref) if networks is None: networks = db.network_get_all_by_instance(admin_context, instance['id']) @@ -768,12 +765,8 @@ class VMOps(object): except AttributeError: device = "0" - VMHelper.create_vif( - self._session, - vm_opaque_ref, - network_ref, - instance.mac_address, - device) + VMHelper.create_vif(self._session, vm_ref, network_ref, + instance.mac_address, device) def reset_network(self, instance): """ @@ -837,9 +830,9 @@ class VMOps(object): Any errors raised by the plugin will in turn raise a RuntimeError here. """ instance_id = vm.id - vm = self._get_vm_opaque_ref(vm) - rec = self._session.get_xenapi().VM.get_record(vm) - args = {'dom_id': rec['domid'], 'path': path} + vm_ref = self._get_vm_opaque_ref(vm) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + args = {'dom_id': vm_rec['domid'], 'path': path} args.update(addl_args) try: task = self._session.async_call_plugin(plugin, method, args) @@ -919,9 +912,9 @@ class VMOps(object): value for 'keys' is passed, the returned dict is filtered to only return the values for those keys. """ - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) data = self._session.call_xenapi_request('VM.get_xenstore_data', - (vm, )) + (vm_ref, )) ret = {} if keys is None: keys = data.keys() @@ -939,11 +932,11 @@ class VMOps(object): """Takes a key/value pair and adds it to the xenstore parameter record for the given vm instance. If the key exists in xenstore, it is overwritten""" - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) self.remove_from_param_xenstore(instance_or_vm, key) jsonval = json.dumps(val) self._session.call_xenapi_request('VM.add_to_xenstore_data', - (vm, key, jsonval)) + (vm_ref, key, jsonval)) def write_to_param_xenstore(self, instance_or_vm, mapping): """Takes a dict and writes each key/value pair to the xenstore @@ -958,14 +951,14 @@ class VMOps(object): them from the xenstore parameter record data for the given VM. If the key doesn't exist, the request is ignored. """ - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) if isinstance(key_or_keys, basestring): keys = [key_or_keys] else: keys = key_or_keys for key in keys: self._session.call_xenapi_request('VM.remove_from_xenstore_data', - (vm, key)) + (vm_ref, key)) def clear_param_xenstore(self, instance_or_vm): """Removes all data from the xenstore parameter record for this VM.""" diff --git a/nova/virt/xenapi/volume_utils.py b/nova/virt/xenapi/volume_utils.py index d5ebd29d5..72284ac02 100644 --- a/nova/virt/xenapi/volume_utils.py +++ b/nova/virt/xenapi/volume_utils.py @@ -117,16 +117,16 @@ class VolumeHelper(HelperBase): def introduce_vdi(cls, session, sr_ref): """Introduce VDI in the host""" try: - vdis = session.get_xenapi().SR.get_VDIs(sr_ref) + vdi_refs = session.get_xenapi().SR.get_VDIs(sr_ref) except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('Unable to introduce VDI on SR %s') % sr_ref) try: - vdi_rec = session.get_xenapi().VDI.get_record(vdis[0]) + vdi_rec = session.get_xenapi().VDI.get_record(vdi_refs[0]) except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('Unable to get record' - ' of VDI %s on') % vdis[0]) + ' of VDI %s on') % vdi_refs[0]) else: try: return session.get_xenapi().VDI.introduce( diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index b63a5f8c3..da42a83b6 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -49,6 +49,12 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. address for the nova-volume host :target_port: iSCSI Target Port, 3260 Default :iqn_prefix: IQN Prefix, e.g. 'iqn.2010-10.org.openstack' + +**Variable Naming Scheme** + +- suffix "_ref" for opaque references +- suffix "_uuid" for UUIDs +- suffix "_rec" for record objects """ import sys @@ -263,6 +269,27 @@ class XenAPIConnection(object): 'username': FLAGS.xenapi_connection_username, 'password': FLAGS.xenapi_connection_password} + def update_available_resource(self, ctxt, host): + """This method is supported only by libvirt.""" + return + + def compare_cpu(self, xml): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + + def ensure_filtering_rules_for_instance(self, instance_ref): + """This method is supported only libvirt.""" + return + + def live_migration(self, context, instance_ref, dest, + post_method, recover_method): + """This method is supported only by libvirt.""" + return + + def unfilter_instance(self, instance_ref): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + class XenAPISession(object): """The session to invoke XenAPI SDK calls""" diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e3744c790..7b4bacdec 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -65,14 +65,14 @@ class VolumeDriver(object): self._execute = execute self._sync_exec = sync_exec - def _try_execute(self, command): + def _try_execute(self, *command): # NOTE(vish): Volume commands can partially fail due to timing, but # running them a second time on failure will usually # recover nicely. tries = 0 while True: try: - self._execute(command) + self._execute(*command) return True except exception.ProcessExecutionError: tries = tries + 1 @@ -84,7 +84,7 @@ class VolumeDriver(object): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" - out, err = self._execute("sudo vgs --noheadings -o name") + out, err = self._execute('sudo', 'vgs', '--noheadings', '-o', 'name') volume_groups = out.split() if not FLAGS.volume_group in volume_groups: raise exception.Error(_("volume group %s doesn't exist") @@ -97,22 +97,22 @@ class VolumeDriver(object): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("sudo lvcreate -L %s -n %s %s" % - (sizestr, + self._try_execute('sudo', 'lvcreate', '-L', sizestr, '-n', volume['name'], - FLAGS.volume_group)) + FLAGS.volume_group) def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute("sudo lvdisplay %s/%s" % + self._try_execute('sudo', 'lvdisplay', + '%s/%s' % (FLAGS.volume_group, volume['name'])) except Exception as e: # If the volume isn't present, then don't attempt to delete return True - self._try_execute("sudo lvremove -f %s/%s" % + self._try_execute('sudo', 'lvremove', '-f', "%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -143,6 +143,10 @@ class VolumeDriver(object): """Undiscover volume on a remote host.""" raise NotImplementedError() + def check_for_export(self, context, volume_id): + """Make sure volume is exported.""" + raise NotImplementedError() + class AOEDriver(VolumeDriver): """Implements AOE specific volume commands.""" @@ -168,12 +172,13 @@ class AOEDriver(VolumeDriver): blade_id) = self.db.volume_allocate_shelf_and_blade(context, volume['id']) self._try_execute( - "sudo vblade-persist setup %s %s %s /dev/%s/%s" % - (shelf_id, + 'sudo', 'vblade-persist', 'setup', + shelf_id, blade_id, FLAGS.aoe_eth_dev, - FLAGS.volume_group, - volume['name'])) + "/dev/%s/%s" % + (FLAGS.volume_group, + volume['name'])) # NOTE(vish): The standard _try_execute does not work here # because these methods throw errors if other # volumes on this host are in the process of @@ -182,9 +187,9 @@ class AOEDriver(VolumeDriver): # just wait a bit for the current volume to # be ready and ignore any errors. time.sleep(2) - self._execute("sudo vblade-persist auto all", + self._execute('sudo', 'vblade-persist', 'auto', 'all', check_exit_code=False) - self._execute("sudo vblade-persist start all", + self._execute('sudo', 'vblade-persist', 'start', 'all', check_exit_code=False) def remove_export(self, context, volume): @@ -192,20 +197,50 @@ class AOEDriver(VolumeDriver): (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) - self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + self._try_execute('sudo', 'vblade-persist', 'stop', + shelf_id, blade_id) + self._try_execute('sudo', 'vblade-persist', 'destroy', + shelf_id, blade_id) - def discover_volume(self, _volume): + def discover_volume(self, context, _volume): """Discover volume on a remote host.""" + (shelf_id, + blade_id) = self.db.volume_get_shelf_and_blade(context, + _volume['id']) self._execute("sudo aoe-discover") - self._execute("sudo aoe-stat", check_exit_code=False) + out, err = self._execute("sudo aoe-stat", check_exit_code=False) + device_path = 'e%(shelf_id)d.%(blade_id)d' % locals() + if out.find(device_path) >= 0: + return "/dev/etherd/%s" % device_path + else: + return def undiscover_volume(self, _volume): """Undiscover volume on a remote host.""" pass + def check_for_export(self, context, volume_id): + """Make sure volume is exported.""" + (shelf_id, + blade_id) = self.db.volume_get_shelf_and_blade(context, + volume_id) + cmd = "sudo vblade-persist ls --no-header" + out, _err = self._execute(cmd) + exported = False + for line in out.split('\n'): + param = line.split(' ') + if len(param) == 6 and param[0] == str(shelf_id) \ + and param[1] == str(blade_id) and param[-1] == "run": + exported = True + break + if not exported: + # Instance will be terminated in this case. + desc = _("Cannot confirm exported volume id:%(volume_id)s. " + "vblade process for e%(shelf_id)s.%(blade_id)s " + "isn't running.") % locals() + raise exception.ProcessExecutionError(out, _err, cmd=cmd, + description=desc) + class FakeAOEDriver(AOEDriver): """Logs calls instead of executing.""" @@ -252,13 +287,16 @@ class ISCSIDriver(VolumeDriver): iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - self._sync_exec("sudo ietadm --op new " - "--tid=%s --params Name=%s" % - (iscsi_target, iscsi_name), + self._sync_exec('sudo', 'ietadm', '--op', 'new', + "--tid=%s" % iscsi_target, + '--params', + "Name=%s" % iscsi_name, check_exit_code=False) - self._sync_exec("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path), + self._sync_exec('sudo', 'ietadm', '--op', 'new', + "--tid=%s" % iscsi_target, + '--lun=0', + '--params', + "Path=%s,Type=fileio" % volume_path, check_exit_code=False) def _ensure_iscsi_targets(self, context, host): @@ -279,12 +317,13 @@ class ISCSIDriver(VolumeDriver): volume['host']) iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - self._execute("sudo ietadm --op new " - "--tid=%s --params Name=%s" % + self._execute('sudo', 'ietadm', '--op', 'new', + '--tid=%s --params Name=%s' % (iscsi_target, iscsi_name)) - self._execute("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path)) + self._execute('sudo', 'ietadm', '--op', 'new', + '--tid=%s' % iscsi_target, + '--lun=0', '--params', + 'Path=%s,Type=fileio' % volume_path) def remove_export(self, context, volume): """Removes an export for a logical volume.""" @@ -299,16 +338,18 @@ class ISCSIDriver(VolumeDriver): try: # ietadm show will exit with an error # this export has already been removed - self._execute("sudo ietadm --op show --tid=%s " % iscsi_target) + self._execute('sudo', 'ietadm', '--op', 'show', + '--tid=%s' % iscsi_target) except Exception as e: LOG.info(_("Skipping remove_export. No iscsi_target " + "is presently exported for volume: %d"), volume['id']) return - self._execute("sudo ietadm --op delete --tid=%s " - "--lun=0" % iscsi_target) - self._execute("sudo ietadm --op delete --tid=%s" % - iscsi_target) + self._execute('sudo', 'ietadm', '--op', 'delete', + '--tid=%s' % iscsi_target, + '--lun=0') + self._execute('sudo', 'ietadm', '--op', 'delete', + '--tid=%s' % iscsi_target) def _do_iscsi_discovery(self, volume): #TODO(justinsb): Deprecate discovery and use stored info @@ -317,8 +358,8 @@ class ISCSIDriver(VolumeDriver): volume_name = volume['name'] - (out, _err) = self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % (volume['host'])) + (out, _err) = self._execute('sudo', 'iscsiadm', '-m', 'discovery', + '-t', 'sendtargets', '-p', volume['host']) for target in out.splitlines(): if FLAGS.iscsi_ip_prefix in target and volume_name in target: return target @@ -395,7 +436,7 @@ class ISCSIDriver(VolumeDriver): (property_key, property_value)) return self._run_iscsiadm(iscsi_properties, iscsi_command) - def discover_volume(self, volume): + def discover_volume(self, context, volume): """Discover volume on a remote host.""" iscsi_properties = self._get_iscsi_properties(volume) @@ -454,6 +495,20 @@ class ISCSIDriver(VolumeDriver): self._run_iscsiadm(iscsi_properties, "--logout") self._run_iscsiadm(iscsi_properties, "--op delete") + def check_for_export(self, context, volume_id): + """Make sure volume is exported.""" + + tid = self.db.volume_get_iscsi_target_num(context, volume_id) + try: + self._execute("sudo ietadm --op show --tid=%(tid)d" % locals()) + except exception.ProcessExecutionError, e: + # Instances remount read-only in this case. + # /etc/init.d/iscsitarget restart and rebooting nova-volume + # is better since ensure_export() works at boot time. + logging.error(_("Cannot confirm exported volume " + "id:%(volume_id)s.") % locals()) + raise + class FakeISCSIDriver(ISCSIDriver): """Logs calls instead of executing.""" @@ -478,7 +533,7 @@ class RBDDriver(VolumeDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" - (stdout, stderr) = self._execute("rados lspools") + (stdout, stderr) = self._execute('rados', 'lspools') pools = stdout.split("\n") if not FLAGS.rbd_pool in pools: raise exception.Error(_("rbd has no pool %s") % @@ -490,16 +545,13 @@ class RBDDriver(VolumeDriver): size = 100 else: size = int(volume['size']) * 1024 - self._try_execute("rbd --pool %s --size %d create %s" % - (FLAGS.rbd_pool, - size, - volume['name'])) + self._try_execute('rbd', '--pool', FLAGS.rbd_pool, + '--size', size, 'create', volume['name']) def delete_volume(self, volume): """Deletes a logical volume.""" - self._try_execute("rbd --pool %s rm %s" % - (FLAGS.rbd_pool, - volume['name'])) + self._try_execute('rbd', '--pool', FLAGS.rbd_pool, + 'rm', voluname['name']) def local_path(self, volume): """Returns the path of the rbd volume.""" @@ -534,7 +586,7 @@ class SheepdogDriver(VolumeDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" try: - (out, err) = self._execute("collie cluster info") + (out, err) = self._execute('collie', 'cluster', 'info') if not out.startswith('running'): raise exception.Error(_("Sheepdog is not working: %s") % out) except exception.ProcessExecutionError: @@ -546,12 +598,13 @@ class SheepdogDriver(VolumeDriver): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("qemu-img create sheepdog:%s %s" % - (volume['name'], sizestr)) + self._try_execute('qemu-img', 'create', + "sheepdog:%s" % volume['name'], + sizestr) def delete_volume(self, volume): """Deletes a logical volume""" - self._try_execute("collie vdi delete %s" % volume['name']) + self._try_execute('collie', 'vdi', 'delete', volume['name']) def local_path(self, volume): return "sheepdog:%s" % volume['name'] diff --git a/nova/volume/manager.py b/nova/volume/manager.py index c53acf1e3..2178389ce 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -49,7 +49,7 @@ from nova import context from nova import exception from nova import flags from nova import log as logging -from nova import scheduler_manager +from nova import manager from nova import utils @@ -64,7 +64,7 @@ flags.DEFINE_boolean('use_local_volumes', True, 'if True, will not discover local volumes') -class VolumeManager(scheduler_manager.SchedulerDependentManager): +class VolumeManager(manager.SchedulerDependentManager): """Manages attachable block storage devices.""" def __init__(self, volume_driver=None, *args, **kwargs): """Load the driver from the one specified in args, or from flags.""" @@ -161,7 +161,7 @@ class VolumeManager(scheduler_manager.SchedulerDependentManager): if volume_ref['host'] == self.host and FLAGS.use_local_volumes: path = self.driver.local_path(volume_ref) else: - path = self.driver.discover_volume(volume_ref) + path = self.driver.discover_volume(context, volume_ref) return path def remove_compute_volume(self, context, volume_id): @@ -172,3 +172,9 @@ class VolumeManager(scheduler_manager.SchedulerDependentManager): return True else: self.driver.undiscover_volume(volume_ref) + + def check_for_export(self, context, instance_id): + """Make sure whether volume is exported.""" + instance_ref = self.db.instance_get(context, instance_id) + for volume in instance_ref['volumes']: + self.driver.check_for_export(context, volume['id']) diff --git a/nova/wsgi.py b/nova/wsgi.py index 1eb66d067..ba0819466 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -36,6 +36,7 @@ import webob.exc from paste import deploy +from nova import exception from nova import flags from nova import log as logging from nova import utils @@ -82,6 +83,35 @@ class Server(object): log=WritableLogger(logger)) +class Request(webob.Request): + + def best_match_content_type(self): + """ + Determine the most acceptable content-type based on the + query extension then the Accept header + """ + + parts = self.path.rsplit(".", 1) + + if len(parts) > 1: + format = parts[1] + if format in ["json", "xml"]: + return "application/{0}".format(parts[1]) + + ctypes = ["application/json", "application/xml"] + bm = self.accept.best_match(ctypes) + + return bm or "application/json" + + def get_content_type(self): + try: + ct = self.headers["Content-Type"] + assert ct in ("application/xml", "application/json") + return ct + except Exception: + raise webob.exc.HTTPBadRequest("Invalid content type") + + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -113,7 +143,7 @@ class Application(object): def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: @@ -199,7 +229,7 @@ class Middleware(Application): """Do whatever you'd like to the response.""" return response - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): response = self.process_request(req) if response: @@ -212,7 +242,7 @@ class Debug(Middleware): """Helper class that can be inserted into any WSGI application chain to get information about the request and response.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): print ("*" * 40) + " REQUEST ENVIRON" for key, value in req.environ.items(): @@ -276,7 +306,7 @@ class Router(object): self._router = routes.middleware.RoutesMiddleware(self._dispatch, self.map) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Route the incoming request to a controller based on self.map. @@ -285,7 +315,7 @@ class Router(object): return self._router @staticmethod - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def _dispatch(req): """ Called by self._router after matching the incoming request to a route @@ -304,11 +334,11 @@ class Controller(object): WSGI app that reads routing information supplied by RoutesMiddleware and calls the requested action method upon itself. All action methods must, in addition to their normal parameters, accept a 'req' argument - which is the incoming webob.Request. They raise a webob.exc exception, + which is the incoming wsgi.Request. They raise a webob.exc exception, or return a dict which will be serialized by requested content type. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Call the method specified in req.environ by RoutesMiddleware. @@ -318,32 +348,45 @@ class Controller(object): method = getattr(self, action) del arg_dict['controller'] del arg_dict['action'] + if 'format' in arg_dict: + del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) + if type(result) is dict: - return self._serialize(result, req) + content_type = req.best_match_content_type() + body = self._serialize(result, content_type) + + response = webob.Response() + response.headers["Content-Type"] = content_type + response.body = body + return response + else: return result - def _serialize(self, data, request): + def _serialize(self, data, content_type): """ - Serialize the given dict to the response type requested in request. + Serialize the given dict to the provided content_type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.to_content_type(data) + serializer = Serializer(_metadata) + try: + return serializer.serialize(data, content_type) + except exception.InvalidContentType: + raise webob.exc.HTTPNotAcceptable() - def _deserialize(self, data, request): + def _deserialize(self, data, content_type): """ - Deserialize the request body to the response type requested in request. + Deserialize the request body to the specefied content type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.deserialize(data) + serializer = Serializer(_metadata) + return serializer.deserialize(data, content_type) class Serializer(object): @@ -351,50 +394,53 @@ class Serializer(object): Serializes and deserializes dictionaries to certain MIME types. """ - def __init__(self, environ, metadata=None): + def __init__(self, metadata=None): """ Create a serializer based on the given WSGI environment. 'metadata' is an optional dict mapping MIME types to information needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = webob.Request.blank('', environ) - suffix = req.path_info.split('.')[-1].lower() - if suffix == 'json': - self.handler = self._to_json - elif suffix == 'xml': - self.handler = self._to_xml - elif 'application/json' in req.accept: - self.handler = self._to_json - elif 'application/xml' in req.accept: - self.handler = self._to_xml - else: - # This is the default - self.handler = self._to_json - def to_content_type(self, data): - """ - Serialize a dictionary into a string. + def _get_serialize_handler(self, content_type): + handlers = { + "application/json": self._to_json, + "application/xml": self._to_xml, + } - The format of the string will be decided based on the Content Type - requested in self.environ: by Accept: header, or by URL suffix. + try: + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() + + def serialize(self, data, content_type): + """ + Serialize a dictionary into a string of the specified content type. """ - return self.handler(data) + return self._get_serialize_handler(content_type)(data) - def deserialize(self, datastring): + def deserialize(self, datastring, content_type): """ Deserialize a string to a dictionary. The string must be in the format of a supported MIME type. """ - datastring = datastring.strip() + return self.get_deserialize_handler(content_type)(datastring) + + def get_deserialize_handler(self, content_type): + handlers = { + "application/json": self._from_json, + "application/xml": self._from_xml, + } + try: - is_xml = (datastring[0] == '<') - if not is_xml: - return utils.loads(datastring) - return self._from_xml(datastring) - except: - return None + return handlers[content_type] + except Exception: + raise exception.InvalidContentType(_("Invalid content type %s" + % content_type)) + + def _from_json(self, datastring): + return utils.loads(datastring) def _from_xml(self, datastring): xmldata = self.metadata.get('application/xml', {}) |