From 2c0f1d78927c14f1d155e617a066b09a00acb100 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 18 Jan 2011 17:40:36 -0600 Subject: Basic stubbing throughout the stack --- nova/api/openstack/servers.py | 37 ++++++++++++++++++++++++++++++++++++- nova/compute/api.py | 14 ++++++++++++++ nova/compute/manager.py | 13 +++++++++++++ nova/virt/fake.py | 12 ++++++++++++ nova/virt/xenapi/vmops.py | 4 ++++ nova/virt/xenapi_conn.py | 4 ++++ 6 files changed, 83 insertions(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8cbcebed2..06104b37e 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -182,9 +182,44 @@ class Controller(wsgi.Controller): return exc.HTTPNoContent() def action(self, req, id): - """ Multi-purpose method used to reboot, rebuild, and + """ Multi-purpose method used to reboot, rebuild, or resize a server """ + + actions = { + 'reboot':self._action_reboot, + 'resize':self._action_resize, + 'confirmResize':self._action_confirm_resize, + 'revertResize':self._action_revert_resize, + 'rebuild':self._action_rebuild + } + input_dict = self._deserialize(req.body, req) + for key in actions.keys(): + if key in input_dict: + return actions[key](input_dict, req, id) + return faults.Fault(exc.HTTPNotImplemented()) + + def _action_confirm_resize(self, input_dict, req, id): + return fault.Fault(exc.HTTPNotImplemented()) + + def _action_revert_resize(self, input_dict, req, id): + return fault.Fault(exc.HTTPNotImplemented()) + + def _action_rebuild(self, input_dict, req, id): + return fault.Fault(exc.HTTPNotImplemented()) + + def _action_resize(self, input_dict, req, id): + """ Resizes a given instance to the flavor size requested """ + try: + resize_flavor = input_dict['resize']['flavorId'] + self.compute_api.resize(req.environ['nova.context'], id, + flavor_id) + except: + return faults.Fault(exc.HTTPUnprocessableEntity()) + return fault.Fault(exc.HTTPAccepted()) + + + def _action_reboot(self, input_dict, req, id): #TODO(sandy): rebuild/resize not supported. try: reboot_type = input_dict['reboot']['type'] diff --git a/nova/compute/api.py b/nova/compute/api.py index cc85ec691..22d5964c6 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -361,6 +361,20 @@ class API(base.Base): """Reboot the given instance.""" self._cast_compute_message('reboot_instance', context, instance_id) + def revert_resize(self, context, instance_id): + """Reverts a resize, deleting the 'new' instance in the process""" + raise NotImplemented() + + def confirm_resize(self, context, instance_id): + """Confirms a migration/resize, deleting the 'old' instance in the + process.""" + raise NotImplemented() + + def resize(self, context, instance_id, flavor_id): + """Resize a running instance.""" + self._cast_compute_message('resize_instance', context, instance_id, + flavor_id) + def pause(self, context, instance_id): """Pause the given instance.""" self._cast_compute_message('pause_instance', context, instance_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 613ee45f6..714cec209 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -376,6 +376,19 @@ class ComputeManager(manager.Manager): """Update instance state when async task completes.""" self._update_state(context, instance_id) + @exception.wrap_exception + @checks_instance_lock + def resize_instance(self, context, instance_id, flavor_id): + """Moves a running instance to another host, possibly changing the RAM + and disk size in the process""" + context = context.elevated() + instance_ref = self.db.instance_get(context. instance_id) + LOG.audit(_('instance %s: migrating'), instance_id, context=context) + self.db.instance_set_state(context, instance_id, power_state.RUNNING, + 'migrating') + self.driver.resize(instance_ref, flavor_id) + self._update_state(context, instance_id) + @exception.wrap_exception @checks_instance_lock def pause_instance(self, context, instance_id): diff --git a/nova/virt/fake.py b/nova/virt/fake.py index a57a8f43b..a6fe24c3e 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -138,6 +138,18 @@ class FakeConnection(object): """ pass + def resize(self, instance, flavor): + """ + Resizes/Migrates the specified instance. + + The flavor parameter determines whether or not the instance RAM and + disk space are modified, and if so, to what size. + + The work will be done asynchronously. This function returns a task + that allows the caller to detect when it is complete. + """ + pass + def set_admin_password(self, instance, new_pass): """ Set the root password on the specified instance. diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6e359ef82..50ee24325 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -207,6 +207,10 @@ class VMOps(object): logging.debug(_("Finished snapshot and upload for VM %s"), instance) + def resize(self, instance, flavor): + """Resize a running instance by changing it's RAM and disk size """ + raise NotImplementedError() + def reboot(self, instance): """Reboot VM instance""" vm = self._get_vm_opaque_ref(instance) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 689844f34..c850dbf80 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -145,6 +145,10 @@ class XenAPIConnection(object): """ Create snapshot from a running VM instance """ self._vmops.snapshot(instance, name) + def resize(self, instance, flavor): + """Resize a VM instance""" + raise NotImplementedError() + def reboot(self, instance): """Reboot VM instance""" self._vmops.reboot(instance) -- cgit From 6d2e2c52012abac8cab322357ce0ffd0ffc2fbaf Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 2 Feb 2011 10:49:02 -0600 Subject: Casting to the scheduler --- nova/compute/api.py | 10 +++++++--- nova/rpc.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 0faa066b5..283845709 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -404,10 +404,14 @@ class API(base.Base): process.""" raise NotImplemented() - def resize(self, context, instance_id, flavor_id): + def resize(self, context, instance_id, flavor): """Resize a running instance.""" - self._cast_compute_message('resize_instance', context, instance_id, - flavor_id) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "resize_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id, + "flavor": flavor}}) def pause(self, context, instance_id): """Pause the given instance.""" diff --git a/nova/rpc.py b/nova/rpc.py index 01fc6d44b..c4c938f4d 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -119,7 +119,7 @@ class Consumer(messaging.Consumer): LOG.error(_("Reconnected to queue")) self.failed_connection = False # NOTE(vish): This is catching all errors because we really don't - # exceptions to be logged 10 times a second if some + # want exceptions to be logged 10 times a second if some # persistent failure occurs. except Exception: # pylint: disable-msg=W0703 if not self.failed_connection: -- cgit From b65e994d9597f0a989b30eafc7a51bc34c4c361f Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 4 Feb 2011 15:13:18 -0600 Subject: Added a bunch of stubbed out functionality --- nova/compute/api.py | 22 +++++--- nova/compute/manager.py | 65 +++++++++++++++++++--- nova/db/api.py | 22 +++++++- nova/db/sqlalchemy/api.py | 53 ++++++++++++++++++ nova/db/sqlalchemy/models.py | 12 +++- nova/virt/xenapi/vmops.py | 18 +++++- nova/virt/xenapi_conn.py | 11 ++++ plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 ++ 8 files changed, 187 insertions(+), 20 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 283845709..f0d5ff2cb 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -379,6 +379,10 @@ class API(base.Base): kwargs = {'method': method, 'args': params} return rpc.call(context, queue, kwargs) + def _cast_scheduler_message(self, context, args) + """Generic handler for RPC calls to the scheduler""" + rpc.cast(context, FLAGS.scheduler_topic, args) + def snapshot(self, context, instance_id, name): """Snapshot the given instance. @@ -397,21 +401,23 @@ class API(base.Base): def revert_resize(self, context, instance_id): """Reverts a resize, deleting the 'new' instance in the process""" - raise NotImplemented() + instance_ref = self.db.instance_get(instance_id) + self._cast_compute_message('revert_resize', context, instance_id, + instance_ref['host']) def confirm_resize(self, context, instance_id): """Confirms a migration/resize, deleting the 'old' instance in the process.""" - raise NotImplemented() + migration_ref = self.db.get_migration_by_instance_id(instance_id) + self._cast_compute_message('confirm_resize', context, instance_id, + migration_ref['source_host']) def resize(self, context, instance_id, flavor): """Resize a running instance.""" - rpc.cast(context, - FLAGS.scheduler_topic, - {"method": "resize_instance", - "args": {"topic": FLAGS.compute_topic, - "instance_id": instance_id, - "flavor": flavor}}) + self._cast_scheduler_message(context, + {"method": "prep_resize", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id, }},) def pause(self, context, instance_id): """Pause the given instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 41ef23980..140db0d3e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -380,18 +380,67 @@ class ComputeManager(manager.Manager): """Update instance state when async task completes.""" self._update_state(context, instance_id) + @exception.wrap_exception @checks_instance_lock - def resize_instance(self, context, instance_id, flavor_id): - """Moves a running instance to another host, possibly changing the RAM - and disk size in the process""" + def prep_resize(self, context, instance_id): + """Initiates the process of moving a running instance to another + host, possibly changing the RAM and disk size in the process""" context = context.elevated() instance_ref = self.db.instance_get(context. instance_id) - LOG.audit(_('instance %s: migrating'), instance_id, context=context) - self.db.instance_set_state(context, instance_id, power_state.RUNNING, - 'migrating') - self.driver.resize(instance_ref, flavor_id) - self._update_state(context, instance_id) + migration_ref = self.db.migration_create(context, + { 'instance_id': instance_id, + 'source_host': instance_ref['host'], + 'dest_host': socket.gethostbyname(socket.gethostname()), + 'status': 'pre-migrating' } + LOG.audit(_('instance %s: migrating to '), instance_id, context=context) + service = self.db.service_get_by_host_and_topic(context, + instance_ref['host'], topic) + topic = self.db.queue_get_for(context, topic, service['id']) + rpc.cast(context, topic, + { 'method': 'resize_instance', + 'migration_id': migration_ref['id'], } + + @exception.wrap_exception + @checks_instance_lock + def resize_instance(self, context, migration_id): + """Starts the migration of a running instance to another host""" + migration_ref = self.db.migration_get(context, migration_id) + self.db.migration_update(context, migration_id, + { 'status': 'migrating', }) + self.driver.transfer_disk(context, instance_id, + migration_ref['dest_host']) + self.db.migration_update(context, migration_id, + { 'status': 'post-migrating', }) + + self.driver.power_off(context, migration_ref['instance_id']) + # This is where we would update the VM record after resizing + + service = self.db.service_get_by_host_and_topic(context, + migration_ref['dest_host'], topic) + topic = self.db.queue_get_for(context, topic, service['id']) + rpc.cast(context, topic, + { 'method': 'finish_resize', + 'migration_id': migration_ref['id'], } + + @exception.wrap_exception + @checks_instance_lock + def finish_resize(self, context, migration_id): + """Completes the migration process by setting up the newly transferred + disk and turning on the instance on its new host machine""" + migration_ref = self.db.migration_get(context, migration_id) + instance_ref = self.db.instance_get(context, + migration_ref['instance_id']) + + # this may get passed into the following spawn instead + self.driver.attach_disk(context, migration_ref['instance_id']) + self.driver.spawn(context, instance_ref, preexisting=True) + + self.db.migration_update(context, migration_id, + {'status': 'finished', }) + + # Cleans up any transferred files and unmounts things + self.driver.cleanup_disk_transfer(context, instance_ref['id']) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/api.py b/nova/db/api.py index 789cb8ebb..5da0e9840 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -80,10 +80,15 @@ def service_destroy(context, instance_id): def service_get(context, service_id): - """Get an service or raise if it does not exist.""" + """Get a service or raise if it does not exist.""" return IMPL.service_get(context, service_id) +def service_get_by_host_and_topic(context, host, topic): + """Get a service by host it's on and topic it listens to""" + return IMPL.service_get(context, host, topic) + + def service_get_all(context, disabled=False): """Get all service.""" return IMPL.service_get_all(context, None, disabled) @@ -255,6 +260,21 @@ def floating_ip_get_by_address(context, address): #################### +def migration_create(context, values): + """Create a migration record""" + return IMPL.migration_create(context, values) + +def migration_get(context, migration_id): + """Finds a migration by the id""" + return IMPL.migration_get(context, migration_id) + +def migration_get_by_instance_id(context, instance_id): + """Finds a migration by the instance id its migrating""" + return IMPL.migration_get_by_instance_id(context, instance_id) + +#################### + + def fixed_ip_associate(context, address, instance_id): """Associate fixed ip to instance. diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 85250d56e..e94f9f4d2 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -156,6 +156,15 @@ def service_get_all_by_topic(context, topic): filter_by(topic=topic).\ all() +@require_admin_context +def service_get_by_host_and_topic(context, host, topic): + session = get_session() + return session.query(models.Service).\ + filter_by(deleted=False).\ + filter_by(disabled=False).\ + filter_by(host=host).\ + filter_by(topic=topic).\ + all() @require_admin_context def service_get_all_by_host(context, host): @@ -1909,6 +1918,50 @@ def host_get_networks(context, host): all() +################### + + +@require_admin_context +def migration_create(context, values): + migration = models.Migration() + migration.update(values) + migration.save() + return migration + + +@require_admin_context +def migration_update(context, migration_id, values): + session = get_session() + with session.begin(): + migration = migration_get(context, migration_id, session=session) + migration.update(values) + return migration + + +@require_admin_context +def migration_get(context, migration_id): + session = get_session() + result = session.query(models.Migration.\ + filter_by(migration_id=migration_id)). + first() + if not result: + raise exception.NotFound(_("No migration found with id %s") + % migration_id) + return result + + +@require_admin_context +def migration_get_by_instance(context, instance_id): + session = get_session() + result = session.query(models.Migration.\ + filter_by(instance_id=instance_id)). + first() + if not result: + raise exception.NotFound(_("No migration found with instance id %s") + % migration_id) + return result + + ################## diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 7efb36c0e..499275504 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -366,6 +366,15 @@ class KeyPair(BASE, NovaBase): public_key = Column(Text) +class Migration(BASE, NovaBase): + """Represents a running host-to-host migration.""" + __tablename__ = 'migrations' + source_host = Column(String(255)) + dest_host = Column(String(255)) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) + status = Column(String(255)) #TODO(_cerberus_): enum + + class Network(BASE, NovaBase): """Represents a network.""" __tablename__ = 'networks' @@ -547,7 +556,8 @@ def register_models(): Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, - Project, Certificate, ConsolePool, Console) # , Image, Host + Project, Certificate, ConsolePool, Console, + Migration) # , Image, Host engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3f9eb39d1..468881355 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -60,6 +60,15 @@ class VMOps(object): vms.append(rec["name_label"]) return vms + def power_on(self, instance): + """Power on a VM instance""" + vm = VMHelper.lookup(self._session, instance.name) + if vm is None: + raise exception(_('Attempted to power on non-existent instance' + ' bad instance id %s') % instance.id) + LOG.debug(_("Starting instance %s"), instance.name) + self._session.call_xenapi('VM.start', vm, False, False) + def spawn(self, instance): """Create VM instance""" vm = VMHelper.lookup(self._session, instance.name) @@ -259,7 +268,8 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def _shutdown(self, instance, vm): + + def _shutdown(self, instance, vm, method='hard'): """Shutdown an instance """ state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: @@ -268,7 +278,11 @@ class VMOps(object): return try: - task = self._session.call_xenapi('Async.VM.hard_shutdown', vm) + task = None + if method == 'clean': + task = self._session.call_xenapi('Async.VM.clean_shutdown', vm) + else: + task = self._session.call_xenapi('Async.VM.hard_shutdown', vm) self._session.wait_for_task(instance.id, task) except self.XenAPI.Failure, exc: LOG.exception(exc) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 4637b4c29..628291764 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -184,6 +184,17 @@ class XenAPIConnection(object): """Unpause paused VM instance""" self._vmops.unpause(instance, callback) + def power_off(self, instance): + """Shuts down a running VM instance""" + self._vmops._shutdown(instance, method='clean') + + def power_on(self, instance): + """powers on a powered off VM instance""" + self._vmops.power_on(instance) + + def transfer_disk(self, instance, dest, callback): + self._vmops.transfer_disk( + def suspend(self, instance, callback): """suspend the specified instance""" self._vmops.suspend(instance, callback) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index aadacce57..3b7ceacc9 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -54,6 +54,10 @@ def copy_kernel_vdi(session,args): _copy_kernel_vdi('/dev/%s' % dev,copy_args)) return filename +def transfer_disk(dest, args): + vdi = exists(args, 'vdi-ref') + + def _copy_kernel_vdi(dest,copy_args): vdi_uuid=copy_args['vdi_uuid'] vdi_size=copy_args['vdi_size'] -- cgit From 855b9443cf109302e9882d527f237049b9624a05 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 4 Feb 2011 15:43:41 -0600 Subject: Didn't mean to actually make changes to the glance plugin --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 3b7ceacc9..aadacce57 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -54,10 +54,6 @@ def copy_kernel_vdi(session,args): _copy_kernel_vdi('/dev/%s' % dev,copy_args)) return filename -def transfer_disk(dest, args): - vdi = exists(args, 'vdi-ref') - - def _copy_kernel_vdi(dest,copy_args): vdi_uuid=copy_args['vdi_uuid'] vdi_size=copy_args['vdi_size'] -- cgit From 2458d674807d951a6b58c28cd334cd8d097822a9 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 7 Feb 2011 10:20:49 -0600 Subject: A few changes --- nova/compute/manager.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 140db0d3e..485efc047 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -380,6 +380,20 @@ class ComputeManager(manager.Manager): """Update instance state when async task completes.""" self._update_state(context, instance_id) + @exception.wrap_exception + @checks_instance_lock + def confirm_resize(self, context, instance_id): + """Destroys the old instance on the source machine""" + pass + + @exception.wrap_exception + @echecks_instance_lock + def revert_resize(self, context, instance_id): + """Destroys the new instance on the destination machine, + reverts the model changes, and powers on the old + instance on the source machine""" + pass + @exception.wrap_exception @checks_instance_lock @@ -395,8 +409,9 @@ class ComputeManager(manager.Manager): 'status': 'pre-migrating' } LOG.audit(_('instance %s: migrating to '), instance_id, context=context) service = self.db.service_get_by_host_and_topic(context, - instance_ref['host'], topic) - topic = self.db.queue_get_for(context, topic, service['id']) + instance_ref['host'], FLAGS.compute_topic) + topic = self.db.queue_get_for(context, FLAGS.compute_topic, + service['id']) rpc.cast(context, topic, { 'method': 'resize_instance', 'migration_id': migration_ref['id'], } @@ -417,8 +432,9 @@ class ComputeManager(manager.Manager): # This is where we would update the VM record after resizing service = self.db.service_get_by_host_and_topic(context, - migration_ref['dest_host'], topic) - topic = self.db.queue_get_for(context, topic, service['id']) + migration_ref['dest_host'], FLAGS.compute_topic) + topic = self.db.queue_get_for(context, FLAGS.compute_topic, + service['id']) rpc.cast(context, topic, { 'method': 'finish_resize', 'migration_id': migration_ref['id'], } -- cgit From e59c62efe5492e59fcc26b7b74f6ac2daa0caabe Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 7 Feb 2011 16:57:02 -0600 Subject: Added data_transfer xapi plugin --- nova/virt/xenapi_conn.py | 2 +- .../xenapi/etc/xapi.d/plugins/data_transfer | 44 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 628291764..acfde6caf 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -193,7 +193,7 @@ class XenAPIConnection(object): self._vmops.power_on(instance) def transfer_disk(self, instance, dest, callback): - self._vmops.transfer_disk( + self._vmops.transfer_disk() def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer new file mode 100644 index 000000000..d310a65d9 --- /dev/null +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +XenAPI Plugin for transfering data between host nodes +""" + +import os.path +import subprocess + +import XenAPIPlugin + +SSH_HOSTS = '/root/.ssh/known_hosts' +DEVNULL = '/dev/null' +KEYSCAN = '/usr/bin/ssh-keyscan' + +def _key_scan_and_add(host): + """SSH scans a remote host and writes the SSH key out to known_hosts""" + open(SSH_HOSTS, 'a').close() + null = open(DEVNULL, 'w') + known_hosts = open(SSH_HOSTS, 'a') + key = subprocess.Popen(['/usr/bin/ssh-keyscan', '-t', 'rsa', host], + stdout=subprocess.PIPE, stderr=null).communicate()[0].strip() + grep = subprocess.call(['/bin/grep', '-o', '%s' % key, SSH_HOSTS], + stdout=null, stderr=null) + if grep == 1: + known_hosts.write(key) + null.close() + known_hosts.close() -- cgit From a40f6041556ec09a1cb79c2b8abcec7fa70e72bf Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 7 Feb 2011 17:12:15 -0600 Subject: Some stuff --- nova/virt/xenapi_conn.py | 2 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index acfde6caf..726106b37 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -193,7 +193,7 @@ class XenAPIConnection(object): self._vmops.power_on(instance) def transfer_disk(self, instance, dest, callback): - self._vmops.transfer_disk() + self._vmops.transfer_disk(dest) def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer index d310a65d9..cde7bb823 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer @@ -28,10 +28,13 @@ import XenAPIPlugin SSH_HOSTS = '/root/.ssh/known_hosts' DEVNULL = '/dev/null' KEYSCAN = '/usr/bin/ssh-keyscan' +RSYNC = '/usr/bin/rsync' def _key_scan_and_add(host): """SSH scans a remote host and writes the SSH key out to known_hosts""" + # Touch the file if it doesn't yet exist open(SSH_HOSTS, 'a').close() + null = open(DEVNULL, 'w') known_hosts = open(SSH_HOSTS, 'a') key = subprocess.Popen(['/usr/bin/ssh-keyscan', '-t', 'rsa', host], @@ -42,3 +45,12 @@ def _key_scan_and_add(host): known_hosts.write(key) null.close() known_hosts.close() + +def transfer_vhd(host, vhd_path): + """Rsyncs a VHD to an adjacent host""" + _key_scan_and_add(host) + if subprocess.call([RSYNC, vhd_path, "%s:/root/" % host]) != 0: + raise Exception("Unexpected VHD transfer failure") + +if __name__ == '__main__': + XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd}) -- cgit From 203c94c89caabc1d4ece4c462819a90c05cde163 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 7 Feb 2011 17:39:53 -0600 Subject: blargh --- nova/compute/api.py | 2 +- nova/virt/xenapi/vmops.py | 8 ++++++++ nova/virt/xenapi_conn.py | 2 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer | 6 +++--- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index f0d5ff2cb..6b2628378 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -379,7 +379,7 @@ class API(base.Base): kwargs = {'method': method, 'args': params} return rpc.call(context, queue, kwargs) - def _cast_scheduler_message(self, context, args) + def _cast_scheduler_message(self, context, args): """Generic handler for RPC calls to the scheduler""" rpc.cast(context, FLAGS.scheduler_topic, args) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 468881355..4b835c707 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -220,6 +220,14 @@ class VMOps(object): logging.debug(_("Finished snapshot and upload for VM %s"), instance) + def transfer_disk(self, instance, dest): + """ Copies a VHD from one host machine to another + + :param instance: the instance that owns the VHD in question + :param dest: the destination host machine + """ + + def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ raise NotImplementedError() diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 726106b37..2e587117a 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -193,7 +193,7 @@ class XenAPIConnection(object): self._vmops.power_on(instance) def transfer_disk(self, instance, dest, callback): - self._vmops.transfer_disk(dest) + self._vmops.transfer_disk(instance, dest) def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer index cde7bb823..2af4a758b 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer @@ -46,11 +46,11 @@ def _key_scan_and_add(host): null.close() known_hosts.close() -def transfer_vhd(host, vhd_path): +def transfer_file(host, file_path): """Rsyncs a VHD to an adjacent host""" _key_scan_and_add(host) - if subprocess.call([RSYNC, vhd_path, "%s:/root/" % host]) != 0: + if subprocess.call([RSYNC, file_path, "%s:/root/" % host]) != 0: raise Exception("Unexpected VHD transfer failure") if __name__ == '__main__': - XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd}) + XenAPIPlugin.dispatch({'transfer_file': transfer_file}) -- cgit From 3f2cd17011e17991ebf1a77605686ce3dc48d92e Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 8 Feb 2011 10:47:23 -0600 Subject: Changes and bug fixes --- nova/compute/manager.py | 8 ++-- nova/db/sqlalchemy/api.py | 6 +-- .../sqlalchemy/migrate_repo/versions/003_cactus.py | 46 ++++++++++++++++++++++ nova/db/sqlalchemy/migration.py | 2 +- nova/virt/xenapi/vmops.py | 2 +- 5 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 485efc047..4189c49a4 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -387,7 +387,7 @@ class ComputeManager(manager.Manager): pass @exception.wrap_exception - @echecks_instance_lock + @checks_instance_lock def revert_resize(self, context, instance_id): """Destroys the new instance on the destination machine, reverts the model changes, and powers on the old @@ -406,7 +406,7 @@ class ComputeManager(manager.Manager): { 'instance_id': instance_id, 'source_host': instance_ref['host'], 'dest_host': socket.gethostbyname(socket.gethostname()), - 'status': 'pre-migrating' } + 'status': 'pre-migrating' }) LOG.audit(_('instance %s: migrating to '), instance_id, context=context) service = self.db.service_get_by_host_and_topic(context, instance_ref['host'], FLAGS.compute_topic) @@ -414,7 +414,7 @@ class ComputeManager(manager.Manager): service['id']) rpc.cast(context, topic, { 'method': 'resize_instance', - 'migration_id': migration_ref['id'], } + 'migration_id': migration_ref['id'], }) @exception.wrap_exception @checks_instance_lock @@ -437,7 +437,7 @@ class ComputeManager(manager.Manager): service['id']) rpc.cast(context, topic, { 'method': 'finish_resize', - 'migration_id': migration_ref['id'], } + 'migration_id': migration_ref['id'], }) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index e94f9f4d2..ece1cd373 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1942,8 +1942,7 @@ def migration_update(context, migration_id, values): def migration_get(context, migration_id): session = get_session() result = session.query(models.Migration.\ - filter_by(migration_id=migration_id)). - first() + filter_by(migration_id=migration_id)).first() if not result: raise exception.NotFound(_("No migration found with id %s") % migration_id) @@ -1954,8 +1953,7 @@ def migration_get(context, migration_id): def migration_get_by_instance(context, instance_id): session = get_session() result = session.query(models.Migration.\ - filter_by(instance_id=instance_id)). - first() + filter_by(instance_id=instance_id)).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") % migration_id) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py new file mode 100644 index 000000000..bbe5cbcb0 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -0,0 +1,46 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.from sqlalchemy import * + +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +# +# New Tables +# + +migrations = Table('migrations', meta, + Column('source_host', String(255)) + Column('dest_host', String(255)) + Column('instance_id', Integer, ForeignKey('instances.id'), nullable=True) + Column('status', String(255)) + ) + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (migrations): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise diff --git a/nova/db/sqlalchemy/migration.py b/nova/db/sqlalchemy/migration.py index 2a13c5466..644e3e45e 100644 --- a/nova/db/sqlalchemy/migration.py +++ b/nova/db/sqlalchemy/migration.py @@ -50,7 +50,7 @@ def db_version(): 'key_pairs', 'networks', 'projects', 'quotas', 'security_group_instance_association', 'security_group_rules', 'security_groups', - 'services', + 'services', 'migrations', 'users', 'user_project_association', 'user_project_role_association', 'user_role_association', diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 4b835c707..6a7621502 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -226,7 +226,7 @@ class VMOps(object): :param instance: the instance that owns the VHD in question :param dest: the destination host machine """ - + vm_ref = VMHelper.lookup(self._session, instance.name) def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ -- cgit From 49e07d0581317daf1bb605d56575c62743a210be Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 8 Feb 2011 11:07:03 -0600 Subject: Commas help --- nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py index bbe5cbcb0..dc384fbc3 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -27,10 +27,11 @@ meta = MetaData() # migrations = Table('migrations', meta, - Column('source_host', String(255)) - Column('dest_host', String(255)) - Column('instance_id', Integer, ForeignKey('instances.id'), nullable=True) - Column('status', String(255)) + Column('source_host', String(255)), + Column('dest_host', String(255)), + Column('instance_id', Integer, ForeignKey('instances.id'), + nullable=True), + Column('status', String(255)) ) def upgrade(migrate_engine): -- cgit From 089286802db0dca22cd67e46f26fab3ab0a3a73b Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 8 Feb 2011 13:12:21 -0600 Subject: Typos and primary keys --- nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py index dc384fbc3..4d01cd874 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -27,6 +27,7 @@ meta = MetaData() # migrations = Table('migrations', meta, + Column('id', Integer(), primary_key=True, nullable=False), Column('source_host', String(255)), Column('dest_host', String(255)), Column('instance_id', Integer, ForeignKey('instances.id'), @@ -38,7 +39,7 @@ def upgrade(migrate_engine): # Upgrade operations go here. Don't create your own engine; # bind migrate_engine to your metadata meta.bind = migrate_engine - for table in (migrations): + for table in (migrations, ): try: table.create() except Exception: -- cgit From ce5e3bdd30712aa6704926e6cdeb5ae73ae8200b Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 9 Feb 2011 15:26:37 -0600 Subject: A lot of stuff --- nova/compute/manager.py | 8 ++-- nova/db/sqlalchemy/models.py | 1 + nova/virt/xenapi/vmops.py | 53 +++++++++++++++++----- nova/virt/xenapi_conn.py | 8 +++- .../xenapi/etc/xapi.d/plugins/data_transfer | 37 +++++++-------- 5 files changed, 71 insertions(+), 36 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 4189c49a4..ac09f7c8c 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -423,12 +423,12 @@ class ComputeManager(manager.Manager): migration_ref = self.db.migration_get(context, migration_id) self.db.migration_update(context, migration_id, { 'status': 'migrating', }) - self.driver.transfer_disk(context, instance_id, + + self.driver.migrate_disk_and_power_off(context, instance, migration_ref['dest_host']) + self.db.migration_update(context, migration_id, { 'status': 'post-migrating', }) - - self.driver.power_off(context, migration_ref['instance_id']) # This is where we would update the VM record after resizing service = self.db.service_get_by_host_and_topic(context, @@ -449,7 +449,7 @@ class ComputeManager(manager.Manager): migration_ref['instance_id']) # this may get passed into the following spawn instead - self.driver.attach_disk(context, migration_ref['instance_id']) + self.driver.attach_disk(context, instance_ref) self.driver.spawn(context, instance_ref, preexisting=True) self.db.migration_update(context, migration_id, diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 499275504..ebf3a382b 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -369,6 +369,7 @@ class KeyPair(BASE, NovaBase): class Migration(BASE, NovaBase): """Represents a running host-to-host migration.""" __tablename__ = 'migrations' + id = Column(Integer, primary_key=True, nullable=False) source_host = Column(String(255)) dest_host = Column(String(255)) instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6a7621502..40b075b3d 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -196,6 +196,26 @@ class VMOps(object): Glance. """ + with self._get_snapshot(instance) as snapshot: + # call plugin to ship snapshot off to glance + VMHelper.upload_image( + self._session, instance.id, snapshot.vdi_uuids, image_id) + + logging.debug(_("Finished snapshot and upload for VM %s"), instance) + + def _get_snapshot(self, instance): + class Snapshot(object): + def __init__(self, virt, instance, vdis): + self.instance = instance + self.vdi_uuids = vdis + self.virt = virt + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.virt._destroy(self.instance, self.vm_ref, shutdown=False) + #TODO(sirp): Add quiesce and VSS locking support when Windows support # is added @@ -204,30 +224,41 @@ class VMOps(object): label = "%s-snapshot" % instance.name try: - template_vm_ref, template_vdi_uuids = VMHelper.create_snapshot( + _, template_vdi_uuids = VMHelper.create_snapshot( self._session, instance.id, vm_ref, label) + return Snapshot(self, instance, template_vdi_uuids) except self.XenAPI.Failure, exc: logging.error(_("Unable to Snapshot %(vm_ref)s: %(exc)s") % locals()) return - try: - # call plugin to ship snapshot off to glance - VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, image_id) - finally: - self._destroy(instance, template_vm_ref, shutdown=False) - - logging.debug(_("Finished snapshot and upload for VM %s"), instance) - - def transfer_disk(self, instance, dest): + def migrate_disk_and_power_off(self, instance, dest): """ Copies a VHD from one host machine to another :param instance: the instance that owns the VHD in question :param dest: the destination host machine + :param disk_type: values are 'primary' or 'cow' """ vm_ref = VMHelper.lookup(self._session, instance.name) + # The primary VDI becomes the COW after the snapshot. We can figure + # this out from the VBD. The base copy is the parent_uuid returned + # from the snapshot creation + with self._get_snapshot(instance) as snapshot: + params = {'host':dest, 'vdi_uuid':snapshot.vdi_uuids[1]} + kwargs = {'params': pickle.dumps(params)} + self._session.async_call_plugin('data_transfer', 'transfer_vhd', + kwargs) + + # Now power down the instance and transfer the COW VHD + self._shutdown(instance, method='clean') + + _, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid']} + kwargs = {'params': pickle.dumps(params)} + self._session.async_call_plugin('data_transfer', 'transfer_vhd', + kwargs) + def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ raise NotImplementedError() diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 2e587117a..98b5e7851 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -192,8 +192,14 @@ class XenAPIConnection(object): """powers on a powered off VM instance""" self._vmops.power_on(instance) - def transfer_disk(self, instance, dest, callback): + def migrate_disk_and_power_off(self, instance, dest): + """Transfers the VHD of a running instance to another host, then shuts + off the instance copies over the COW disk""" self._vmops.transfer_disk(instance, dest) + + def move_disk(self, instance_ref): + """Moves the copied VDIs into the SR""" + pass def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer index 2af4a758b..bd46e1c0b 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer @@ -21,6 +21,7 @@ XenAPI Plugin for transfering data between host nodes """ import os.path +import pickle import subprocess import XenAPIPlugin @@ -30,27 +31,23 @@ DEVNULL = '/dev/null' KEYSCAN = '/usr/bin/ssh-keyscan' RSYNC = '/usr/bin/rsync' -def _key_scan_and_add(host): - """SSH scans a remote host and writes the SSH key out to known_hosts""" - # Touch the file if it doesn't yet exist - open(SSH_HOSTS, 'a').close() - - null = open(DEVNULL, 'w') - known_hosts = open(SSH_HOSTS, 'a') - key = subprocess.Popen(['/usr/bin/ssh-keyscan', '-t', 'rsa', host], - stdout=subprocess.PIPE, stderr=null).communicate()[0].strip() - grep = subprocess.call(['/bin/grep', '-o', '%s' % key, SSH_HOSTS], - stdout=null, stderr=null) - if grep == 1: - known_hosts.write(key) - null.close() - known_hosts.close() - -def transfer_file(host, file_path): + +def transfer_vhd(session, args): """Rsyncs a VHD to an adjacent host""" - _key_scan_and_add(host) - if subprocess.call([RSYNC, file_path, "%s:/root/" % host]) != 0: + params = pickle.dumps(args) + instance_id = params['instance_id'] + host = params['host'] + vdi_uuid = params['vdi_uuid'] + sr_path = get_sr_path(session) + vhd_path = "%s.vhd" % vdi_uuid + + source_path = "%s/%s" % (sr_path, vhd_path) + dest_path = '%s:/images/instance%d/' % (host, instance_id) + rsync_args = [['nohup', RSYNC, '-av', '--progress', + '-e "ssh -o StrictHostKeyChecking=no"', source_path, dest_path] + + if subprocess.call(rsync_args) != 0: raise Exception("Unexpected VHD transfer failure") if __name__ == '__main__': - XenAPIPlugin.dispatch({'transfer_file': transfer_file}) + XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd}) -- cgit From 482c7b57a3d0ac8bf6df98539bf8a1220470e0f7 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 9 Feb 2011 15:27:23 -0600 Subject: Renamed migration plugin --- .../xenapi/etc/xapi.d/plugins/data_transfer | 53 ---------------------- .../xenserver/xenapi/etc/xapi.d/plugins/migration | 53 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 53 deletions(-) delete mode 100644 plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer create mode 100644 plugins/xenserver/xenapi/etc/xapi.d/plugins/migration diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer b/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer deleted file mode 100644 index bd46e1c0b..000000000 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/data_transfer +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -XenAPI Plugin for transfering data between host nodes -""" - -import os.path -import pickle -import subprocess - -import XenAPIPlugin - -SSH_HOSTS = '/root/.ssh/known_hosts' -DEVNULL = '/dev/null' -KEYSCAN = '/usr/bin/ssh-keyscan' -RSYNC = '/usr/bin/rsync' - - -def transfer_vhd(session, args): - """Rsyncs a VHD to an adjacent host""" - params = pickle.dumps(args) - instance_id = params['instance_id'] - host = params['host'] - vdi_uuid = params['vdi_uuid'] - sr_path = get_sr_path(session) - vhd_path = "%s.vhd" % vdi_uuid - - source_path = "%s/%s" % (sr_path, vhd_path) - dest_path = '%s:/images/instance%d/' % (host, instance_id) - rsync_args = [['nohup', RSYNC, '-av', '--progress', - '-e "ssh -o StrictHostKeyChecking=no"', source_path, dest_path] - - if subprocess.call(rsync_args) != 0: - raise Exception("Unexpected VHD transfer failure") - -if __name__ == '__main__': - XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd}) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration new file mode 100644 index 000000000..bd46e1c0b --- /dev/null +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +XenAPI Plugin for transfering data between host nodes +""" + +import os.path +import pickle +import subprocess + +import XenAPIPlugin + +SSH_HOSTS = '/root/.ssh/known_hosts' +DEVNULL = '/dev/null' +KEYSCAN = '/usr/bin/ssh-keyscan' +RSYNC = '/usr/bin/rsync' + + +def transfer_vhd(session, args): + """Rsyncs a VHD to an adjacent host""" + params = pickle.dumps(args) + instance_id = params['instance_id'] + host = params['host'] + vdi_uuid = params['vdi_uuid'] + sr_path = get_sr_path(session) + vhd_path = "%s.vhd" % vdi_uuid + + source_path = "%s/%s" % (sr_path, vhd_path) + dest_path = '%s:/images/instance%d/' % (host, instance_id) + rsync_args = [['nohup', RSYNC, '-av', '--progress', + '-e "ssh -o StrictHostKeyChecking=no"', source_path, dest_path] + + if subprocess.call(rsync_args) != 0: + raise Exception("Unexpected VHD transfer failure") + +if __name__ == '__main__': + XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd}) -- cgit From ac33f61c5c382fc7c8e8ab872192858860672d70 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 9 Feb 2011 16:38:27 -0600 Subject: Plugin tidying and more migration implementation --- nova/virt/xenapi/vm_utils.py | 16 +++--- nova/virt/xenapi/vmops.py | 35 +++++++---- nova/virt/xenapi_conn.py | 9 ++- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 +- .../xenserver/xenapi/etc/xapi.d/plugins/migration | 67 +++++++++++++++++++++- 5 files changed, 108 insertions(+), 23 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 4bbd522c1..e16662aad 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -478,6 +478,14 @@ class VMHelper(HelperBase): except cls.XenAPI.Failure as e: return {"Unable to retrieve diagnostics": e} + @classmethod + def scan_sr(cls, session, instance_id, sr_ref): + LOG.debug(_("Re-scanning SR %s"), sr_ref) + task = session.call_xenapi('Async.SR.scan', sr_ref) + session.wait_for_task(instance_id, task) + + + def get_rrd(host, uuid): """Return the VM RRD XML as a string""" @@ -520,12 +528,6 @@ def get_vhd_parent_uuid(session, vdi_ref): return None -def scan_sr(session, instance_id, sr_ref): - LOG.debug(_("Re-scanning SR %s"), sr_ref) - task = session.call_xenapi('Async.SR.scan', sr_ref) - session.wait_for_task(instance_id, task) - - def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, original_parent_uuid): """ Spin until the parent VHD is coalesced into its parent VHD @@ -550,7 +552,7 @@ def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, " %(max_attempts)d), giving up...") % locals()) raise exception.Error(msg) - scan_sr(session, instance_id, sr_ref) + VMHelper.scan_sr(session, instance_id, sr_ref) parent_uuid = get_vhd_parent_uuid(session, vdi_ref) if original_parent_uuid and (parent_uuid != original_parent_uuid): LOG.debug(_("Parent %(parent_uuid)s doesn't match original parent" diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 40b075b3d..ea4b7899b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -146,7 +146,7 @@ class VMOps(object): """ vm = None try: - if instance_or_vm.startswith("OpaqueRef:"): + if instance_or_vm.startswith("OpaqueRef:") # Got passed an opaque ref; return it return instance_or_vm else: @@ -241,23 +241,38 @@ class VMOps(object): """ vm_ref = VMHelper.lookup(self._session, instance.name) - # The primary VDI becomes the COW after the snapshot. We can figure - # this out from the VBD. The base copy is the parent_uuid returned + # The primary VDI becomes the COW after the snapshot, and we can + # identify it via the VBD. The base copy is the parent_uuid returned # from the snapshot creation + + #TODO(mdietz): explicitly forcing the base_copy and cow names is + #pretty fugly with self._get_snapshot(instance) as snapshot: - params = {'host':dest, 'vdi_uuid':snapshot.vdi_uuids[1]} - kwargs = {'params': pickle.dumps(params)} - self._session.async_call_plugin('data_transfer', 'transfer_vhd', - kwargs) + params = {'host':dest, 'vdi_uuid':snapshot.vdi_uuids[1], + 'dest_name':'base_copy.vhd'} + self._session.async_call_plugin('migration', 'transfer_vhd', + {'params': pickle.dumps(params)}) # Now power down the instance and transfer the COW VHD self._shutdown(instance, method='clean') _, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) - params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid']} - kwargs = {'params': pickle.dumps(params)} + params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], + 'dest_name': 'cow.vhd'} self._session.async_call_plugin('data_transfer', 'transfer_vhd', - kwargs) + {'params': pickle.dumps(params)}) + return snapshot.vdi_uuids[1], vm_vdi_rec['uuid'] + + def attach_disk(self, instance): + vm_ref = VMHelper.lookup(self._session, instance.name) + + params = { 'instance_id': instance.id } + self._session.async_call_plugin('migration', 'move_vhds_into_sr', + {'params': pickle.dumps(params)}) + + + _, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + VMHelper.scan_sr(self._session, instance.id, vm_vdi_rec['SR']) def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 98b5e7851..cc43050b4 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -164,6 +164,9 @@ class XenAPIConnection(object): """Resize a VM instance""" raise NotImplementedError() + def attach_disk(self, instance_ref): + + def reboot(self, instance): """Reboot VM instance""" self._vmops.reboot(instance) @@ -195,11 +198,11 @@ class XenAPIConnection(object): def migrate_disk_and_power_off(self, instance, dest): """Transfers the VHD of a running instance to another host, then shuts off the instance copies over the COW disk""" - self._vmops.transfer_disk(instance, dest) + self._vmops.migrate_disk_and_power_off(instance, dest) - def move_disk(self, instance_ref): + def attach_disk(self, instance): """Moves the copied VDIs into the SR""" - pass + self._vmops.attach_disk(instance) def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index aadacce57..817269769 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -138,8 +138,8 @@ def get_sr_path(session): return sr_path -#TODO(sirp): both objectstore and glance need this, should this be refactored -#into common lib +#TODO(sirp): objectstore, migration and glance need this, should this be +# refactored into common lib def find_sr(session): host = get_this_host(session) srs = session.xenapi.SR.get_all() diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index bd46e1c0b..e81b18a5e 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -20,16 +20,79 @@ XenAPI Plugin for transfering data between host nodes """ +import os import os.path import pickle +import shutil import subprocess +import uuid import XenAPIPlugin +from pluginlib_nova import * + SSH_HOSTS = '/root/.ssh/known_hosts' DEVNULL = '/dev/null' KEYSCAN = '/usr/bin/ssh-keyscan' RSYNC = '/usr/bin/rsync' +FILE_SR_PATH = '/var/run/sr-mount' +IMAGE_PATH = '/images/' +VHD_UTIL = '/usr/sbin/vhd-util' + +def get_sr_path(session): + sr_ref = find_sr(session) + + if sr_ref is None: + raise Exception('Cannot find SR to read VDI from') + + sr_rec = session.xenapi.SR.get_record(sr_ref) + sr_uuid = sr_rec["uuid"] + sr_path = os.path.join(FILE_SR_PATH, sr_uuid) + return sr_path + +def find_sr(session): + host = get_this_host(session) + srs = session.xenapi.SR.get_all() + for sr in srs: + sr_rec = session.xenapi.SR.get_record(sr) + if not ('i18n-key' in sr_rec['other_config'] and + sr_rec['other_config']['i18n-key'] == 'local-storage'): + continue + for pbd in sr_rec['PBDs']: + pbd_rec = session.xenapi.PBD.get_record(pbd) + if pbd_rec['host'] == host: + return sr + return None + +def move_vhds_into_sr(session, args): + """Moves the VHDs from their copied location to the SR""" + params = pickle.dumps(args) + instance_id = params['instance_id'] + + sr_path = get_sr_path(session) + + # Discover the copied VHDs locally, and then set up paths to copy + # them to under the SR + source_image_path = "%s/instance%d" % (IMAGE_PATH, instance_id) + source_base_copy_path = "%s/base_copy.vhd" % source_image_path + source_cow_path = "%s/cow.vhd" % source_image_path + + temp_vhd_path = "%s/instance%d/" % (sr_path, instance_id) + new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, str(uuid.uuid4())) + new_cow_path = "%s/%s.vhd" % (temp_vhd_path, str(uuid.uuid4())) + + os.mkdir(temp_vhd_path) + shutil.move(source_base_copy_path, new_base_copy_path) + shutil.move(source_cow_path, new_cow_path) + + os.rmdir(source_image_path) + + # Link the COW to the base copy + subprocess.call([VHD_UTIL, 'modify', '-n', new_cow_path, '-p', + new_base_copy_path]) + + shutil.move("%s/*.vhd" % temp_vhd_path, sr_path) + os.rmdir(temp_vhd_path) def transfer_vhd(session, args): @@ -38,11 +101,13 @@ def transfer_vhd(session, args): instance_id = params['instance_id'] host = params['host'] vdi_uuid = params['vdi_uuid'] + dest_name = params['dest_name'] sr_path = get_sr_path(session) vhd_path = "%s.vhd" % vdi_uuid source_path = "%s/%s" % (sr_path, vhd_path) - dest_path = '%s:/images/instance%d/' % (host, instance_id) + dest_path = '%s:%sinstance%d/%s' % (host, IMAGE_PATH, instance_id, + dest_name) rsync_args = [['nohup', RSYNC, '-av', '--progress', '-e "ssh -o StrictHostKeyChecking=no"', source_path, dest_path] -- cgit From d8a7a76cd4fd22a6ad9fc1a7b879a8dbffcede5f Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Feb 2011 13:42:57 -0600 Subject: Some more cleanup --- nova/compute/manager.py | 7 ++++--- nova/virt/xenapi/vmops.py | 3 ++- nova/virt/xenapi_conn.py | 5 +---- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 3 ++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ac09f7c8c..54c3412f4 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -429,7 +429,8 @@ class ComputeManager(manager.Manager): self.db.migration_update(context, migration_id, { 'status': 'post-migrating', }) - # This is where we would update the VM record after resizing + #TODO(mdietz): This is where we would update the VM record + #after resizing service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_host'], FLAGS.compute_topic) @@ -449,8 +450,8 @@ class ComputeManager(manager.Manager): migration_ref['instance_id']) # this may get passed into the following spawn instead - self.driver.attach_disk(context, instance_ref) - self.driver.spawn(context, instance_ref, preexisting=True) + disk_info = self.driver.attach_disk(context, instance_ref) + self.driver.spawn(context, instance_ref, disk_info=disk_info) self.db.migration_update(context, migration_id, {'status': 'finished', }) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ea4b7899b..7d88876e4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -106,7 +106,8 @@ class VMOps(object): instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, pv_kernel) - VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) + VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, vdi_ref=vdi_ref, + userdevice=0, bootable=True) if network_ref: VMHelper.create_vif(self._session, vm_ref, diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index cc43050b4..8c756a7e3 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -152,7 +152,7 @@ class XenAPIConnection(object): """List VM instances""" return self._vmops.list_instances() - def spawn(self, instance): + def spawn(self, instance, disk_info=None): """Create VM instance""" self._vmops.spawn(instance) @@ -164,9 +164,6 @@ class XenAPIConnection(object): """Resize a VM instance""" raise NotImplementedError() - def attach_disk(self, instance_ref): - - def reboot(self, instance): """Reboot VM instance""" self._vmops.reboot(instance) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index e81b18a5e..0fb7b5806 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -70,6 +70,7 @@ def move_vhds_into_sr(session, args): instance_id = params['instance_id'] sr_path = get_sr_path(session) + sr_temp_path = "%s/images/" % sr_path # Discover the copied VHDs locally, and then set up paths to copy # them to under the SR @@ -77,7 +78,7 @@ def move_vhds_into_sr(session, args): source_base_copy_path = "%s/base_copy.vhd" % source_image_path source_cow_path = "%s/cow.vhd" % source_image_path - temp_vhd_path = "%s/instance%d/" % (sr_path, instance_id) + temp_vhd_path = "%s/instance%d/" % (sr_temp_path, instance_id) new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, str(uuid.uuid4())) new_cow_path = "%s/%s.vhd" % (temp_vhd_path, str(uuid.uuid4())) -- cgit From a6ce3b777221690df17137e70d6b7bf35ad10b02 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Feb 2011 13:59:54 -0600 Subject: Spawn from disk --- nova/compute/manager.py | 2 +- nova/virt/xenapi/vmops.py | 47 ++++++++++++++++++++++++++--------------------- nova/virt/xenapi_conn.py | 4 ++-- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 54c3412f4..5f7d070af 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -451,7 +451,7 @@ class ComputeManager(manager.Manager): # this may get passed into the following spawn instead disk_info = self.driver.attach_disk(context, instance_ref) - self.driver.spawn(context, instance_ref, disk_info=disk_info) + self.driver.spawn(context, instance_ref, disk=disk_info) self.db.migration_update(context, migration_id, {'status': 'finished', }) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 7d88876e4..ad46bb40d 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -69,7 +69,7 @@ class VMOps(object): LOG.debug(_("Starting instance %s"), instance.name) self._session.call_xenapi('VM.start', vm, False, False) - def spawn(self, instance): + def spawn(self, instance, disk): """Create VM instance""" vm = VMHelper.lookup(self._session, instance.name) if vm is not None: @@ -83,27 +83,32 @@ class VMOps(object): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - #if kernel is not present we must download a raw disk - if instance.kernel_id: - disk_image_type = ImageType.DISK + + vdi_ref = kernel = ramdisk = pv_kernel = None + + # Are we building from a pre-existing disk? + if not disk: + #if kernel is not present we must download a raw disk + if instance.kernel_id: + disk_image_type = ImageType.DISK + else: + disk_image_type = ImageType.DISK_RAW + vdi_uuid = VMHelper.fetch_image(self._session, instance.id, + instance.image_id, user, project, disk_image_type) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) + #Have a look at the VDI and see if it has a PV kernel + if not instance.kernel_id: + pv_kernel = VMHelper.lookup_image(self._session, instance.id, + vdi_ref) + if instance.kernel_id: + kernel = VMHelper.fetch_image(self._session, instance.id, + instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) + if instance.ramdisk_id: + ramdisk = VMHelper.fetch_image(self._session, instance.id, + instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) else: - disk_image_type = ImageType.DISK_RAW - vdi_uuid = VMHelper.fetch_image(self._session, instance.id, - instance.image_id, user, project, disk_image_type) - vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - #Have a look at the VDI and see if it has a PV kernel - pv_kernel = False - if not instance.kernel_id: - pv_kernel = VMHelper.lookup_image(self._session, instance.id, - vdi_ref) - kernel = None - if instance.kernel_id: - kernel = VMHelper.fetch_image(self._session, instance.id, - instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) - ramdisk = None - if instance.ramdisk_id: - ramdisk = VMHelper.fetch_image(self._session, instance.id, - instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', disk) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, pv_kernel) VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, vdi_ref=vdi_ref, diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 8c756a7e3..2fddb8c7f 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -152,9 +152,9 @@ class XenAPIConnection(object): """List VM instances""" return self._vmops.list_instances() - def spawn(self, instance, disk_info=None): + def spawn(self, instance, disk=None): """Create VM instance""" - self._vmops.spawn(instance) + self._vmops.spawn(instance, disk) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ -- cgit From a70ac6609713f2b610923a7ae382208f4d46b74a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Feb 2011 15:01:38 -0600 Subject: Typo fixes and some stupidity about the models --- nova/api/openstack/servers.py | 11 +++++------ nova/compute/manager.py | 2 +- nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py | 4 ++++ nova/virt/xenapi/vmops.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 61dd3be36..06a40e92c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -207,27 +207,26 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotImplemented()) def _action_confirm_resize(self, input_dict, req, id): - return fault.Fault(exc.HTTPNotImplemented()) + return faults.Fault(exc.HTTPNotImplemented()) def _action_revert_resize(self, input_dict, req, id): - return fault.Fault(exc.HTTPNotImplemented()) + return faults.Fault(exc.HTTPNotImplemented()) def _action_rebuild(self, input_dict, req, id): - return fault.Fault(exc.HTTPNotImplemented()) + return faults.Fault(exc.HTTPNotImplemented()) def _action_resize(self, input_dict, req, id): """ Resizes a given instance to the flavor size requested """ try: - resize_flavor = input_dict['resize']['flavorId'] + flavor_id = input_dict['resize']['flavorId'] self.compute_api.resize(req.environ['nova.context'], id, flavor_id) except: return faults.Fault(exc.HTTPUnprocessableEntity()) - return fault.Fault(exc.HTTPAccepted()) + return faults.Fault(exc.HTTPAccepted()) def _action_reboot(self, input_dict, req, id): - #TODO(sandy): rebuild/resize not supported. try: reboot_type = input_dict['reboot']['type'] except Exception: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 5f7d070af..9e8361563 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -401,7 +401,7 @@ class ComputeManager(manager.Manager): """Initiates the process of moving a running instance to another host, possibly changing the RAM and disk size in the process""" context = context.elevated() - instance_ref = self.db.instance_get(context. instance_id) + instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_create(context, { 'instance_id': instance_id, 'source_host': instance_ref['host'], diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py index 4d01cd874..499465fce 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -27,6 +27,10 @@ meta = MetaData() # migrations = Table('migrations', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), Column('id', Integer(), primary_key=True, nullable=False), Column('source_host', String(255)), Column('dest_host', String(255)), diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ad46bb40d..3b14390b4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -152,7 +152,7 @@ class VMOps(object): """ vm = None try: - if instance_or_vm.startswith("OpaqueRef:") + if instance_or_vm.startswith("OpaqueRef:"): # Got passed an opaque ref; return it return instance_or_vm else: -- cgit From 3fc68b805bb5326ef4fa2b8a51a58862ec23a6a4 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Feb 2011 15:04:06 -0600 Subject: Forgot the metadata includes --- nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py index 499465fce..38b711775 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License.from sqlalchemy import * +from sqlalchemy import * from migrate import * from nova import log as logging -- cgit From 68b7ae27036e1a9b16ceb835c5dc6b934e3b964a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Feb 2011 15:06:27 -0600 Subject: Forgot the metadata includes --- nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py index 38b711775..02d9177bd 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -23,6 +23,12 @@ from nova import log as logging meta = MetaData() +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + # # New Tables # -- cgit From 363371ddc6bbe008a536bda06da016385850a98a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 10 Feb 2011 17:20:10 -0600 Subject: Forgot the metadata includes --- nova/db/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/api.py b/nova/db/api.py index 5da0e9840..03232385c 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -86,7 +86,7 @@ def service_get(context, service_id): def service_get_by_host_and_topic(context, host, topic): """Get a service by host it's on and topic it listens to""" - return IMPL.service_get(context, host, topic) + return IMPL.service_get_by_host_and_topic(context, host, topic) def service_get_all(context, disabled=False): -- cgit From 42bd44db235ed2b2fb10e05d70de8d04b0fa869d Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 11:14:51 -0600 Subject: First, not all --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 98be39506..a6be844f8 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -164,7 +164,7 @@ def service_get_by_host_and_topic(context, host, topic): filter_by(disabled=False).\ filter_by(host=host).\ filter_by(topic=topic).\ - all() + first() @require_admin_context def service_get_all_by_host(context, host): -- cgit From f181051ac04084f2937438b61c988804fc2ef845 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 17:39:04 -0600 Subject: Cast to host --- nova/compute/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 9e8361563..63632b538 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -411,7 +411,7 @@ class ComputeManager(manager.Manager): service = self.db.service_get_by_host_and_topic(context, instance_ref['host'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, - service['id']) + service['host']) rpc.cast(context, topic, { 'method': 'resize_instance', 'migration_id': migration_ref['id'], }) @@ -435,7 +435,7 @@ class ComputeManager(manager.Manager): service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_host'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, - service['id']) + service['host']) rpc.cast(context, topic, { 'method': 'finish_resize', 'migration_id': migration_ref['id'], }) -- cgit From 66365ece306023c1cf848d452d5af2c418e4e14c Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 18:04:00 -0600 Subject: More typos --- nova/compute/manager.py | 12 ++++++++++-- nova/db/sqlalchemy/api.py | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 63632b538..9aa163b73 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -414,7 +414,11 @@ class ComputeManager(manager.Manager): service['host']) rpc.cast(context, topic, { 'method': 'resize_instance', - 'migration_id': migration_ref['id'], }) + 'args': { + 'migration_id': migration_ref['id'], + 'instance_id': instance_id, + }, + }) @exception.wrap_exception @checks_instance_lock @@ -438,7 +442,11 @@ class ComputeManager(manager.Manager): service['host']) rpc.cast(context, topic, { 'method': 'finish_resize', - 'migration_id': migration_ref['id'], }) + 'args': { + 'migration_id': migration_ref['id'], + 'instance_id': instance_id, + }, + }) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index a6be844f8..af343bc56 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1941,7 +1941,7 @@ def migration_update(context, migration_id, values): @require_admin_context def migration_get(context, migration_id): session = get_session() - result = session.query(models.Migration.\ + result = session.query(models.Migration).\ filter_by(migration_id=migration_id)).first() if not result: raise exception.NotFound(_("No migration found with id %s") @@ -1952,7 +1952,7 @@ def migration_get(context, migration_id): @require_admin_context def migration_get_by_instance(context, instance_id): session = get_session() - result = session.query(models.Migration.\ + result = session.query(models.Migration).\ filter_by(instance_id=instance_id)).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") -- cgit From 384a5aff50926784590ad66b92919b4d0408319d Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 18:05:02 -0600 Subject: More typos --- nova/db/sqlalchemy/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index af343bc56..6d52790a5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1942,7 +1942,7 @@ def migration_update(context, migration_id, values): def migration_get(context, migration_id): session = get_session() result = session.query(models.Migration).\ - filter_by(migration_id=migration_id)).first() + filter_by(migration_id=migration_id).first() if not result: raise exception.NotFound(_("No migration found with id %s") % migration_id) @@ -1953,7 +1953,7 @@ def migration_get(context, migration_id): def migration_get_by_instance(context, instance_id): session = get_session() result = session.query(models.Migration).\ - filter_by(instance_id=instance_id)).first() + filter_by(instance_id=instance_id).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") % migration_id) -- cgit From f3b25fc06e3eff6f1b0e8fed4a0bf90612bf0230 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 18:07:34 -0600 Subject: More typos --- nova/compute/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 9aa163b73..7c9e918fb 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -422,7 +422,7 @@ class ComputeManager(manager.Manager): @exception.wrap_exception @checks_instance_lock - def resize_instance(self, context, migration_id): + def resize_instance(self, context, instance_id, migration_id): """Starts the migration of a running instance to another host""" migration_ref = self.db.migration_get(context, migration_id) self.db.migration_update(context, migration_id, @@ -443,14 +443,14 @@ class ComputeManager(manager.Manager): rpc.cast(context, topic, { 'method': 'finish_resize', 'args': { - 'migration_id': migration_ref['id'], + 'migration_id': migration_id, 'instance_id': instance_id, }, }) @exception.wrap_exception @checks_instance_lock - def finish_resize(self, context, migration_id): + def finish_resize(self, context, instance_id, migration_id): """Completes the migration process by setting up the newly transferred disk and turning on the instance on its new host machine""" migration_ref = self.db.migration_get(context, migration_id) -- cgit From 520b1b50bc2b1d039ad2f89d791bba21b7a35f05 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 18:09:11 -0600 Subject: More typos --- nova/db/sqlalchemy/api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 6d52790a5..8566bb91f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1930,19 +1930,19 @@ def migration_create(context, values): @require_admin_context -def migration_update(context, migration_id, values): +def migration_update(context, id, values): session = get_session() with session.begin(): - migration = migration_get(context, migration_id, session=session) + migration = migration_get(context, id, session=session) migration.update(values) return migration @require_admin_context -def migration_get(context, migration_id): +def migration_get(context, id): session = get_session() result = session.query(models.Migration).\ - filter_by(migration_id=migration_id).first() + filter_by(id=id).first() if not result: raise exception.NotFound(_("No migration found with id %s") % migration_id) -- cgit From 252ebfe9a039fb883e3e88eda8feafae037e750e Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 11 Feb 2011 18:12:18 -0600 Subject: More typos --- nova/db/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/db/api.py b/nova/db/api.py index 03232385c..887f57885 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -259,6 +259,9 @@ def floating_ip_get_by_address(context, address): #################### +def migration_update(context, id, values): + """Update a migration instance""" + return IMPL.migration_update(context, id, values) def migration_create(context, values): """Create a migration record""" -- cgit From 875c4e1bab5e364a23695e46df69f1b21d9a8200 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 12:44:07 -0600 Subject: Derp --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 8566bb91f..861d13716 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1933,7 +1933,7 @@ def migration_create(context, values): def migration_update(context, id, values): session = get_session() with session.begin(): - migration = migration_get(context, id, session=session) + migration = migration_get(context, id) migration.update(values) return migration -- cgit From 4f23e417bb5ac3db8ac28dfb4b032a3e233c9821 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 12:50:54 -0600 Subject: More fixes --- nova/compute/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 7c9e918fb..3fd37d831 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -425,10 +425,11 @@ class ComputeManager(manager.Manager): def resize_instance(self, context, instance_id, migration_id): """Starts the migration of a running instance to another host""" migration_ref = self.db.migration_get(context, migration_id) + instance_ref = self.db.instance_get(context, instance_id) self.db.migration_update(context, migration_id, { 'status': 'migrating', }) - self.driver.migrate_disk_and_power_off(context, instance, + self.driver.migrate_disk_and_power_off(instance_ref, migration_ref['dest_host']) self.db.migration_update(context, migration_id, -- cgit From 1631196f3f277608fb0569c7242a7d8391605d0d Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 13:38:05 -0600 Subject: wharrgarbl --- nova/virt/xenapi/vmops.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3b14390b4..d5b2c821c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -230,7 +230,7 @@ class VMOps(object): label = "%s-snapshot" % instance.name try: - _, template_vdi_uuids = VMHelper.create_snapshot( + vdi_ref, template_vdi_uuids = VMHelper.create_snapshot( self._session, instance.id, vm_ref, label) return Snapshot(self, instance, template_vdi_uuids) except self.XenAPI.Failure, exc: @@ -262,22 +262,21 @@ class VMOps(object): # Now power down the instance and transfer the COW VHD self._shutdown(instance, method='clean') - _, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], 'dest_name': 'cow.vhd'} self._session.async_call_plugin('data_transfer', 'transfer_vhd', {'params': pickle.dumps(params)}) return snapshot.vdi_uuids[1], vm_vdi_rec['uuid'] - def attach_disk(self, instance): + def attach_disk(self, instance):hh vm_ref = VMHelper.lookup(self._session, instance.name) params = { 'instance_id': instance.id } self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) - - _, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) VMHelper.scan_sr(self._session, instance.id, vm_vdi_rec['SR']) def resize(self, instance, flavor): -- cgit From 9f22390532332b955cb8d78ebfd8cf9670a63ac8 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 13:50:06 -0600 Subject: Snapshot correctly --- nova/virt/xenapi/vmops.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d5b2c821c..fc5cd84e1 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -211,10 +211,11 @@ class VMOps(object): def _get_snapshot(self, instance): class Snapshot(object): - def __init__(self, virt, instance, vdis): + def __init__(self, virt, instance, vm_ref, vdis): self.instance = instance self.vdi_uuids = vdis self.virt = virt + self.vm_ref = vm_ref def __enter__(self): return self @@ -230,9 +231,10 @@ class VMOps(object): label = "%s-snapshot" % instance.name try: - vdi_ref, template_vdi_uuids = VMHelper.create_snapshot( + template_vm_ref, template_vdi_uuids = VMHelper.create_snapshot( self._session, instance.id, vm_ref, label) - return Snapshot(self, instance, template_vdi_uuids) + return Snapshot(self, instance, template_vm_ref, + template_vdi_uuids) except self.XenAPI.Failure, exc: logging.error(_("Unable to Snapshot %(vm_ref)s: %(exc)s") % locals()) @@ -269,7 +271,7 @@ class VMOps(object): {'params': pickle.dumps(params)}) return snapshot.vdi_uuids[1], vm_vdi_rec['uuid'] - def attach_disk(self, instance):hh + def attach_disk(self, instance): vm_ref = VMHelper.lookup(self._session, instance.name) params = { 'instance_id': instance.id } -- cgit From fc8394a80b28f94561aa9ebf94c067ce2d1efd3b Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 14:25:00 -0600 Subject: Snapshot correctly --- nova/virt/xenapi/vmops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index fc5cd84e1..a327f1d36 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -22,6 +22,7 @@ Management class for VM-related functions (spawn, reboot, etc). import json import M2Crypto import os +import pickle import subprocess import tempfile import uuid @@ -262,7 +263,7 @@ class VMOps(object): {'params': pickle.dumps(params)}) # Now power down the instance and transfer the COW VHD - self._shutdown(instance, method='clean') + self._shutdown(instance, snapshot.vm_ref, method='clean') vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], -- cgit From 411d828fc3511a09420e579ceee65a9470242509 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 14:40:58 -0600 Subject: hurr --- nova/virt/xenapi/vm_utils.py | 38 +++++++++++++++++++++----------------- nova/virt/xenapi/vmops.py | 8 +++++--- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index e16662aad..2c4ae6aa2 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -238,6 +238,24 @@ class VMHelper(HelperBase): % locals()) return vdi_ref + @classmethod + def get_vdi_for_vm_safely(cls, session, vm_ref): + vdi_refs = VMHelper.lookup_vm_vdis(session, vm_ref) + if vdi_refs is None: + raise Exception(_("No VDIs found for VM %s") % vm_ref) + else: + num_vdis = len(vdi_refs) + if num_vdis != 1: + raise Exception(_("Unexpected number of VDIs (%(num_vdis)s) found" + " for VM %(vm_ref)s") % locals()) + + vdi_ref = vdi_refs[0] + vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) + return vdi_ref, vdi_rec + + + + @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): """ Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI, @@ -248,7 +266,7 @@ class VMHelper(HelperBase): LOG.debug(_("Snapshotting VM %(vm_ref)s with label '%(label)s'...") % locals()) - vm_vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + vm_vdi_ref, vm_vdi_rec = self.get_vdi_for_vm_safely(session, vm_ref) vm_vdi_uuid = vm_vdi_rec["uuid"] sr_ref = vm_vdi_rec["SR"] @@ -256,7 +274,8 @@ class VMHelper(HelperBase): task = session.call_xenapi('Async.VM.snapshot', vm_ref, label) template_vm_ref = session.wait_for_task(instance_id, task) - template_vdi_rec = get_vdi_for_vm_safely(session, template_vm_ref)[1] + template_vdi_rec = self.get_vdi_for_vm_safely(session, + template_vm_ref)[1] template_vdi_uuid = template_vdi_rec["uuid"] LOG.debug(_('Created snapshot %(template_vm_ref)s from' @@ -568,21 +587,6 @@ def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, return parent_uuid -def get_vdi_for_vm_safely(session, vm_ref): - vdi_refs = VMHelper.lookup_vm_vdis(session, vm_ref) - if vdi_refs is None: - raise Exception(_("No VDIs found for VM %s") % vm_ref) - else: - num_vdis = len(vdi_refs) - if num_vdis != 1: - raise Exception(_("Unexpected number of VDIs (%(num_vdis)s) found" - " for VM %(vm_ref)s") % locals()) - - vdi_ref = vdi_refs[0] - vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) - return vdi_ref, vdi_rec - - def find_sr(session): host = session.get_xenapi_host() srs = session.get_xenapi().SR.get_all() diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index a327f1d36..6a7308c74 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -263,9 +263,10 @@ class VMOps(object): {'params': pickle.dumps(params)}) # Now power down the instance and transfer the COW VHD - self._shutdown(instance, snapshot.vm_ref, method='clean') + self._shutdown(instance, vm_ref, method='clean') - vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + vdi_ref, vm_vdi_rec = \ + VMHelper.get_vdi_for_vm_safely(session, vm_ref) params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], 'dest_name': 'cow.vhd'} self._session.async_call_plugin('data_transfer', 'transfer_vhd', @@ -279,7 +280,8 @@ class VMOps(object): self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) - vdi_ref, vm_vdi_rec = get_vdi_for_vm_safely(session, vm_ref) + vdi_ref, vm_vdi_rec = \ + VMHelper.get_vdi_for_vm_safely(session, vm_ref) VMHelper.scan_sr(self._session, instance.id, vm_vdi_rec['SR']) def resize(self, instance, flavor): -- cgit From 7bb6122549ad5ac549465f0012020f8e5dc9d506 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 15:26:08 -0600 Subject: Some refactoring --- nova/compute/manager.py | 4 ++-- nova/virt/xenapi/vm_utils.py | 12 ++++++++---- nova/virt/xenapi/vmops.py | 13 ++++++++----- nova/virt/xenapi_conn.py | 2 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 8 ++++++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3fd37d831..0b0966324 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -405,7 +405,7 @@ class ComputeManager(manager.Manager): migration_ref = self.db.migration_create(context, { 'instance_id': instance_id, 'source_host': instance_ref['host'], - 'dest_host': socket.gethostbyname(socket.gethostname()), + 'dest_host': socket.gethostname(), 'status': 'pre-migrating' }) LOG.audit(_('instance %s: migrating to '), instance_id, context=context) service = self.db.service_get_by_host_and_topic(context, @@ -459,7 +459,7 @@ class ComputeManager(manager.Manager): migration_ref['instance_id']) # this may get passed into the following spawn instead - disk_info = self.driver.attach_disk(context, instance_ref) + disk_info = self.driver.attach_disk(instance_ref) self.driver.spawn(context, instance_ref, disk=disk_info) self.db.migration_update(context, migration_id, diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 2c4ae6aa2..eeb5502ed 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -266,7 +266,7 @@ class VMHelper(HelperBase): LOG.debug(_("Snapshotting VM %(vm_ref)s with label '%(label)s'...") % locals()) - vm_vdi_ref, vm_vdi_rec = self.get_vdi_for_vm_safely(session, vm_ref) + vm_vdi_ref, vm_vdi_rec = cls.get_vdi_for_vm_safely(session, vm_ref) vm_vdi_uuid = vm_vdi_rec["uuid"] sr_ref = vm_vdi_rec["SR"] @@ -274,7 +274,7 @@ class VMHelper(HelperBase): task = session.call_xenapi('Async.VM.snapshot', vm_ref, label) template_vm_ref = session.wait_for_task(instance_id, task) - template_vdi_rec = self.get_vdi_for_vm_safely(session, + template_vdi_rec = cls.get_vdi_for_vm_safely(session, template_vm_ref)[1] template_vdi_uuid = template_vdi_rec["uuid"] @@ -287,6 +287,12 @@ class VMHelper(HelperBase): #TODO(sirp): we need to assert only one parent, not parents two deep return template_vm_ref, [template_vdi_uuid, parent_uuid] + @classmethod + def get_sr(cls, session, sr_label='slices'): + """ Finds the SR named by the given name label and returns + the UUID """ + return session.call_xenapi('SR.get_by_name_label', sr_label)[0] + @classmethod def upload_image(cls, session, instance_id, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and @@ -504,8 +510,6 @@ class VMHelper(HelperBase): session.wait_for_task(instance_id, task) - - def get_rrd(host, uuid): """Return the VM RRD XML as a string""" try: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6a7308c74..470b6ea8c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -266,7 +266,7 @@ class VMOps(object): self._shutdown(instance, vm_ref, method='clean') vdi_ref, vm_vdi_rec = \ - VMHelper.get_vdi_for_vm_safely(session, vm_ref) + VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], 'dest_name': 'cow.vhd'} self._session.async_call_plugin('data_transfer', 'transfer_vhd', @@ -277,12 +277,15 @@ class VMOps(object): vm_ref = VMHelper.lookup(self._session, instance.name) params = { 'instance_id': instance.id } - self._session.async_call_plugin('migration', 'move_vhds_into_sr', + new_base_copy_uuid, new_cow_uuid = self._session.async_call_plugin( + 'migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) - vdi_ref, vm_vdi_rec = \ - VMHelper.get_vdi_for_vm_safely(session, vm_ref) - VMHelper.scan_sr(self._session, instance.id, vm_vdi_rec['SR']) + # Now we rescan the SR so we find the VHDs + sr_ref = VMHelper.get_sr(self._session) + VMHelper.scan_sr(self._session, instance.id, sr_ref) + + return new_base_copy_uuid def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 2fddb8c7f..6869ce8d8 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -199,7 +199,7 @@ class XenAPIConnection(object): def attach_disk(self, instance): """Moves the copied VDIs into the SR""" - self._vmops.attach_disk(instance) + return self._vmops.attach_disk(instance) def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 0fb7b5806..71d4473c5 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -78,9 +78,12 @@ def move_vhds_into_sr(session, args): source_base_copy_path = "%s/base_copy.vhd" % source_image_path source_cow_path = "%s/cow.vhd" % source_image_path + new_base_copy_uuid = str(uuid.uuid4()) + new_cow_uuid = str(uuid.uuid4()) + temp_vhd_path = "%s/instance%d/" % (sr_temp_path, instance_id) - new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, str(uuid.uuid4())) - new_cow_path = "%s/%s.vhd" % (temp_vhd_path, str(uuid.uuid4())) + new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid) + new_cow_path = "%s/%s.vhd" % (temp_vhd_path, new_cow_uuid) os.mkdir(temp_vhd_path) shutil.move(source_base_copy_path, new_base_copy_path) @@ -94,6 +97,7 @@ def move_vhds_into_sr(session, args): shutil.move("%s/*.vhd" % temp_vhd_path, sr_path) os.rmdir(temp_vhd_path) + return (new_base_copy_uuid, new_cow_uuid) def transfer_vhd(session, args): -- cgit From fad5baf307b74a92fd5b9d8e2d1479f558e180aa Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 15:55:52 -0600 Subject: hurr --- nova/virt/xenapi/vm_utils.py | 12 ++++++++---- nova/virt/xenapi/vmops.py | 13 ++++++++----- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 7 ++++--- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index eeb5502ed..23f9547d7 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -504,10 +504,14 @@ class VMHelper(HelperBase): return {"Unable to retrieve diagnostics": e} @classmethod - def scan_sr(cls, session, instance_id, sr_ref): - LOG.debug(_("Re-scanning SR %s"), sr_ref) - task = session.call_xenapi('Async.SR.scan', sr_ref) - session.wait_for_task(instance_id, task) + def scan_sr(cls, session, instance_id=None, sr_ref=None): + if sr_ref: + LOG.debug(_("Re-scanning SR %s"), sr_ref) + task = session.call_xenapi('Async.SR.scan', sr_ref) + session.wait_for_task(instance_id, task) + else: + sr_ref = cls.get_sr(session) + session.call_xen_api('SR.scan', sr_ref) def get_rrd(host, uuid): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 470b6ea8c..17d42d542 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -276,14 +276,17 @@ class VMOps(object): def attach_disk(self, instance): vm_ref = VMHelper.lookup(self._session, instance.name) - params = { 'instance_id': instance.id } - new_base_copy_uuid, new_cow_uuid = self._session.async_call_plugin( - 'migration', 'move_vhds_into_sr', + new_base_copy_uuid = str(uuid.uuid4()) + + params = { 'instance_id': instance.id, + 'new_base_copy_uuid': new_base_copy_uuid, + 'new_cow_uuid': str(uuid.uuid4() } + + self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) # Now we rescan the SR so we find the VHDs - sr_ref = VMHelper.get_sr(self._session) - VMHelper.scan_sr(self._session, instance.id, sr_ref) + VMHelper.scan_sr(self._session) return new_base_copy_uuid diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 71d4473c5..e73480445 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -69,6 +69,9 @@ def move_vhds_into_sr(session, args): params = pickle.dumps(args) instance_id = params['instance_id'] + new_base_copy_uuid = params['new_base_copy_uuid'] + new_cow_uuid = params['new_cow_uuid'] + sr_path = get_sr_path(session) sr_temp_path = "%s/images/" % sr_path @@ -78,8 +81,6 @@ def move_vhds_into_sr(session, args): source_base_copy_path = "%s/base_copy.vhd" % source_image_path source_cow_path = "%s/cow.vhd" % source_image_path - new_base_copy_uuid = str(uuid.uuid4()) - new_cow_uuid = str(uuid.uuid4()) temp_vhd_path = "%s/instance%d/" % (sr_temp_path, instance_id) new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid) @@ -97,7 +98,7 @@ def move_vhds_into_sr(session, args): shutil.move("%s/*.vhd" % temp_vhd_path, sr_path) os.rmdir(temp_vhd_path) - return (new_base_copy_uuid, new_cow_uuid) + return None def transfer_vhd(session, args): -- cgit From 9a71c79dc3beb554c86a1b1b5d03ab66c6e96edc Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 16:24:51 -0600 Subject: Typo fixes --- nova/compute/manager.py | 2 +- nova/virt/xenapi/vm_utils.py | 2 +- nova/virt/xenapi/vmops.py | 2 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 0b0966324..23d2b80ac 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -460,7 +460,7 @@ class ComputeManager(manager.Manager): # this may get passed into the following spawn instead disk_info = self.driver.attach_disk(instance_ref) - self.driver.spawn(context, instance_ref, disk=disk_info) + self.driver.spawn(instance_ref, disk=disk_info) self.db.migration_update(context, migration_id, {'status': 'finished', }) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 23f9547d7..08064b786 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -511,7 +511,7 @@ class VMHelper(HelperBase): session.wait_for_task(instance_id, task) else: sr_ref = cls.get_sr(session) - session.call_xen_api('SR.scan', sr_ref) + session.call_xenapi('SR.scan', sr_ref) def get_rrd(host, uuid): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 17d42d542..6c6d04dbf 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -280,7 +280,7 @@ class VMOps(object): params = { 'instance_id': instance.id, 'new_base_copy_uuid': new_base_copy_uuid, - 'new_cow_uuid': str(uuid.uuid4() } + 'new_cow_uuid': str(uuid.uuid4()) } self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index e73480445..cf7f378fa 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -25,7 +25,6 @@ import os.path import pickle import shutil import subprocess -import uuid import XenAPIPlugin @@ -114,7 +113,7 @@ def transfer_vhd(session, args): source_path = "%s/%s" % (sr_path, vhd_path) dest_path = '%s:%sinstance%d/%s' % (host, IMAGE_PATH, instance_id, dest_name) - rsync_args = [['nohup', RSYNC, '-av', '--progress', + rsync_args = ['nohup', RSYNC, '-av', '--progress', '-e "ssh -o StrictHostKeyChecking=no"', source_path, dest_path] if subprocess.call(rsync_args) != 0: -- cgit From 0bd48e3d53c6fce04b0c5e483537b3fd31c7364a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 17:24:33 -0600 Subject: bad plugin --- nova/virt/xenapi/vmops.py | 2 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 6c6d04dbf..5ab73d562 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -269,7 +269,7 @@ class VMOps(object): VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], 'dest_name': 'cow.vhd'} - self._session.async_call_plugin('data_transfer', 'transfer_vhd', + self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) return snapshot.vdi_uuids[1], vm_vdi_rec['uuid'] diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index cf7f378fa..c68fc93c5 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -120,4 +120,5 @@ def transfer_vhd(session, args): raise Exception("Unexpected VHD transfer failure") if __name__ == '__main__': - XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd}) + XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd, + 'move_vhd_into_sr':move_vhd_into_sr, }) -- cgit From e44a91ced3d19a3bca10457239592307bf6f829b Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 17:31:20 -0600 Subject: bad plugin --- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index c68fc93c5..c1f5b7528 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -65,7 +65,7 @@ def find_sr(session): def move_vhds_into_sr(session, args): """Moves the VHDs from their copied location to the SR""" - params = pickle.dumps(args) + params = pickle.loads(exists(args, 'params')) instance_id = params['instance_id'] new_base_copy_uuid = params['new_base_copy_uuid'] @@ -102,7 +102,7 @@ def move_vhds_into_sr(session, args): def transfer_vhd(session, args): """Rsyncs a VHD to an adjacent host""" - params = pickle.dumps(args) + params = pickle.loads(exists(args, 'params')) instance_id = params['instance_id'] host = params['host'] vdi_uuid = params['vdi_uuid'] @@ -121,4 +121,4 @@ def transfer_vhd(session, args): if __name__ == '__main__': XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd, - 'move_vhd_into_sr':move_vhd_into_sr, }) + 'move_vhds_into_sr':move_vhds_into_sr, }) -- cgit From b7cf8f233a585043f0aa85f4d26dc2fb5a6701c7 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 17:34:54 -0600 Subject: bad plugin --- nova/virt/xenapi/vmops.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 5ab73d562..ba0db22f1 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -258,7 +258,8 @@ class VMOps(object): #pretty fugly with self._get_snapshot(instance) as snapshot: params = {'host':dest, 'vdi_uuid':snapshot.vdi_uuids[1], - 'dest_name':'base_copy.vhd'} + 'dest_name': 'base_copy.vhd', + 'instance_id': instance.id, } self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) @@ -268,7 +269,8 @@ class VMOps(object): vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], - 'dest_name': 'cow.vhd'} + 'dest_name': 'cow.vhd', + 'instance_id': instance.id, } self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) return snapshot.vdi_uuids[1], vm_vdi_rec['uuid'] -- cgit From 3014c0896202b592858fc1a7fc9c29b92a6f5d1b Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 18:04:07 -0600 Subject: plugin --- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index c1f5b7528..9c56cb379 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -111,14 +111,18 @@ def transfer_vhd(session, args): vhd_path = "%s.vhd" % vdi_uuid source_path = "%s/%s" % (sr_path, vhd_path) - dest_path = '%s:%sinstance%d/%s' % (host, IMAGE_PATH, instance_id, - dest_name) - rsync_args = ['nohup', RSYNC, '-av', '--progress', - '-e "ssh -o StrictHostKeyChecking=no"', source_path, dest_path] + dest_path = '%sinstance%d/' % (IMAGE_PATH, instance_id) + + dest_path_with_vhd="$s:%s/%s" % (host, dest_path, dest_name) + ssh_cmd = '-e "ssh -o StrictHostKeyChecking=no \'mkdir -p %s\' " ' % dest_path + + rsync_args = ['nohup', RSYNC, '-av', '--progress', ssh_cmd, source_path, + dest_path_with_vhd] if subprocess.call(rsync_args) != 0: raise Exception("Unexpected VHD transfer failure") + if __name__ == '__main__': XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd, 'move_vhds_into_sr':move_vhds_into_sr, }) -- cgit From e7fe96453760320ef897b9edfc39e057d565e6c0 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 23:22:37 -0600 Subject: Refactored --- nova/compute/manager.py | 9 ++-- nova/virt/xenapi/vmops.py | 50 ++++++++++++---------- nova/virt/xenapi_conn.py | 2 +- .../xenserver/xenapi/etc/xapi.d/plugins/migration | 17 ++++---- 4 files changed, 43 insertions(+), 35 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 23d2b80ac..2308c8315 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -429,14 +429,15 @@ class ComputeManager(manager.Manager): self.db.migration_update(context, migration_id, { 'status': 'migrating', }) - self.driver.migrate_disk_and_power_off(instance_ref, + disk_info = self.driver.migrate_disk_and_power_off(instance_ref, migration_ref['dest_host']) self.db.migration_update(context, migration_id, { 'status': 'post-migrating', }) + #TODO(mdietz): This is where we would update the VM record #after resizing - + service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_host'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, @@ -451,7 +452,7 @@ class ComputeManager(manager.Manager): @exception.wrap_exception @checks_instance_lock - def finish_resize(self, context, instance_id, migration_id): + def finish_resize(self, context, instance_id, migration_id, disk_info): """Completes the migration process by setting up the newly transferred disk and turning on the instance on its new host machine""" migration_ref = self.db.migration_get(context, migration_id) @@ -459,7 +460,7 @@ class ComputeManager(manager.Manager): migration_ref['instance_id']) # this may get passed into the following spawn instead - disk_info = self.driver.attach_disk(instance_ref) + new_disk_info = self.driver.attach_disk(instance_ref, disk_info) self.driver.spawn(instance_ref, disk=disk_info) self.db.migration_update(context, migration_id, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ba0db22f1..127a09ad1 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -103,17 +103,19 @@ class VMOps(object): vdi_ref) if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, - instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) + instance.kernel_id, user, project, + ImageType.KERNEL_RAMDISK) if instance.ramdisk_id: ramdisk = VMHelper.fetch_image(self._session, instance.id, - instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) + instance.ramdisk_id, user, project, + ImageType.KERNEL_RAMDISK) else: vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', disk) vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, pv_kernel) - VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, vdi_ref=vdi_ref, - userdevice=0, bootable=True) + VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, + vdi_ref=vdi_ref, userdevice=0, bootable=True) if network_ref: VMHelper.create_vif(self._session, vm_ref, @@ -234,7 +236,7 @@ class VMOps(object): try: template_vm_ref, template_vdi_uuids = VMHelper.create_snapshot( self._session, instance.id, vm_ref, label) - return Snapshot(self, instance, template_vm_ref, + return Snapshot(self, instance, template_vm_ref, template_vdi_uuids) except self.XenAPI.Failure, exc: logging.error(_("Unable to Snapshot %(vm_ref)s: %(exc)s") @@ -254,35 +256,40 @@ class VMOps(object): # identify it via the VBD. The base copy is the parent_uuid returned # from the snapshot creation - #TODO(mdietz): explicitly forcing the base_copy and cow names is - #pretty fugly + base_copy_uuid = cow_uuid = None with self._get_snapshot(instance) as snapshot: - params = {'host':dest, 'vdi_uuid':snapshot.vdi_uuids[1], - 'dest_name': 'base_copy.vhd', + # transfer the base copy + base_copy_uuid = snapshot.vdi_uuids[1] + vdi_ref, vm_vdi_rec = \ + VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) + cow_uuid = vm_vdi_rec['uuid'] + + params = {'host': dest, 'vdi_uuid': base_copy_uuid, 'instance_id': instance.id, } + self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) # Now power down the instance and transfer the COW VHD self._shutdown(instance, vm_ref, method='clean') - vdi_ref, vm_vdi_rec = \ - VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) - params = {'host':dest, 'vdi_uuid': vm_vdi_rec['uuid'], - 'dest_name': 'cow.vhd', + params = {'host': dest, 'vdi_uuid': cow_uuid, 'instance_id': instance.id, } self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) - return snapshot.vdi_uuids[1], vm_vdi_rec['uuid'] - def attach_disk(self, instance): - vm_ref = VMHelper.lookup(self._session, instance.name) + # TODO(mdietz): we could also consider renaming these to something + # sensible so we don't need to blindly pass around dictionaries + return {'base_copy': base_copy_uuid, 'cow': cow_uuid} + def attach_disk(self, instance, disk_info): + vm_ref = VMHelper.lookup(self._session, instance.name) new_base_copy_uuid = str(uuid.uuid4()) - - params = { 'instance_id': instance.id, - 'new_base_copy_uuid': new_base_copy_uuid, - 'new_cow_uuid': str(uuid.uuid4()) } + params = {'instance_id': instance.id, + 'old_base_copy_uuid': disk_info['base_copy'], + 'old_cow_uuid': disk_info['cow'], + 'new_base_copy_uuid': new_base_copy_uuid, + 'new_cow_uuid': str(uuid.uuid4())} self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) @@ -290,7 +297,7 @@ class VMOps(object): # Now we rescan the SR so we find the VHDs VMHelper.scan_sr(self._session) - return new_base_copy_uuid + return new_base_copy_uuid def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ @@ -340,7 +347,6 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def _shutdown(self, instance, vm, method='hard'): """Shutdown an instance """ state = self.get_info(instance['name'])['state'] diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 6869ce8d8..21892ca37 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -197,7 +197,7 @@ class XenAPIConnection(object): off the instance copies over the COW disk""" self._vmops.migrate_disk_and_power_off(instance, dest) - def attach_disk(self, instance): + def attach_disk(self, instance, disk_info): """Moves the copied VDIs into the SR""" return self._vmops.attach_disk(instance) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 9c56cb379..3d3ad4e67 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -68,6 +68,9 @@ def move_vhds_into_sr(session, args): params = pickle.loads(exists(args, 'params')) instance_id = params['instance_id'] + old_base_copy_uuid = params['old_base_copy_uuid'] + old_cow_uuid = params['old_cow_uuid'] + new_base_copy_uuid = params['new_base_copy_uuid'] new_cow_uuid = params['new_cow_uuid'] @@ -77,9 +80,9 @@ def move_vhds_into_sr(session, args): # Discover the copied VHDs locally, and then set up paths to copy # them to under the SR source_image_path = "%s/instance%d" % (IMAGE_PATH, instance_id) - source_base_copy_path = "%s/base_copy.vhd" % source_image_path - source_cow_path = "%s/cow.vhd" % source_image_path - + source_base_copy_path = "%s/%s.vhd" % (source_image_path, + old_base_copy_uuid) + source_cow_path = "%s/%s.vhd" % (source_image_path, old_cow_uuid) temp_vhd_path = "%s/instance%d/" % (sr_temp_path, instance_id) new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid) @@ -106,18 +109,16 @@ def transfer_vhd(session, args): instance_id = params['instance_id'] host = params['host'] vdi_uuid = params['vdi_uuid'] - dest_name = params['dest_name'] sr_path = get_sr_path(session) vhd_path = "%s.vhd" % vdi_uuid source_path = "%s/%s" % (sr_path, vhd_path) - dest_path = '%sinstance%d/' % (IMAGE_PATH, instance_id) + dest_path = '%s:%sinstance%d/' % (host, IMAGE_PATH, instance_id) - dest_path_with_vhd="$s:%s/%s" % (host, dest_path, dest_name) - ssh_cmd = '-e "ssh -o StrictHostKeyChecking=no \'mkdir -p %s\' " ' % dest_path + ssh_cmd = '-e "ssh -o StrictHostKeyChecking=no " ' rsync_args = ['nohup', RSYNC, '-av', '--progress', ssh_cmd, source_path, - dest_path_with_vhd] + dest_path] if subprocess.call(rsync_args) != 0: raise Exception("Unexpected VHD transfer failure") -- cgit From 4574bcdfe303a76a46eb7579a5a70de4e54cc926 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 14 Feb 2011 23:58:21 -0600 Subject: Tons o loggin --- nova/compute/manager.py | 1 + nova/virt/xenapi_conn.py | 4 ++-- .../xenserver/xenapi/etc/xapi.d/plugins/migration | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 2308c8315..6a87bb6f1 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -447,6 +447,7 @@ class ComputeManager(manager.Manager): 'args': { 'migration_id': migration_id, 'instance_id': instance_id, + 'disk_info': disk_info, }, }) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 21892ca37..6d40d4615 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -195,11 +195,11 @@ class XenAPIConnection(object): def migrate_disk_and_power_off(self, instance, dest): """Transfers the VHD of a running instance to another host, then shuts off the instance copies over the COW disk""" - self._vmops.migrate_disk_and_power_off(instance, dest) + return self._vmops.migrate_disk_and_power_off(instance, dest) def attach_disk(self, instance, disk_info): """Moves the copied VDIs into the SR""" - return self._vmops.attach_disk(instance) + return self._vmops.attach_disk(instance, disk_info) def suspend(self, instance, callback): """suspend the specified instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 3d3ad4e67..63de5bfba 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -29,6 +29,7 @@ import subprocess import XenAPIPlugin from pluginlib_nova import * +configure_logging('migration') SSH_HOSTS = '/root/.ssh/known_hosts' DEVNULL = '/dev/null' @@ -88,17 +89,27 @@ def move_vhds_into_sr(session, args): new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid) new_cow_path = "%s/%s.vhd" % (temp_vhd_path, new_cow_uuid) + logging.debug('Creating temporary SR path' % temp_vhd_path) os.mkdir(temp_vhd_path) + + logging.debug('Moving %s into %s' % (source_base_copy_path, temp_vhd_path)) shutil.move(source_base_copy_path, new_base_copy_path) + + logging.debug('Moving %s into %s' % (source_cow_path, temp_vhd_path)) shutil.move(source_cow_path, new_cow_path) + logging.debug('Cleaning up %s' % source_image_path) os.rmdir(source_image_path) # Link the COW to the base copy + logging.debug('Attaching COW to the base copy...') subprocess.call([VHD_UTIL, 'modify', '-n', new_cow_path, '-p', new_base_copy_path]) + logging.debug('Moving VHDs into SR %s' % sr_path) shutil.move("%s/*.vhd" % temp_vhd_path, sr_path) + + loggin.debug('Cleaning up temporary SR path %s' % temp_vhd_path) os.rmdir(temp_vhd_path) return None @@ -115,12 +126,19 @@ def transfer_vhd(session, args): source_path = "%s/%s" % (sr_path, vhd_path) dest_path = '%s:%sinstance%d/' % (host, IMAGE_PATH, instance_id) + logging.debug("Preparing to transmit %s to %s" % (source_path, + dest_path)) + ssh_cmd = '-e "ssh -o StrictHostKeyChecking=no " ' rsync_args = ['nohup', RSYNC, '-av', '--progress', ssh_cmd, source_path, dest_path] - if subprocess.call(rsync_args) != 0: + logging.debug('rsync %s' % (' '.join(rsync_args, ))) + + rsync_proc = subprocess.POpen(rsync_args) + logging.debug('Rsync output: \n %s' % rsync_proc.communicate()[0]) + if rsync_proc.returncode != 0 raise Exception("Unexpected VHD transfer failure") -- cgit -- cgit From bf82637cad867b0e8fb6ad868f60c6dcd66d7f97 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 11:05:20 -0600 Subject: Better host acquisition --- nova/compute/manager.py | 7 ++++--- nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py | 3 ++- nova/db/sqlalchemy/models.py | 3 ++- nova/virt/xenapi_conn.py | 4 ++++ plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6a87bb6f1..7e929d715 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -404,12 +404,13 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_create(context, { 'instance_id': instance_id, - 'source_host': instance_ref['host'], - 'dest_host': socket.gethostname(), + 'source_compute': instance_ref['host'], + 'dest_compute': socket.gethostname(), + 'dest_host': self.driver.get_host_ip_addr(), 'status': 'pre-migrating' }) LOG.audit(_('instance %s: migrating to '), instance_id, context=context) service = self.db.service_get_by_host_and_topic(context, - instance_ref['host'], FLAGS.compute_topic) + migration_ref['source_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, service['host']) rpc.cast(context, topic, diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py index 02d9177bd..4aab5bdc6 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py @@ -39,7 +39,8 @@ migrations = Table('migrations', meta, Column('deleted_at', DateTime(timezone=False)), Column('deleted', Boolean(create_constraint=True, name=None)), Column('id', Integer(), primary_key=True, nullable=False), - Column('source_host', String(255)), + Column('source_compute', String(255)), + Column('dest_compute', String(255)), Column('dest_host', String(255)), Column('instance_id', Integer, ForeignKey('instances.id'), nullable=True), diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index ebf3a382b..1c84e15dd 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -370,7 +370,8 @@ class Migration(BASE, NovaBase): """Represents a running host-to-host migration.""" __tablename__ = 'migrations' id = Column(Integer, primary_key=True, nullable=False) - source_host = Column(String(255)) + source_compute = Column(String(255)) + dest_compute = Column(String(255)) dest_host = Column(String(255)) instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) status = Column(String(255)) #TODO(_cerberus_): enum diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 6d40d4615..19b5269f5 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -224,6 +224,10 @@ class XenAPIConnection(object): def get_ajax_console(self, instance): """Return link to instance's ajax console""" return self._vmops.get_ajax_console(instance) + + def get_host_ip_addr(self): + xs_url = urlparse.urlpase(FLAGS.xenapi_connection_url) + return xs_url.netloc def attach_volume(self, instance_name, device_path, mountpoint): """Attach volume storage to VM instance""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 63de5bfba..97c970da5 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -136,7 +136,7 @@ def transfer_vhd(session, args): logging.debug('rsync %s' % (' '.join(rsync_args, ))) - rsync_proc = subprocess.POpen(rsync_args) + rsync_proc = subprocess.Popen(rsync_args) logging.debug('Rsync output: \n %s' % rsync_proc.communicate()[0]) if rsync_proc.returncode != 0 raise Exception("Unexpected VHD transfer failure") -- cgit From 03a8d1baae00a4150a02ac2f0b04c413dd3b00e0 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 11:19:27 -0600 Subject: derp --- nova/compute/manager.py | 2 +- nova/virt/xenapi_conn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 7e929d715..546d07a09 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -440,7 +440,7 @@ class ComputeManager(manager.Manager): #after resizing service = self.db.service_get_by_host_and_topic(context, - migration_ref['dest_host'], FLAGS.compute_topic) + migration_ref['dest_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, service['host']) rpc.cast(context, topic, diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 19b5269f5..2671f1a7b 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -226,7 +226,7 @@ class XenAPIConnection(object): return self._vmops.get_ajax_console(instance) def get_host_ip_addr(self): - xs_url = urlparse.urlpase(FLAGS.xenapi_connection_url) + xs_url = urlparse.urlparse(FLAGS.xenapi_connection_url) return xs_url.netloc def attach_volume(self, instance_name, device_path, mountpoint): -- cgit From 0020f14f43aa6f024d9aab7dc67c79caaaeb8257 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 15 Feb 2011 11:15:59 -0800 Subject: zone/info works --- bin/nova-combined | 4 ++-- nova/api/openstack/__init__.py | 6 +++--- nova/api/openstack/zones.py | 7 ++++++- nova/flags.py | 5 +++++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bin/nova-combined b/bin/nova-combined index 913c866bf..a0f552d64 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -53,11 +53,11 @@ if __name__ == '__main__': compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') - volume = service.Service.create(binary='nova-volume') + #volume = service.Service.create(binary='nova-volume') scheduler = service.Service.create(binary='nova-scheduler') #objectstore = service.Service.create(binary='nova-objectstore') - service.serve(compute, network, volume, scheduler) + service.serve(compute, network, scheduler) apps = [] paste_config_file = wsgi.paste_config_file('nova-api.conf') diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 33d040ab3..95fce7f84 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -76,13 +76,13 @@ class APIRouter(wsgi.Router): LOG.debug(_("Including admin operations in API.")) server_members['pause'] = 'POST' server_members['unpause'] = 'POST' - server_members["diagnostics"] = "GET" - server_members["actions"] = "GET" + server_members['diagnostics'] = 'GET' + server_members['actions'] = 'GET' server_members['suspend'] = 'POST' server_members['resume'] = 'POST' mapper.resource("zone", "zones", controller=zones.Controller(), - collection={'detail': 'GET'}) + collection={'detail': 'GET', 'info': 'GET'}), mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 830464ffd..16e5e366b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -42,7 +42,7 @@ class Controller(wsgi.Controller): _serialization_metadata = { 'application/xml': { "attributes": { - "zone": ["id", "api_url"]}}} + "zone": ["id", "api_url", "name", "capabilities"]}}} def index(self, req): """Return all zones in brief""" @@ -55,6 +55,11 @@ class Controller(wsgi.Controller): """Return all zones in detail""" return self.index(req) + def info(self, req): + """Return name and capabilities for this zone.""" + return dict(zone=dict(name=FLAGS.zone_name, + capabilities=FLAGS.zone_capabilities)) + def show(self, req, id): """Return data about the given zone id""" zone_id = int(id) diff --git a/nova/flags.py b/nova/flags.py index 3ba3fe6fa..0a45499f3 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -312,3 +312,8 @@ DEFINE_string('host', socket.gethostname(), DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') + +DEFINE_string('zone_name', 'nova', 'name of this zone') +DEFINE_string('zone_capabilities', 'xen, linux', + 'comma-delimited list of tags which represent boolean' + ' capabilities of this zone') -- cgit From c97d408842a4a5a8e9d379acc13c9c1f5871827f Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 14:10:43 -0600 Subject: Plugin changes --- nova/virt/xenapi/vmops.py | 11 +++++++---- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 12 +++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 127a09ad1..882d52f38 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -267,16 +267,18 @@ class VMOps(object): params = {'host': dest, 'vdi_uuid': base_copy_uuid, 'instance_id': instance.id, } - self._session.async_call_plugin('migration', 'transfer_vhd', + task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) + self._session.wait_for_task(instance.id, task) # Now power down the instance and transfer the COW VHD self._shutdown(instance, vm_ref, method='clean') params = {'host': dest, 'vdi_uuid': cow_uuid, 'instance_id': instance.id, } - self._session.async_call_plugin('migration', 'transfer_vhd', + task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) + self._session.wait_for_task(instance.id, task) # TODO(mdietz): we could also consider renaming these to something # sensible so we don't need to blindly pass around dictionaries @@ -291,8 +293,9 @@ class VMOps(object): 'new_base_copy_uuid': new_base_copy_uuid, 'new_cow_uuid': str(uuid.uuid4())} - self._session.async_call_plugin('migration', 'move_vhds_into_sr', - {'params': pickle.dumps(params)}) + task = self._session.async_call_plugin('migration', + 'move_vhds_into_sr', {'params': pickle.dumps(params)}) + self._session.wait_for_task(instance.id, task) # Now we rescan the SR so we find the VHDs VMHelper.scan_sr(self._session) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 97c970da5..4a4ed0e73 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -129,17 +129,19 @@ def transfer_vhd(session, args): logging.debug("Preparing to transmit %s to %s" % (source_path, dest_path)) - ssh_cmd = '-e "ssh -o StrictHostKeyChecking=no " ' + ssh_cmd = '-e \'ssh -o StrictHostKeyChecking=no\' ' - rsync_args = ['nohup', RSYNC, '-av', '--progress', ssh_cmd, source_path, - dest_path] + rsync_args = ['nohup > /root/instance%d_progress' % instance_id, RSYNC, + '-av', '--progress', ssh_cmd, source_path, dest_path] logging.debug('rsync %s' % (' '.join(rsync_args, ))) - rsync_proc = subprocess.Popen(rsync_args) + rsync_proc = subprocess.Popen(rsync_args, stdout=subprocess.PIPE) logging.debug('Rsync output: \n %s' % rsync_proc.communicate()[0]) - if rsync_proc.returncode != 0 + logging.debug('Rsync return: %d' % rsync_proc.returncode) + if rsync_proc.returncode != 0: raise Exception("Unexpected VHD transfer failure") + return "" if __name__ == '__main__': -- cgit From 00f2905a5debc5835b742dab8dce003f53e33fc2 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 14:29:31 -0600 Subject: plugin lol --- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 4a4ed0e73..bfc2a2ed4 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -89,7 +89,7 @@ def move_vhds_into_sr(session, args): new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid) new_cow_path = "%s/%s.vhd" % (temp_vhd_path, new_cow_uuid) - logging.debug('Creating temporary SR path' % temp_vhd_path) + logging.debug('Creating temporary SR path %s' % temp_vhd_path) os.mkdir(temp_vhd_path) logging.debug('Moving %s into %s' % (source_base_copy_path, temp_vhd_path)) @@ -129,10 +129,10 @@ def transfer_vhd(session, args): logging.debug("Preparing to transmit %s to %s" % (source_path, dest_path)) - ssh_cmd = '-e \'ssh -o StrictHostKeyChecking=no\' ' + ssh_cmd = 'ssh -o StrictHostKeyChecking=no' - rsync_args = ['nohup > /root/instance%d_progress' % instance_id, RSYNC, - '-av', '--progress', ssh_cmd, source_path, dest_path] + rsync_args = ['nohup', RSYNC, '-av', '--progress', '-e', ssh_cmd, + source_path, dest_path] logging.debug('rsync %s' % (' '.join(rsync_args, ))) -- cgit From 9f77e0a46cac2ebaf9a18c4a175099b208db1adb Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 15:27:23 -0600 Subject: More plugin lol --- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index bfc2a2ed4..cc72b5d26 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -90,7 +90,7 @@ def move_vhds_into_sr(session, args): new_cow_path = "%s/%s.vhd" % (temp_vhd_path, new_cow_uuid) logging.debug('Creating temporary SR path %s' % temp_vhd_path) - os.mkdir(temp_vhd_path) + os.makedirs(temp_vhd_path) logging.debug('Moving %s into %s' % (source_base_copy_path, temp_vhd_path)) shutil.move(source_base_copy_path, new_base_copy_path) @@ -107,11 +107,12 @@ def move_vhds_into_sr(session, args): new_base_copy_path]) logging.debug('Moving VHDs into SR %s' % sr_path) - shutil.move("%s/*.vhd" % temp_vhd_path, sr_path) + shutil.move("%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid), sr_path) + shutil.move("%s/%s.vhd" % (temp_vhd_path, new_cow_uuid), sr_path) - loggin.debug('Cleaning up temporary SR path %s' % temp_vhd_path) + logginig.debug('Cleaning up temporary SR path %s' % temp_vhd_path) os.rmdir(temp_vhd_path) - return None + return "" def transfer_vhd(session, args): -- cgit From 5afc1f05d303cd58fb0bf94e5d35e5bc28b9d75c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 15 Feb 2011 13:40:54 -0800 Subject: polling working --- nova/scheduler/manager.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index e9b47512e..00cab60cf 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -22,6 +22,9 @@ Scheduler Service """ import functools +import novatools + +from datetime import datetime from nova import db from nova import flags @@ -35,20 +38,72 @@ FLAGS = flags.FLAGS flags.DEFINE_string('scheduler_driver', 'nova.scheduler.chance.ChanceScheduler', 'Driver to use for the scheduler') +flags.DEFINE_integer('zone_db_check_interval', + 60, + 'Seconds between getting fresh zone info from db.') + + +class ZoneState(object): + """Holds the state of all connected child zones.""" + def __init__(self): + self.is_active = True + self.name = None + self.capabilities = None + self.retry = 0 + self.last_seen = datetime.min + + def update(self, zone): + self.zone_id = zone.id + self.api_url = zone.api_url + self.username = zone.username + self.password = zone.password +class ZoneManager(object): + """Keeps the zone states updated.""" + def __init__(self): + self.last_zone_db_check = datetime.min + self.zone_states = {} + + def _refresh_from_db(self, context): + zones = db.zone_get_all(context) + existing = self.zone_states.keys() + for zone in zones: + if zone.id not in existing: + self.zone_state[zone.id] = ZoneState() + self.zone_state[zones.id].update(zone) + + def _poll_zones(self, context): + pass + + def ping(self, context=None): + """Ping should be called periodically to update zone status.""" + logging.debug("ZoneManager PING") + diff = datetime.now() - self.last_zone_db_check + if diff.seconds >= FLAGS.zone_db_check_interval: + logging.debug("ZoneManager RECHECKING DB ") + self.last_zone_db_check = datetime.now() + self._refresh_from_db(context) + self._poll_zones(context) + + class SchedulerManager(manager.Manager): """Chooses a host to run instances on.""" def __init__(self, scheduler_driver=None, *args, **kwargs): if not scheduler_driver: scheduler_driver = FLAGS.scheduler_driver self.driver = utils.import_object(scheduler_driver) + self.zone_manager = ZoneManager() super(SchedulerManager, self).__init__(*args, **kwargs) def __getattr__(self, key): """Converts all method calls to use the schedule method""" return functools.partial(self._schedule, key) + def periodic_tasks(self, context=None): + """Poll child zones periodically to get status.""" + self.zone_manager.ping(context) + def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. -- cgit From f6bf7e8c1e2481e870ed4baa9f2a6aa8001b5514 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 16:46:17 -0600 Subject: fail --- nova/compute/manager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 546d07a09..19e1d9f46 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -402,10 +402,14 @@ class ComputeManager(manager.Manager): host, possibly changing the RAM and disk size in the process""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) + if instance_ref['host'] == FLAGS.host: + raise exception.Error(_( + 'Migration error: destination same as source!')) + migration_ref = self.db.migration_create(context, { 'instance_id': instance_id, 'source_compute': instance_ref['host'], - 'dest_compute': socket.gethostname(), + 'dest_compute': FLAGS.host, 'dest_host': self.driver.get_host_ip_addr(), 'status': 'pre-migrating' }) LOG.audit(_('instance %s: migrating to '), instance_id, context=context) @@ -463,7 +467,7 @@ class ComputeManager(manager.Manager): # this may get passed into the following spawn instead new_disk_info = self.driver.attach_disk(instance_ref, disk_info) - self.driver.spawn(instance_ref, disk=disk_info) + self.driver.spawn(instance_ref, disk=new_disk_info) self.db.migration_update(context, migration_id, {'status': 'finished', }) -- cgit From a6ea6759450aab7eb021e202c68e5301667c74a9 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 17:58:57 -0600 Subject: foo --- nova/virt/xenapi/vmops.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 882d52f38..c2f5ddc41 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -97,21 +97,22 @@ class VMOps(object): vdi_uuid = VMHelper.fetch_image(self._session, instance.id, instance.image_id, user, project, disk_image_type) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - #Have a look at the VDI and see if it has a PV kernel - if not instance.kernel_id: - pv_kernel = VMHelper.lookup_image(self._session, instance.id, - vdi_ref) - if instance.kernel_id: - kernel = VMHelper.fetch_image(self._session, instance.id, - instance.kernel_id, user, project, - ImageType.KERNEL_RAMDISK) - if instance.ramdisk_id: - ramdisk = VMHelper.fetch_image(self._session, instance.id, - instance.ramdisk_id, user, project, - ImageType.KERNEL_RAMDISK) else: vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', disk) + #Have a look at the VDI and see if it has a PV kernel + if not instance.kernel_id: + pv_kernel = VMHelper.lookup_image(self._session, instance.id, + vdi_ref) + if instance.kernel_id: + kernel = VMHelper.fetch_image(self._session, instance.id, + instance.kernel_id, user, project, + ImageType.KERNEL_RAMDISK) + if instance.ramdisk_id: + ramdisk = VMHelper.fetch_image(self._session, instance.id, + instance.ramdisk_id, user, project, + ImageType.KERNEL_RAMDISK) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, pv_kernel) VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, -- cgit From cd5aba9d1d00d9daad87efd89f78e49079bee2c7 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 18:02:57 -0600 Subject: foo --- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index cc72b5d26..5bf0fe994 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -110,7 +110,7 @@ def move_vhds_into_sr(session, args): shutil.move("%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid), sr_path) shutil.move("%s/%s.vhd" % (temp_vhd_path, new_cow_uuid), sr_path) - logginig.debug('Cleaning up temporary SR path %s' % temp_vhd_path) + logging.debug('Cleaning up temporary SR path %s' % temp_vhd_path) os.rmdir(temp_vhd_path) return "" -- cgit From bb98e2055002ff3ed2099f60bbe4058d5f5c7b35 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 23:10:29 -0600 Subject: hurr durr --- nova/virt/xenapi/vmops.py | 5 +++-- plugins/xenserver/xenapi/etc/xapi.d/plugins/migration | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c2f5ddc41..7a176442a 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -288,11 +288,12 @@ class VMOps(object): def attach_disk(self, instance, disk_info): vm_ref = VMHelper.lookup(self._session, instance.name) new_base_copy_uuid = str(uuid.uuid4()) + new_cow_uuid = str(uuid.uuid4()) params = {'instance_id': instance.id, 'old_base_copy_uuid': disk_info['base_copy'], 'old_cow_uuid': disk_info['cow'], 'new_base_copy_uuid': new_base_copy_uuid, - 'new_cow_uuid': str(uuid.uuid4())} + 'new_cow_uuid': new_cow_uuid, } task = self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) @@ -301,7 +302,7 @@ class VMOps(object): # Now we rescan the SR so we find the VHDs VMHelper.scan_sr(self._session) - return new_base_copy_uuid + return new_cow_uuid def resize(self, instance, flavor): """Resize a running instance by changing it's RAM and disk size """ diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 5bf0fe994..7a6eefda2 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -102,7 +102,8 @@ def move_vhds_into_sr(session, args): os.rmdir(source_image_path) # Link the COW to the base copy - logging.debug('Attaching COW to the base copy...') + logging.debug('Attaching COW to the base copy %s -> %s' % + (new_cow_path, new_base_copy_path)) subprocess.call([VHD_UTIL, 'modify', '-n', new_cow_path, '-p', new_base_copy_path]) -- cgit From 98b038c6878772f6b272cb169b1c74bd7c9838b8 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 15 Feb 2011 23:56:00 -0600 Subject: Foo --- nova/api/openstack/servers.py | 12 ++++++++++-- nova/compute/api.py | 20 +++++++++++++++----- nova/compute/manager.py | 16 +++------------- nova/db/sqlalchemy/api.py | 1 + 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 06a40e92c..83b421127 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -207,10 +207,18 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotImplemented()) def _action_confirm_resize(self, input_dict, req, id): - return faults.Fault(exc.HTTPNotImplemented()) + try: + self.compute_api.confirm_resize(req.environ['nova.context'], id) + except: + return faults.Fault(exc.HTTPBadRequest()) + return exc.HTTPNoContent() def _action_revert_resize(self, input_dict, req, id): - return faults.Fault(exc.HTTPNotImplemented()) + try: + self.compute_api.confirm_resize(req.environ['nova.context'], id) + except: + return faults.Fault(exc.HTTPBadRequest()) + return exc.HTTPAccepted() def _action_rebuild(self, input_dict, req, id): return faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/compute/api.py b/nova/compute/api.py index 6b2628378..b8c4a8597 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -401,16 +401,26 @@ class API(base.Base): def revert_resize(self, context, instance_id): """Reverts a resize, deleting the 'new' instance in the process""" + context = context.elevated() instance_ref = self.db.instance_get(instance_id) - self._cast_compute_message('revert_resize', context, instance_id, - instance_ref['host']) + self._cast_compute_message('revert_resize', context, instance_id) def confirm_resize(self, context, instance_id): """Confirms a migration/resize, deleting the 'old' instance in the process.""" - migration_ref = self.db.get_migration_by_instance_id(instance_id) - self._cast_compute_message('confirm_resize', context, instance_id, - migration_ref['source_host']) + context = context.elevated() + migration_ref = self.db.migration_get_by_instance_id(context, + instance_id) + if migration_ref['status'] != 'finished': + raise exception.Error(_("Migration has incorrect status %s" % + migration_ref['status'])) + instance_ref = self.db.instance_get(context, instance_id) + + self._cast_compute_message('terminate_instance', context, instance_id, + migration_ref['source_compute']) + + self.db.instance_update(context, instance_id, + {'host': migration_ref['dest_compute'], }) def resize(self, context, instance_id, flavor): """Resize a running instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 19e1d9f46..169509163 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -379,12 +379,7 @@ class ComputeManager(manager.Manager): def _update_state_callback(self, context, instance_id, result): """Update instance state when async task completes.""" self._update_state(context, instance_id) - - @exception.wrap_exception - @checks_instance_lock - def confirm_resize(self, context, instance_id): - """Destroys the old instance on the source machine""" - pass + @exception.wrap_exception @checks_instance_lock @@ -413,10 +408,8 @@ class ComputeManager(manager.Manager): 'dest_host': self.driver.get_host_ip_addr(), 'status': 'pre-migrating' }) LOG.audit(_('instance %s: migrating to '), instance_id, context=context) - service = self.db.service_get_by_host_and_topic(context, - migration_ref['source_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, - service['host']) + instance_ref['host']) rpc.cast(context, topic, { 'method': 'resize_instance', 'args': { @@ -446,7 +439,7 @@ class ComputeManager(manager.Manager): service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, - service['host']) + migration_ref['dest_compute']) rpc.cast(context, topic, { 'method': 'finish_resize', 'args': { @@ -472,9 +465,6 @@ class ComputeManager(manager.Manager): self.db.migration_update(context, migration_id, {'status': 'finished', }) - # Cleans up any transferred files and unmounts things - self.driver.cleanup_disk_transfer(context, instance_ref['id']) - @exception.wrap_exception @checks_instance_lock def pause_instance(self, context, instance_id): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 861d13716..1b6eaf138 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1935,6 +1935,7 @@ def migration_update(context, id, values): with session.begin(): migration = migration_get(context, id) migration.update(values) + migration.save() return migration -- cgit From 8e536500e83b311bf8d006ca23234c50962dc6aa Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Feb 2011 00:06:29 -0600 Subject: I fail at sessions --- nova/compute/manager.py | 1 - nova/db/sqlalchemy/api.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 169509163..b405e3763 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -435,7 +435,6 @@ class ComputeManager(manager.Manager): #TODO(mdietz): This is where we would update the VM record #after resizing - service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_compute'], FLAGS.compute_topic) topic = self.db.queue_get_for(context, FLAGS.compute_topic, diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 1b6eaf138..f96430e67 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1933,15 +1933,16 @@ def migration_create(context, values): def migration_update(context, id, values): session = get_session() with session.begin(): - migration = migration_get(context, id) + migration = migration_get(context, id, session=session) migration.update(values) - migration.save() + migration.save(session=session) return migration @require_admin_context -def migration_get(context, id): - session = get_session() +def migration_get(context, id, session=None): + if not session: + session = get_session() result = session.query(models.Migration).\ filter_by(id=id).first() if not result: -- cgit From c735796e0668b2bf7c45eeef6396a3fb33d22d6e Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Feb 2011 00:14:26 -0600 Subject: I fail at sessions --- nova/compute/api.py | 2 +- nova/db/api.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index b8c4a8597..3fb852ab0 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -409,7 +409,7 @@ class API(base.Base): """Confirms a migration/resize, deleting the 'old' instance in the process.""" context = context.elevated() - migration_ref = self.db.migration_get_by_instance_id(context, + migration_ref = self.db.migration_get_by_instance(context, instance_id) if migration_ref['status'] != 'finished': raise exception.Error(_("Migration has incorrect status %s" % diff --git a/nova/db/api.py b/nova/db/api.py index 887f57885..9ed5efedb 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -271,9 +271,9 @@ def migration_get(context, migration_id): """Finds a migration by the id""" return IMPL.migration_get(context, migration_id) -def migration_get_by_instance_id(context, instance_id): +def migration_get_by_instance(context, instance_id): """Finds a migration by the instance id its migrating""" - return IMPL.migration_get_by_instance_id(context, instance_id) + return IMPL.migration_get_by_instance(context, instance_id) #################### -- cgit From 89a2ee5ee5ea7dc3d9fed4a2d5aa2fe2faed9f2b Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Feb 2011 11:04:48 -0800 Subject: novatools call to child zones done --- nova/scheduler/manager.py | 51 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 00cab60cf..693f8cb4b 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -23,8 +23,10 @@ Scheduler Service import functools import novatools +import thread from datetime import datetime +from eventlet.greenpool import GreenPool from nova import db from nova import flags @@ -46,17 +48,28 @@ flags.DEFINE_integer('zone_db_check_interval', class ZoneState(object): """Holds the state of all connected child zones.""" def __init__(self): - self.is_active = True - self.name = None - self.capabilities = None - self.retry = 0 - self.last_seen = datetime.min - + self.is_active = True + self.name = None + self.capabilities = None + self.retry = 0 + self.last_seen = datetime.min + def update(self, zone): - self.zone_id = zone.id - self.api_url = zone.api_url - self.username = zone.username - self.password = zone.password + """Update zone credentials from db""" + self.zone_id = zone.id + self.api_url = zone.api_url + self.username = zone.username + self.password = zone.password + + +def _poll_zone(zone): + """Eventlet worker to poll a zone.""" + logging.debug("_POLL_ZONE: STARTING") + os = novatools.OpenStack(zone.username, zone.password, zone.api_url) + zone_metadata = os.zones.info() + logging.debug("_POLL_ZONE: GOT %s" % zone_metadata._info) + + # Stuff this in our cache. class ZoneManager(object): @@ -66,15 +79,27 @@ class ZoneManager(object): self.zone_states = {} def _refresh_from_db(self, context): + """Make our zone state map match the db.""" + # Add/update existing zones ... zones = db.zone_get_all(context) existing = self.zone_states.keys() + db_keys = [] for zone in zones: + db_keys.append(zone.id) if zone.id not in existing: - self.zone_state[zone.id] = ZoneState() - self.zone_state[zones.id].update(zone) + self.zone_states[zone.id] = ZoneState() + self.zone_states[zone.id].update(zone) + + # Cleanup zones removed from db ... + for zone_id in self.zone_states.keys(): + if zone_id not in db_keys: + del self.zone_states[zone_id] def _poll_zones(self, context): - pass + """Try to connect to each child zone and get update.""" + + green_pool = GreenPool() + green_pool.imap(_poll_zone, self.zone_states.values()) def ping(self, context=None): """Ping should be called periodically to update zone status.""" -- cgit From ec39332356d204d14c8910bf72056efd9e943dd0 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Feb 2011 11:05:03 -0800 Subject: novatools call to child zones done --- nova/scheduler/zone_manager.py | 126 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 nova/scheduler/zone_manager.py diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py new file mode 100644 index 000000000..dd910eb09 --- /dev/null +++ b/nova/scheduler/zone_manager.py @@ -0,0 +1,126 @@ +# Copyright (c) 2010 Openstack, LLC. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +ZoneManager oversees all communications with child Zones. +""" + +import novatools +import thread + +from datetime import datetime +from eventlet.greenpool import GreenPool + +from nova import db +from nova import flags +from nova import log as logging + +FLAGS = flags.FLAGS +flags.DEFINE_integer('zone_db_check_interval', 60, + 'Seconds between getting fresh zone info from db.') +flags.DEFINE_integer('zone_failures_to_offline', 3, + 'Number of consecutive errors before marking zone offline') + + +class ZoneState(object): + """Holds the state of all connected child zones.""" + def __init__(self): + self.is_active = True + self.name = None + self.capabilities = None + self.attempt = 0 + self.last_seen = datetime.min + self.last_exception = None + self.last_exception_time = None + + def update_credentials(self, zone): + """Update zone credentials from db""" + self.zone_id = zone.id + self.api_url = zone.api_url + self.username = zone.username + self.password = zone.password + + def update_metadata(self, zone_metadata): + """Update zone metadata after successful communications with + child zone.""" + self.last_seen = datetime.now() + self.attempt = 0 + self.name = zone_metadata["name"] + self.capabilities = zone_metadata["capabilities"] + self.is_active = True + + def log_error(self, exception): + """Something went wrong. Check to see if zone should be + marked as offline.""" + self.last_exception = exception + self.last_exception_time = datetime.now() + logging.warning(_("%s error talking to zone %s") % (exception, + zone.api_url, FLAGS.zone_failures_to_offline)) + + self.attempt += 1 + if self.attempt >= FLAGS.zone_failures_to_offline: + self.is_active = False + logging.error(_("No answer from zone %s after %d " + "attempts. Marking inactive.") % (zone.api_url, + FLAGS.zone_failures_to_offline)) + +def _poll_zone(zone): + """Eventlet worker to poll a zone.""" + logging.debug("_POLL_ZONE: STARTING") + os = novatools.OpenStack(zone.username, zone.password, zone.api_url) + try: + zone.update_metadata(os.zones.info()._info) + except Exception, e: + zone.log_error(e) + +class ZoneManager(object): + """Keeps the zone states updated.""" + def __init__(self): + self.last_zone_db_check = datetime.min + self.zone_states = {} + + def _refresh_from_db(self, context): + """Make our zone state map match the db.""" + # Add/update existing zones ... + zones = db.zone_get_all(context) + existing = self.zone_states.keys() + db_keys = [] + for zone in zones: + db_keys.append(zone.id) + if zone.id not in existing: + self.zone_states[zone.id] = ZoneState() + self.zone_states[zone.id].update_credentials(zone) + + # Cleanup zones removed from db ... + for zone_id in self.zone_states.keys(): + if zone_id not in db_keys: + del self.zone_states[zone_id] + + def _poll_zones(self, context): + """Try to connect to each child zone and get update.""" + green_pool = GreenPool() + green_pool.imap(_poll_zone, self.zone_states.values()) + + def ping(self, context=None): + """Ping should be called periodically to update zone status.""" + logging.debug("ZoneManager PING") + diff = datetime.now() - self.last_zone_db_check + if diff.seconds >= FLAGS.zone_db_check_interval: + logging.debug("ZoneManager RECHECKING DB ") + self.last_zone_db_check = datetime.now() + self._refresh_from_db(context) + self._poll_zones(context) -- cgit From 879845496a50477ebc2709291c159ae1e8d5aa2a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Feb 2011 13:47:14 -0600 Subject: Derp --- nova/compute/api.py | 26 +++++++++++++++++--------- nova/compute/manager.py | 32 +++++++++++++++++++++++++++++--- nova/db/sqlalchemy/api.py | 5 +++-- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 3fb852ab0..58dea5db6 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -402,22 +402,30 @@ class API(base.Base): def revert_resize(self, context, instance_id): """Reverts a resize, deleting the 'new' instance in the process""" context = context.elevated() - instance_ref = self.db.instance_get(instance_id) - self._cast_compute_message('revert_resize', context, instance_id) + migration_ref = self.db.migration_get_by_instance_and_status(context, + instance_id, 'finished') + if not migration_ref: + raise exception.Error(_("No finished migrations found for + instance")) + + params = { 'migration_id': migration_ref['id']) + self._cast_compute_message('revert_resize', context, instance_id, + migration_ref['dest_compute'], params=params) def confirm_resize(self, context, instance_id): """Confirms a migration/resize, deleting the 'old' instance in the process.""" context = context.elevated() - migration_ref = self.db.migration_get_by_instance(context, - instance_id) - if migration_ref['status'] != 'finished': - raise exception.Error(_("Migration has incorrect status %s" % - migration_ref['status'])) + migration_ref = self.db.migration_get_by_instance_and_status(context, + instance_id, 'finished') + if not migration_ref: + raise exception.Error(_("No finished migrations found for + instance")) instance_ref = self.db.instance_get(context, instance_id) - self._cast_compute_message('terminate_instance', context, instance_id, - migration_ref['source_compute']) + params = { 'migration_id': migration_ref['id']) + self._cast_compute_message('confirm_resize', context, instance_id, + migration_ref['source_compute'], params=param) self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b405e3763..c05edd140 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -380,15 +380,41 @@ class ComputeManager(manager.Manager): """Update instance state when async task completes.""" self._update_state(context, instance_id) + @exception.wrap_exception + @echecks_instance_lock + def confirm_resize(self, context, instance_id, migration_id): + """ Destroys the source instance """ + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + migration_ref = self.db.migration_get(context, migration_id) + self.driver.destroy(instance_ref) + self.db.migration_update(context, migration_id, + { 'status': 'confirmed' }) @exception.wrap_exception @checks_instance_lock - def revert_resize(self, context, instance_id): + def revert_resize(self, context, instance_id, migration_id): """Destroys the new instance on the destination machine, reverts the model changes, and powers on the old instance on the source machine""" - pass - + instance_ref = self.db.instance_get(context, instance_id) + migration_ref = self.db.migration_get(context, migration_id) + + if migration_ref['source_compute'] == instance_ref['host']: + self.driver.power_on(instance_ref) + self.db.migration_update(context, migration_id, + { 'status': 'reverted' }) + else: + self.driver.destroy(instance_ref) + topic = self.db.queue_get_for(context, FLAGS.compute_topic, + instance_ref['host']) + rpc.cast(context, topic, + { 'method': 'resize_instance', + 'args': { + 'migration_id': migration_ref['id'], + 'instance_id': instance_id, + }, + }) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index f96430e67..62484805c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1952,10 +1952,11 @@ def migration_get(context, id, session=None): @require_admin_context -def migration_get_by_instance(context, instance_id): +def migration_get_by_instance_and_status(context, instance_id, status): session = get_session() result = session.query(models.Migration).\ - filter_by(instance_id=instance_id).first() + filter_by(instance_id=instance_id). + filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") % migration_id) -- cgit From 905cf54f06f6dde95039599ae5ea30d2f070f398 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Feb 2011 13:53:47 -0600 Subject: Typos --- nova/compute/api.py | 8 ++++---- nova/compute/manager.py | 2 +- nova/db/sqlalchemy/api.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 4262a771b..2f39b8b47 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -405,10 +405,10 @@ class API(base.Base): migration_ref = self.db.migration_get_by_instance_and_status(context, instance_id, 'finished') if not migration_ref: - raise exception.Error(_("No finished migrations found for + raise exception.Error(_("No finished migrations found for \ instance")) - params = { 'migration_id': migration_ref['id']) + params = { 'migration_id': migration_ref['id'] } self._cast_compute_message('revert_resize', context, instance_id, migration_ref['dest_compute'], params=params) @@ -419,11 +419,11 @@ class API(base.Base): migration_ref = self.db.migration_get_by_instance_and_status(context, instance_id, 'finished') if not migration_ref: - raise exception.Error(_("No finished migrations found for + raise exception.Error(_("No finished migrations found for \ instance")) instance_ref = self.db.instance_get(context, instance_id) - params = { 'migration_id': migration_ref['id']) + params = { 'migration_id': migration_ref['id'] } self._cast_compute_message('confirm_resize', context, instance_id, migration_ref['source_compute'], params=param) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d78318bec..4bab7081a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -381,7 +381,7 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) @exception.wrap_exception - @echecks_instance_lock + @checks_instance_lock def confirm_resize(self, context, instance_id, migration_id): """ Destroys the source instance """ context = context.elevated() diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index b6fb57df7..f4dc8a630 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1961,7 +1961,7 @@ def migration_get(context, id, session=None): def migration_get_by_instance_and_status(context, instance_id, status): session = get_session() result = session.query(models.Migration).\ - filter_by(instance_id=instance_id). + filter_by(instance_id=instance_id).\ filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") -- cgit From 49a7e430ca30768a68a111223068652c781206fe Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Feb 2011 13:17:42 -0800 Subject: zone manager tests --- nova/scheduler/zone_manager.py | 2 +- nova/tests/test_zones.py | 132 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 nova/tests/test_zones.py diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index dd910eb09..0974f271b 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -71,7 +71,7 @@ class ZoneState(object): logging.warning(_("%s error talking to zone %s") % (exception, zone.api_url, FLAGS.zone_failures_to_offline)) - self.attempt += 1 + self.attempt += 1 if self.attempt >= FLAGS.zone_failures_to_offline: self.is_active = False logging.error(_("No answer from zone %s after %d " diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py new file mode 100644 index 000000000..b4c8815d5 --- /dev/null +++ b/nova/tests/test_zones.py @@ -0,0 +1,132 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For ZoneManager +""" + +import datetime +import mox + +from nova import context +from nova import db +from nova import flags +from nova import service +from nova import test +from nova import rpc +from nova import utils +from nova.auth import manager as auth_manager +from nova.scheduler import zone_manager + + +class FakeZone: + """Represents a fake zone from the db""" + def __init__(self, *args, **kwargs): + for k, v in kwargs.iteritems(): + setattr(self, k, v) + + +class ZoneManagerTestCase(test.TestCase): + """Test case for zone manager""" + def test_ping(self): + zm = zone_manager.ZoneManager() + self.mox.StubOutWithMock(zm, '_refresh_from_db') + self.mox.StubOutWithMock(zm, '_poll_zones') + zm._refresh_from_db(mox.IgnoreArg()) + zm._poll_zones(mox.IgnoreArg()) + + self.mox.ReplayAll() + zm.ping(None) + self.mox.VerifyAll() + + def test_refresh_from_db_new(self): + zm = zone_manager.ZoneManager() + + self.mox.StubOutWithMock(db, 'zone_get_all') + db.zone_get_all(mox.IgnoreArg()).AndReturn([ + FakeZone(id=1, api_url='http://foo.com', username='user1', + password='pass1'), + ]) + + self.assertEquals(len(zm.zone_states), 0) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 1) + self.assertEquals(zm.zone_states[1].username, 'user1') + + def test_refresh_from_db_replace_existing(self): + zm = zone_manager.ZoneManager() + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com', + username='user1', password='pass1')) + zm.zone_states[1] = zone_state + + self.mox.StubOutWithMock(db, 'zone_get_all') + db.zone_get_all(mox.IgnoreArg()).AndReturn([ + FakeZone(id=1, api_url='http://foo.com', username='user2', + password='pass2'), + ]) + + self.assertEquals(len(zm.zone_states), 1) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 1) + self.assertEquals(zm.zone_states[1].username, 'user2') + + def test_refresh_from_db_missing(self): + zm = zone_manager.ZoneManager() + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com', + username='user1', password='pass1')) + zm.zone_states[1] = zone_state + + self.mox.StubOutWithMock(db, 'zone_get_all') + db.zone_get_all(mox.IgnoreArg()).AndReturn([ ]) + + self.assertEquals(len(zm.zone_states), 1) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 0) + + def test_refresh_from_db_add_and_delete(self): + zm = zone_manager.ZoneManager() + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=1, api_url='http://foo.com', + username='user1', password='pass1')) + zm.zone_states[1] = zone_state + + self.mox.StubOutWithMock(db, 'zone_get_all') + + db.zone_get_all(mox.IgnoreArg()).AndReturn([ + FakeZone(id=2, api_url='http://foo.com', username='user2', + password='pass2'), + ]) + self.assertEquals(len(zm.zone_states), 1) + + self.mox.ReplayAll() + zm._refresh_from_db(None) + self.mox.VerifyAll() + + self.assertEquals(len(zm.zone_states), 1) + self.assertEquals(zm.zone_states[2].username, 'user2') -- cgit From 8f206774ee75c2d96c15dd2c604ae5da9601d91f Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Feb 2011 17:02:57 -0600 Subject: Better exceptions --- nova/api/openstack/servers.py | 15 +++++++++------ nova/db/api.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 83b421127..2fc105d07 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -209,15 +209,17 @@ class Controller(wsgi.Controller): def _action_confirm_resize(self, input_dict, req, id): try: self.compute_api.confirm_resize(req.environ['nova.context'], id) - except: - return faults.Fault(exc.HTTPBadRequest()) + except Exception, e: + LOG.exception(_("Error in confirm-resize %s"), e) + return faults.Fault(exc.HTTPBadRequest(e)) return exc.HTTPNoContent() def _action_revert_resize(self, input_dict, req, id): try: self.compute_api.confirm_resize(req.environ['nova.context'], id) - except: - return faults.Fault(exc.HTTPBadRequest()) + except Exception, e: + LOG.exception(_("Error in revert-resize %s"), e) + return faults.Fault(exc.HTTPBadRequest(e)) return exc.HTTPAccepted() def _action_rebuild(self, input_dict, req, id): @@ -229,8 +231,9 @@ class Controller(wsgi.Controller): flavor_id = input_dict['resize']['flavorId'] self.compute_api.resize(req.environ['nova.context'], id, flavor_id) - except: - return faults.Fault(exc.HTTPUnprocessableEntity()) + except Exception, e: + LOG.exception(_("Error in resize %s"), e) + return faults.Fault(exc.HTTPUnprocessableEntity(e)) return faults.Fault(exc.HTTPAccepted()) diff --git a/nova/db/api.py b/nova/db/api.py index 9ed5efedb..295d1a90a 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -271,7 +271,7 @@ def migration_get(context, migration_id): """Finds a migration by the id""" return IMPL.migration_get(context, migration_id) -def migration_get_by_instance(context, instance_id): +def migration_get_by_instance_and_status(context, instance_id, status): """Finds a migration by the instance id its migrating""" return IMPL.migration_get_by_instance(context, instance_id) -- cgit From c01519112245f5e991ab438fe983bf9331d4e952 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Feb 2011 17:51:43 -0600 Subject: fixed --- nova/compute/api.py | 5 ++++- nova/compute/manager.py | 2 -- nova/db/api.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 2f39b8b47..635632b73 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -425,7 +425,10 @@ class API(base.Base): params = { 'migration_id': migration_ref['id'] } self._cast_compute_message('confirm_resize', context, instance_id, - migration_ref['source_compute'], params=param) + migration_ref['source_compute'], params=params) + + self.db.migration_update(context, migration_id, + { 'status': 'confirmed' }) self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 4bab7081a..33fad50fd 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -388,8 +388,6 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_get(context, migration_id) self.driver.destroy(instance_ref) - self.db.migration_update(context, migration_id, - { 'status': 'confirmed' }) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/api.py b/nova/db/api.py index 295d1a90a..ab871c67e 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -273,7 +273,8 @@ def migration_get(context, migration_id): def migration_get_by_instance_and_status(context, instance_id, status): """Finds a migration by the instance id its migrating""" - return IMPL.migration_get_by_instance(context, instance_id) + return IMPL.migration_get_by_instance_and_status(context, instance_id, + status) #################### -- cgit From 719dbda7f8b856af334744de4807036e6ee704c1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Feb 2011 18:30:56 -0800 Subject: polling tests --- nova/scheduler/zone_manager.py | 15 ++++++++++----- nova/tests/test_zones.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 0974f271b..a6bbc2ebd 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -68,22 +68,27 @@ class ZoneState(object): marked as offline.""" self.last_exception = exception self.last_exception_time = datetime.now() - logging.warning(_("%s error talking to zone %s") % (exception, - zone.api_url, FLAGS.zone_failures_to_offline)) + logging.warning(_("'%s' error talking to zone %s") % (exception, + self.api_url)) self.attempt += 1 if self.attempt >= FLAGS.zone_failures_to_offline: self.is_active = False logging.error(_("No answer from zone %s after %d " - "attempts. Marking inactive.") % (zone.api_url, + "attempts. Marking inactive.") % (self.api_url, FLAGS.zone_failures_to_offline)) + +def _call_novatools(zone): + """Call novatools. Broken out for testing purposes.""" + os = novatools.OpenStack(zone.username, zone.password, zone.api_url) + return os.zones.info()._info + def _poll_zone(zone): """Eventlet worker to poll a zone.""" logging.debug("_POLL_ZONE: STARTING") - os = novatools.OpenStack(zone.username, zone.password, zone.api_url) try: - zone.update_metadata(os.zones.info()._info) + zone.update_metadata(_call_novatools(zone)) except Exception, e: zone.log_error(e) diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index b4c8815d5..2cb070aca 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -19,6 +19,7 @@ Tests For ZoneManager import datetime import mox +import novatools from nova import context from nova import db @@ -30,6 +31,8 @@ from nova import utils from nova.auth import manager as auth_manager from nova.scheduler import zone_manager +FLAGS = flags.FLAGS + class FakeZone: """Represents a fake zone from the db""" @@ -38,6 +41,11 @@ class FakeZone: setattr(self, k, v) +def exploding_novatools(zone): + """Used when we want to simulate a novatools call failing.""" + raise Exception("kaboom") + + class ZoneManagerTestCase(test.TestCase): """Test case for zone manager""" def test_ping(self): @@ -130,3 +138,36 @@ class ZoneManagerTestCase(test.TestCase): self.assertEquals(len(zm.zone_states), 1) self.assertEquals(zm.zone_states[2].username, 'user2') + + def test_poll_zone(self): + self.mox.StubOutWithMock(zone_manager, '_call_novatools') + zone_manager._call_novatools(mox.IgnoreArg()).AndReturn( + dict(name='zohan', capabilities='hairdresser')) + + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=2, + api_url='http://foo.com', username='user2', + password='pass2')) + zone_state.attempt = 1 + + self.mox.ReplayAll() + zone_manager._poll_zone(zone_state) + self.mox.VerifyAll() + self.assertEquals(zone_state.attempt, 0) + self.assertEquals(zone_state.name, 'zohan') + + def test_poll_zone_fails(self): + self.stubs.Set(zone_manager, "_call_novatools", exploding_novatools) + + zone_state = zone_manager.ZoneState() + zone_state.update_credentials(FakeZone(id=2, + api_url='http://foo.com', username='user2', + password='pass2')) + zone_state.attempt = FLAGS.zone_failures_to_offline - 1 + + self.mox.ReplayAll() + zone_manager._poll_zone(zone_state) + self.mox.VerifyAll() + self.assertEquals(zone_state.attempt, 3) + self.assertFalse(zone_state.is_active) + self.assertEquals(zone_state.name, None) -- cgit From 984db08a205bdd9196c3e1cc3415873a853c33ba Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Feb 2011 18:35:43 -0800 Subject: style cleanup --- nova/scheduler/zone_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index a6bbc2ebd..a35acb000 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -84,14 +84,16 @@ def _call_novatools(zone): os = novatools.OpenStack(zone.username, zone.password, zone.api_url) return os.zones.info()._info + def _poll_zone(zone): """Eventlet worker to poll a zone.""" - logging.debug("_POLL_ZONE: STARTING") + logging.debug(_("Polling zone: %s") % zone.api_url) try: zone.update_metadata(_call_novatools(zone)) except Exception, e: zone.log_error(e) + class ZoneManager(object): """Keeps the zone states updated.""" def __init__(self): @@ -122,10 +124,9 @@ class ZoneManager(object): def ping(self, context=None): """Ping should be called periodically to update zone status.""" - logging.debug("ZoneManager PING") diff = datetime.now() - self.last_zone_db_check if diff.seconds >= FLAGS.zone_db_check_interval: - logging.debug("ZoneManager RECHECKING DB ") + logging.debug("Updating zone cache from db.") self.last_zone_db_check = datetime.now() self._refresh_from_db(context) self._poll_zones(context) -- cgit From e67927c181a1f24df35a6df5663e397e260979cf Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Feb 2011 13:28:39 -0600 Subject: Foo --- nova/api/openstack/servers.py | 2 +- nova/compute/manager.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 2fc105d07..5f369463d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -216,7 +216,7 @@ class Controller(wsgi.Controller): def _action_revert_resize(self, input_dict, req, id): try: - self.compute_api.confirm_resize(req.environ['nova.context'], id) + self.compute_api.revert_resize(req.environ['nova.context'], id) except Exception, e: LOG.exception(_("Error in revert-resize %s"), e) return faults.Fault(exc.HTTPBadRequest(e)) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 33fad50fd..4e6532ef4 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -398,7 +398,8 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_get(context, migration_id) - if migration_ref['source_compute'] == instance_ref['host']: + #TODO(mdietz): we may want to split these into separate methods. + if migration_ref['source_compute'] == FLAGS.host: self.driver.power_on(instance_ref) self.db.migration_update(context, migration_id, { 'status': 'reverted' }) -- cgit From b1fe9a64143505235eb2e3dbe6a6c0966a85ae76 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Feb 2011 14:10:08 -0600 Subject: Stop blowing away the ramdisk --- nova/compute/manager.py | 2 +- nova/virt/xenapi/vmops.py | 29 ++++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 4e6532ef4..e81ee3148 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -408,7 +408,7 @@ class ComputeManager(manager.Manager): topic = self.db.queue_get_for(context, FLAGS.compute_topic, instance_ref['host']) rpc.cast(context, topic, - { 'method': 'resize_instance', + { 'method': 'revert_resize', 'args': { 'migration_id': migration_ref['id'], 'instance_id': instance_id, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 13c13d0f6..17b91d27c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -234,7 +234,8 @@ class VMOps(object): return self def __exit__(self, type, value, traceback): - self.virt._destroy(self.instance, self.vm_ref, shutdown=False) + self.virt._destroy(self.instance, self.vm_ref, shutdown=False, + destroy_kernel_ramdisk=False) #TODO(sirp): Add quiesce and VSS locking support when Windows support # is added @@ -393,7 +394,7 @@ class VMOps(object): except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_vm(self, instance, vm): + def _destroy_vm(self, instance, vm, destroy_kernel_ramdisk): """Destroys a VM record """ try: kernel = None @@ -402,16 +403,18 @@ class VMOps(object): (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk( self._session, vm) task1 = self._session.call_xenapi('Async.VM.destroy', vm) - LOG.debug(_("Removing kernel/ramdisk files")) - fn = "remove_kernel_ramdisk" - args = {} - if kernel: - args['kernel-file'] = kernel - if ramdisk: - args['ramdisk-file'] = ramdisk - task2 = self._session.async_call_plugin('glance', fn, args) + if destroy_kernel_ramdisk: + LOG.debug(_("Removing kernel/ramdisk files")) + fn = "remove_kernel_ramdisk" + args = {} + if kernel: + args['kernel-file'] = kernel + if ramdisk: + args['ramdisk-file'] = ramdisk + task2 = self._session.async_call_plugin('glance', fn, args) self._session.wait_for_task(instance.id, task1) - self._session.wait_for_task(instance.id, task2) + if destroy_kernel_ramdisk: + self._session.wait_for_task(instance.id, task2) LOG.debug(_("kernel/ramdisk files removed")) except self.XenAPI.Failure, exc: LOG.exception(exc) @@ -426,7 +429,7 @@ class VMOps(object): vm = VMHelper.lookup(self._session, instance.name) return self._destroy(instance, vm, shutdown=True) - def _destroy(self, instance, vm, shutdown=True): + def _destroy(self, instance, vm, shutdown=True, destroy_kernel_ramdisk=True): """ Destroys VM instance by performing: @@ -443,7 +446,7 @@ class VMOps(object): self._shutdown(instance, vm) self._destroy_vdis(instance, vm) - self._destroy_vm(instance, vm) + self._destroy_vm(instance, vm, destroy_kernel_ramdisk) def _wait_with_callback(self, instance_id, task, callback): ret = None -- cgit From aa71a25c9f9bf5df3aea781138fa8d69654f06d9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 12:12:19 -0800 Subject: zone list now comes from scheduler zonemanager --- nova/api/openstack/zones.py | 32 +++++++++++++++-- nova/scheduler/manager.py | 82 ++++-------------------------------------- nova/scheduler/zone_manager.py | 11 +++++- 3 files changed, 46 insertions(+), 79 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 16e5e366b..bd2c488d9 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -19,6 +19,7 @@ import logging from nova import flags from nova import wsgi from nova import db +from nova import rpc FLAGS = flags.FLAGS @@ -33,6 +34,10 @@ def _filter_keys(item, keys): return dict((k, v) for k, v in item.iteritems() if k in keys) +def _exclude_keys(item, keys): + return dict((k, v) for k, v in item.iteritems() if k not in keys) + + def _scrub_zone(zone): return _filter_keys(zone, ('id', 'api_url')) @@ -44,11 +49,34 @@ class Controller(wsgi.Controller): "attributes": { "zone": ["id", "api_url", "name", "capabilities"]}}} + def _call_scheduler(self, method, context, params=None): + """Generic handler for RPC calls to the scheduler. + + :param params: Optional dictionary of arguments to be passed to the + scheduler worker + + :retval: Result returned by scheduler worker + """ + if not params: + params = {} + queue = FLAGS.scheduler_topic + kwargs = {'method': method, 'args': params} + return rpc.call(context, queue, kwargs) + def index(self, req): """Return all zones in brief""" - items = db.zone_get_all(req.environ['nova.context']) + # Ask the ZoneManager in the Scheduler for most recent data. + items = self._call_scheduler('get_zone_list', + req.environ['nova.context']) + for item in items: + item['api_url'] = item['api_url'].replace('\\/', '/') + + # Or fall-back to the database ... + if len(items) == 0: + items = db.zone_get_all(req.environ['nova.context']) items = common.limited(items, req) - items = [_scrub_zone(item) for item in items] + items = [_exclude_keys(item, ['username', 'password']) + for item in items] return dict(zones=items) def detail(self, req): diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 693f8cb4b..00a0f4100 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -22,11 +22,6 @@ Scheduler Service """ import functools -import novatools -import thread - -from datetime import datetime -from eventlet.greenpool import GreenPool from nova import db from nova import flags @@ -34,83 +29,14 @@ from nova import log as logging from nova import manager from nova import rpc from nova import utils +from nova.scheduler.zone_manager import ZoneManager LOG = logging.getLogger('nova.scheduler.manager') FLAGS = flags.FLAGS flags.DEFINE_string('scheduler_driver', 'nova.scheduler.chance.ChanceScheduler', 'Driver to use for the scheduler') -flags.DEFINE_integer('zone_db_check_interval', - 60, - 'Seconds between getting fresh zone info from db.') - - -class ZoneState(object): - """Holds the state of all connected child zones.""" - def __init__(self): - self.is_active = True - self.name = None - self.capabilities = None - self.retry = 0 - self.last_seen = datetime.min - - def update(self, zone): - """Update zone credentials from db""" - self.zone_id = zone.id - self.api_url = zone.api_url - self.username = zone.username - self.password = zone.password - - -def _poll_zone(zone): - """Eventlet worker to poll a zone.""" - logging.debug("_POLL_ZONE: STARTING") - os = novatools.OpenStack(zone.username, zone.password, zone.api_url) - zone_metadata = os.zones.info() - logging.debug("_POLL_ZONE: GOT %s" % zone_metadata._info) - - # Stuff this in our cache. - - -class ZoneManager(object): - """Keeps the zone states updated.""" - def __init__(self): - self.last_zone_db_check = datetime.min - self.zone_states = {} - - def _refresh_from_db(self, context): - """Make our zone state map match the db.""" - # Add/update existing zones ... - zones = db.zone_get_all(context) - existing = self.zone_states.keys() - db_keys = [] - for zone in zones: - db_keys.append(zone.id) - if zone.id not in existing: - self.zone_states[zone.id] = ZoneState() - self.zone_states[zone.id].update(zone) - - # Cleanup zones removed from db ... - for zone_id in self.zone_states.keys(): - if zone_id not in db_keys: - del self.zone_states[zone_id] - - def _poll_zones(self, context): - """Try to connect to each child zone and get update.""" - - green_pool = GreenPool() - green_pool.imap(_poll_zone, self.zone_states.values()) - - def ping(self, context=None): - """Ping should be called periodically to update zone status.""" - logging.debug("ZoneManager PING") - diff = datetime.now() - self.last_zone_db_check - if diff.seconds >= FLAGS.zone_db_check_interval: - logging.debug("ZoneManager RECHECKING DB ") - self.last_zone_db_check = datetime.now() - self._refresh_from_db(context) - self._poll_zones(context) - + class SchedulerManager(manager.Manager): """Chooses a host to run instances on.""" @@ -129,6 +55,10 @@ class SchedulerManager(manager.Manager): """Poll child zones periodically to get status.""" self.zone_manager.ping(context) + def get_zone_list(self, context=None): + """Get a list of zones from the ZoneManager.""" + return self.zone_manager.get_zone_list() + def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index a35acb000..4fa528973 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -21,6 +21,7 @@ ZoneManager oversees all communications with child Zones. import novatools import thread +import traceback from datetime import datetime from eventlet.greenpool import GreenPool @@ -63,6 +64,11 @@ class ZoneState(object): self.capabilities = zone_metadata["capabilities"] self.is_active = True + def to_dict(self): + return dict(name=self.name, capabilities=self.capabilities, + is_active=self.is_active, api_url=self.api_url, + id=self.zone_id) + def log_error(self, exception): """Something went wrong. Check to see if zone should be marked as offline.""" @@ -91,7 +97,7 @@ def _poll_zone(zone): try: zone.update_metadata(_call_novatools(zone)) except Exception, e: - zone.log_error(e) + zone.log_error(traceback.format_exc()) class ZoneManager(object): @@ -100,6 +106,9 @@ class ZoneManager(object): self.last_zone_db_check = datetime.min self.zone_states = {} + def get_zone_list(self): + return [ zone.to_dict() for zone in self.zone_states.values() ] + def _refresh_from_db(self, context): """Make our zone state map match the db.""" # Add/update existing zones ... -- cgit From e77f8751dd59e5d650d047a6711c3d137947dda7 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 16:18:03 -0400 Subject: fixup --- bin/nova-combined | 4 ++-- nova/api/openstack/zones.py | 6 +++--- nova/flags.py | 2 +- nova/scheduler/manager.py | 2 +- nova/scheduler/zone_manager.py | 18 +++++++++--------- nova/tests/test_zones.py | 10 +++++----- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bin/nova-combined b/bin/nova-combined index a0f552d64..913c866bf 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -53,11 +53,11 @@ if __name__ == '__main__': compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') - #volume = service.Service.create(binary='nova-volume') + volume = service.Service.create(binary='nova-volume') scheduler = service.Service.create(binary='nova-scheduler') #objectstore = service.Service.create(binary='nova-objectstore') - service.serve(compute, network, scheduler) + service.serve(compute, network, volume, scheduler) apps = [] paste_config_file = wsgi.paste_config_file('nova-api.conf') diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index bd2c488d9..f75176824 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -66,7 +66,7 @@ class Controller(wsgi.Controller): def index(self, req): """Return all zones in brief""" # Ask the ZoneManager in the Scheduler for most recent data. - items = self._call_scheduler('get_zone_list', + items = self._call_scheduler('get_zone_list', req.environ['nova.context']) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') @@ -75,7 +75,7 @@ class Controller(wsgi.Controller): if len(items) == 0: items = db.zone_get_all(req.environ['nova.context']) items = common.limited(items, req) - items = [_exclude_keys(item, ['username', 'password']) + items = [_exclude_keys(item, ['username', 'password']) for item in items] return dict(zones=items) @@ -85,7 +85,7 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - return dict(zone=dict(name=FLAGS.zone_name, + return dict(zone=dict(name=FLAGS.zone_name, capabilities=FLAGS.zone_capabilities)) def show(self, req, id): diff --git a/nova/flags.py b/nova/flags.py index 0a45499f3..60d7cdd06 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -314,6 +314,6 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'xen, linux', +DEFINE_string('zone_capabilities', 'xen, linux', 'comma-delimited list of tags which represent boolean' ' capabilities of this zone') diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 00a0f4100..7ced33b9c 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -36,7 +36,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('scheduler_driver', 'nova.scheduler.chance.ChanceScheduler', 'Driver to use for the scheduler') - + class SchedulerManager(manager.Manager): """Chooses a host to run instances on.""" diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 4fa528973..e7c37a9a6 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -47,14 +47,14 @@ class ZoneState(object): self.last_seen = datetime.min self.last_exception = None self.last_exception_time = None - + def update_credentials(self, zone): """Update zone credentials from db""" self.zone_id = zone.id self.api_url = zone.api_url self.username = zone.username self.password = zone.password - + def update_metadata(self, zone_metadata): """Update zone metadata after successful communications with child zone.""" @@ -79,10 +79,10 @@ class ZoneState(object): self.attempt += 1 if self.attempt >= FLAGS.zone_failures_to_offline: - self.is_active = False - logging.error(_("No answer from zone %s after %d " - "attempts. Marking inactive.") % (self.api_url, - FLAGS.zone_failures_to_offline)) + self.is_active = False + logging.error(_("No answer from zone %s after %d " + "attempts. Marking inactive.") % (self.api_url, + FLAGS.zone_failures_to_offline)) def _call_novatools(zone): @@ -107,7 +107,7 @@ class ZoneManager(object): self.zone_states = {} def get_zone_list(self): - return [ zone.to_dict() for zone in self.zone_states.values() ] + return [zone.to_dict() for zone in self.zone_states.values()] def _refresh_from_db(self, context): """Make our zone state map match the db.""" @@ -125,7 +125,7 @@ class ZoneManager(object): for zone_id in self.zone_states.keys(): if zone_id not in db_keys: del self.zone_states[zone_id] - + def _poll_zones(self, context): """Try to connect to each child zone and get update.""" green_pool = GreenPool() @@ -134,7 +134,7 @@ class ZoneManager(object): def ping(self, context=None): """Ping should be called periodically to update zone status.""" diff = datetime.now() - self.last_zone_db_check - if diff.seconds >= FLAGS.zone_db_check_interval: + if diff.seconds >= FLAGS.zone_db_check_interval: logging.debug("Updating zone cache from db.") self.last_zone_db_check = datetime.now() self._refresh_from_db(context) diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 2cb070aca..7036ebe58 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -67,7 +67,7 @@ class ZoneManagerTestCase(test.TestCase): FakeZone(id=1, api_url='http://foo.com', username='user1', password='pass1'), ]) - + self.assertEquals(len(zm.zone_states), 0) self.mox.ReplayAll() @@ -89,7 +89,7 @@ class ZoneManagerTestCase(test.TestCase): FakeZone(id=1, api_url='http://foo.com', username='user2', password='pass2'), ]) - + self.assertEquals(len(zm.zone_states), 1) self.mox.ReplayAll() @@ -107,8 +107,8 @@ class ZoneManagerTestCase(test.TestCase): zm.zone_states[1] = zone_state self.mox.StubOutWithMock(db, 'zone_get_all') - db.zone_get_all(mox.IgnoreArg()).AndReturn([ ]) - + db.zone_get_all(mox.IgnoreArg()).AndReturn([]) + self.assertEquals(len(zm.zone_states), 1) self.mox.ReplayAll() @@ -125,7 +125,7 @@ class ZoneManagerTestCase(test.TestCase): zm.zone_states[1] = zone_state self.mox.StubOutWithMock(db, 'zone_get_all') - + db.zone_get_all(mox.IgnoreArg()).AndReturn([ FakeZone(id=2, api_url='http://foo.com', username='user2', password='pass2'), -- cgit From 3dd6e369c0aa2e3092eaa32a6b04cbba712ba5ad Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Feb 2011 14:23:20 -0600 Subject: Move the ramdisk logging stuff --- nova/virt/xenapi/vmops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 17b91d27c..f7e434300 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -412,10 +412,10 @@ class VMOps(object): if ramdisk: args['ramdisk-file'] = ramdisk task2 = self._session.async_call_plugin('glance', fn, args) + LOG.debug(_("kernel/ramdisk files removed")) self._session.wait_for_task(instance.id, task1) if destroy_kernel_ramdisk: self._session.wait_for_task(instance.id, task2) - LOG.debug(_("kernel/ramdisk files removed")) except self.XenAPI.Failure, exc: LOG.exception(exc) -- cgit From b48201be9a5fa08ce21ef241052071800e5777ca Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 12:43:22 -0800 Subject: fixed zone list tests --- nova/tests/api/openstack/test_zones.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 5542a1cf3..6c06fa8b8 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -52,7 +52,20 @@ def zone_delete(context, zone_id): pass -def zone_get_all(context): +def zone_get_all_scheduler(x, y, z): + return [ + dict(id=1, api_url='http://foo.com', username='bob', + password='xxx'), + dict(id=2, api_url='http://blah.com', username='alice', + password='qwerty') + ] + + +def zone_get_all_scheduler_empty(x, y, z): + return [] + + +def zone_get_all_db(context): return [ dict(id=1, api_url='http://foo.com', username='bob', password='xxx'), @@ -74,7 +87,6 @@ class ZonesTest(unittest.TestCase): FLAGS.allow_admin_api = True self.stubs.Set(nova.db, 'zone_get', zone_get) - self.stubs.Set(nova.db, 'zone_get_all', zone_get_all) self.stubs.Set(nova.db, 'zone_update', zone_update) self.stubs.Set(nova.db, 'zone_create', zone_create) self.stubs.Set(nova.db, 'zone_delete', zone_delete) @@ -83,7 +95,9 @@ class ZonesTest(unittest.TestCase): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin - def test_get_zone_list(self): + def test_get_zone_list_scheduler(self): + self.stubs.Set(zones.Controller, '_call_scheduler', + zone_get_all_scheduler) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -91,6 +105,18 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(len(res_dict['zones']), 2) + def test_get_zone_list_db(self): + self.stubs.Set(zones.Controller, '_call_scheduler', + zone_get_all_scheduler_empty) + self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) + req = webob.Request.blank('/v1.0/zones') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(len(res_dict['zones']), 2) + + def test_get_zone_by_id(self): req = webob.Request.blank('/v1.0/zones/1') res = req.get_response(fakes.wsgi_app()) -- cgit From 0e3c86dcdc49890eecaa2d1ea64c0012e569682f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 17 Feb 2011 22:07:00 +0100 Subject: Use a semaphore to ensure we don't run more than one iptables-restore at a time. --- nova/virt/libvirt_conn.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4e0fd106f..7548fff63 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -46,6 +46,7 @@ from xml.dom import minidom from eventlet import greenthread from eventlet import event +from eventlet import semaphore from eventlet import tpool import IPy @@ -63,6 +64,7 @@ from nova.compute import power_state from nova.virt import disk from nova.virt import images +libvirt_semaphore = semaphore.Semaphore() libvirt = None libxml2 = None Template = None @@ -1237,17 +1239,19 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter') - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter)) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save -t filter') + with libvirt_semaphore: + current_filter, _ = self.execute('sudo iptables-save -t filter') current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', + new_filter = self.modify_rules(current_lines, 4) + self.execute('sudo iptables-restore', process_input='\n'.join(new_filter)) + if(FLAGS.use_ipv6): + current_filter, _ = self.execute('sudo ip6tables-save ' + '-t filter') + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, 6) + self.execute('sudo ip6tables-restore', + process_input='\n'.join(new_filter)) def modify_rules(self, current_lines, ip_version=4): ctxt = context.get_admin_context() -- cgit From b9a03524d03c0ce7fa98fab5531db720941bbfdb Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 13:23:56 -0800 Subject: multi positional string fix --- nova/scheduler/zone_manager.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index e7c37a9a6..af0b90f9f 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -74,15 +74,17 @@ class ZoneState(object): marked as offline.""" self.last_exception = exception self.last_exception_time = datetime.now() - logging.warning(_("'%s' error talking to zone %s") % (exception, - self.api_url)) + api_url = self.api_url + logging.warning(_("'%(exception)s' error talking to " + "zone %(api_url)s") % locals()) + max_errors = FLAGS.zone_failures_to_offline: self.attempt += 1 - if self.attempt >= FLAGS.zone_failures_to_offline: + if self.attempt >= max_errors: self.is_active = False - logging.error(_("No answer from zone %s after %d " - "attempts. Marking inactive.") % (self.api_url, - FLAGS.zone_failures_to_offline)) + logging.error(_("No answer from zone %(api_url)s " + "after %(max_errors)d " + "attempts. Marking inactive.") % locals()) def _call_novatools(zone): -- cgit From a33f5495fe261641713131901fee1e83ccc4890f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 13:29:19 -0800 Subject: fixed strings --- nova/scheduler/zone_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index af0b90f9f..4bf6e36c6 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -78,7 +78,7 @@ class ZoneState(object): logging.warning(_("'%(exception)s' error talking to " "zone %(api_url)s") % locals()) - max_errors = FLAGS.zone_failures_to_offline: + max_errors = FLAGS.zone_failures_to_offline self.attempt += 1 if self.attempt >= max_errors: self.is_active = False -- cgit From 46269872192b843c80d72206a05c8b759c9f66a8 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 17:32:25 -0400 Subject: merge from dev --- nova/tests/api/openstack/test_zones.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 6c06fa8b8..65cc1c023 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -62,7 +62,7 @@ def zone_get_all_scheduler(x, y, z): def zone_get_all_scheduler_empty(x, y, z): - return [] + return [] def zone_get_all_db(context): @@ -106,7 +106,7 @@ class ZonesTest(unittest.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_list_db(self): - self.stubs.Set(zones.Controller, '_call_scheduler', + self.stubs.Set(zones.Controller, '_call_scheduler', zone_get_all_scheduler_empty) self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) req = webob.Request.blank('/v1.0/zones') @@ -116,7 +116,6 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(len(res_dict['zones']), 2) - def test_get_zone_by_id(self): req = webob.Request.blank('/v1.0/zones/1') res = req.get_response(fakes.wsgi_app()) -- cgit From e5443fa3e436a95de9d1c353e8772436c7cba8b6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 17:58:38 -0400 Subject: pip requires novatools --- tools/pip-requires | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/pip-requires b/tools/pip-requires index 3587df644..a5d4675be 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -10,6 +10,7 @@ boto==1.9b carrot==0.10.5 eventlet==0.9.12 lockfile==0.8 +python-novatools==2.0 python-daemon==1.5.5 python-gflags==1.3 redis==2.0.0 -- cgit From e5d979596ff8c588c7bbe82b7f1cb90de8af041a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Feb 2011 18:49:30 -0400 Subject: missing docstring and fixed copyrights --- nova/scheduler/zone_manager.py | 3 +-- nova/tests/test_zones.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 4bf6e36c6..3e7c1eba8 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -1,6 +1,4 @@ # Copyright (c) 2010 Openstack, LLC. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -109,6 +107,7 @@ class ZoneManager(object): self.zone_states = {} def get_zone_list(self): + """Return the list of zones we know about.""" return [zone.to_dict() for zone in self.zone_states.values()] def _refresh_from_db(self, context): diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 7036ebe58..c273230a9 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -1,5 +1,4 @@ # Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may -- cgit From 3f3dddee0245cb143004dfb8c20204c511bec658 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Feb 2011 16:52:31 -0600 Subject: a few changes and a bunch of unit tests --- nova/api/openstack/servers.py | 16 ++-- .../sqlalchemy/migrate_repo/versions/003_cactus.py | 60 ------------- .../versions/004_add_instance_migrations.py | 60 +++++++++++++ nova/tests/api/openstack/common.py | 30 +++++++ nova/tests/api/openstack/test_servers.py | 98 +++++++++++++++++++++- 5 files changed, 197 insertions(+), 67 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py create mode 100644 nova/tests/api/openstack/common.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index fd6b10d5b..a719f5e15 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -227,7 +227,7 @@ class Controller(wsgi.Controller): self.compute_api.confirm_resize(req.environ['nova.context'], id) except Exception, e: LOG.exception(_("Error in confirm-resize %s"), e) - return faults.Fault(exc.HTTPBadRequest(e)) + return faults.Fault(exc.HTTPBadRequest()) return exc.HTTPNoContent() def _action_revert_resize(self, input_dict, req, id): @@ -235,7 +235,7 @@ class Controller(wsgi.Controller): self.compute_api.revert_resize(req.environ['nova.context'], id) except Exception, e: LOG.exception(_("Error in revert-resize %s"), e) - return faults.Fault(exc.HTTPBadRequest(e)) + return faults.Fault(exc.HTTPBadRequest()) return exc.HTTPAccepted() def _action_rebuild(self, input_dict, req, id): @@ -244,12 +244,16 @@ class Controller(wsgi.Controller): def _action_resize(self, input_dict, req, id): """ Resizes a given instance to the flavor size requested """ try: - flavor_id = input_dict['resize']['flavorId'] - self.compute_api.resize(req.environ['nova.context'], id, - flavor_id) + if 'resize' in input_dict and 'flavorId' in input_dict['resize']: + flavor_id = input_dict['resize']['flavorId'] + self.compute_api.resize(req.environ['nova.context'], id, + flavor_id) + else: + LOG.exception(_("Missing arguments for resize")) + return faults.Fault(exc.HTTPUnprocessableEntity()) except Exception, e: LOG.exception(_("Error in resize %s"), e) - return faults.Fault(exc.HTTPUnprocessableEntity(e)) + return faults.Fault(exc.HTTPBadRequest()) return faults.Fault(exc.HTTPAccepted()) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py b/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py deleted file mode 100644 index 4aab5bdc6..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/003_cactus.py +++ /dev/null @@ -1,60 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License.from sqlalchemy import * - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -# Just for the ForeignKey and column creation to succeed, these are not the -# actual definitions of instances or services. -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -# -# New Tables -# - -migrations = Table('migrations', meta, - Column('created_at', DateTime(timezone=False)), - Column('updated_at', DateTime(timezone=False)), - Column('deleted_at', DateTime(timezone=False)), - Column('deleted', Boolean(create_constraint=True, name=None)), - Column('id', Integer(), primary_key=True, nullable=False), - Column('source_compute', String(255)), - Column('dest_compute', String(255)), - Column('dest_host', String(255)), - Column('instance_id', Integer, ForeignKey('instances.id'), - nullable=True), - Column('status', String(255)) - ) - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - for table in (migrations, ): - try: - table.create() - except Exception: - logging.info(repr(table)) - logging.exception('Exception while creating table') - raise diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py new file mode 100644 index 000000000..4aab5bdc6 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py @@ -0,0 +1,60 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.from sqlalchemy import * + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +migrations = Table('migrations', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('source_compute', String(255)), + Column('dest_compute', String(255)), + Column('dest_host', String(255)), + Column('instance_id', Integer, ForeignKey('instances.id'), + nullable=True), + Column('status', String(255)) + ) + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (migrations, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py new file mode 100644 index 000000000..b55d3087b --- /dev/null +++ b/nova/tests/api/openstack/common.py @@ -0,0 +1,30 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import webob + +def webob_factory(url): + base_url = url + def web_request(url, method, body=None): + req = webob.Request.blank("%s%s" % (base_url, url)) + req.method = method + req.body = json.dumps(body) + return req + return web_request + diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..878b62f85 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 OpenStack LLC. +# Copyright 2010-2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -29,6 +29,7 @@ from nova.api.openstack import servers import nova.db.api from nova.db.sqlalchemy.models import Instance import nova.rpc +from nova.tests.api.openstack import common from nova.tests.api.openstack import fakes @@ -138,6 +139,8 @@ class ServersTest(unittest.TestCase): self.stubs.Set(nova.compute.API, "get_actions", fake_compute_api) self.allow_admin = FLAGS.allow_admin_api + self.webreq = common.webob_factor('/v1.0/servers') + def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin @@ -411,6 +414,99 @@ class ServersTest(unittest.TestCase): self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) + def test_resize_server(self): + req = self.webreq('/1/action', 'POST', dict(resize={flavorId=3}})) + res = req.get_response(fakes.wsgi_app()) + + self.resize_called = False + def resize_mock(context, inst_id, flavor): + self.resize_called = True + + self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) + self.assertEqual(res.status_int, 202) + self.assertEqual(self.resize_called, True) + + def test_resize_bad_param_fails(self): + req = self.webreq('/1/action', 'POST', dict('ferp')) + res = req.get_response(fakes.wsgi_app()) + + self.resize_called = False + def resize_mock(context, inst_id, flavor): + self.resize_called = True + + self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) + self.assertEqual(res.status_int, 422) + self.assertEqual(self.resize_called, False) + + def test_resize_bad_flavor_fails(self): + req = self.webreq('/1/action', 'POST', dict(resize={derp=3})) + res = req.get_response(fakes.wsgi_app()) + + self.resize_called = False + def resize_mock(context, inst_id, flavor): + self.resize_called = True + + self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) + self.assertEqual(res.status_int, 422) + self.assertEqual(self.resize_called, False) + + def test_resize_raises_fails(self): + req = self.webreq('/1/action', 'POST', dict(resize={flavorId=3})) + res = req.get_response(fakes.wsgi_app()) + + def resize_mock(context, inst_id, flavor): + raise Exception, 'hurr durr' + + self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) + self.assertEqual(res.status_int, 500) + + def test_confirm_resize_server(self): + req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) + res = req.get_response(fakes.wsgi_app()) + + self.resize_called = False + def confirm_resize_mock(context, inst_id): + self.resize_called = True + + self.stubs.Set(nova.compute.api, 'confirm_resize', + confirm_resize_mock) + self.assertEqual(res.status_int, 202) + self.assertEqual(self.resize_called, True) + + def test_confirm_resize_server_fails(self): + req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) + res = req.get_response(fakes.wsgi_app()) + + def confirm_resize_mock(context, inst_id): + raise Exception, 'hurr durr' + + self.stubs.Set(nova.compute.api, 'confirm_resize', + confirm_resize_mock) + self.assertEqual(res.status_int, 500) + + def test_revert_resize_server(self): + req = self.webreq('/1/action', 'POST', dict(revertResize=None)) + res = req.get_response(fakes.wsgi_app()) + + self.resize_called = False + def revert_resize_mock(context, inst_id): + self.resize_called = True + + self.stubs.Set(nova.compute.api, 'revert_resize', + confirm_resize_mock) + self.assertEqual(res.status_int, 202) + self.assertEqual(self.resize_called, True) + + def test_revert_resize_server_fails(self): + req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) + res = req.get_response(fakes.wsgi_app()) + + def revert_resize_mock(context, inst_id): + raise Exception, 'hurr durr' + + self.stubs.Set(nova.compute.api, 'revert_resize', + confirm_resize_mock) + self.assertEqual(res.status_int, 500) if __name__ == "__main__": unittest.main() -- cgit From 88aa545b53d96c25da01218c79e8be8c1ae3370f Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Thu, 17 Feb 2011 23:55:56 +0000 Subject: Test changes --- nova/tests/api/openstack/test_servers.py | 82 +++++++++++++++----------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 878b62f85..665551c55 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -26,6 +26,7 @@ from nova import db from nova import flags import nova.api.openstack from nova.api.openstack import servers +import nova.compute.api import nova.db.api from nova.db.sqlalchemy.models import Instance import nova.rpc @@ -139,7 +140,7 @@ class ServersTest(unittest.TestCase): self.stubs.Set(nova.compute.API, "get_actions", fake_compute_api) self.allow_admin = FLAGS.allow_admin_api - self.webreq = common.webob_factor('/v1.0/servers') + self.webreq = common.webob_factory('/v1.0/servers') def tearDown(self): self.stubs.UnsetAll() @@ -415,98 +416,93 @@ class ServersTest(unittest.TestCase): self.assertEqual(self.server_delete_called, True) def test_resize_server(self): - req = self.webreq('/1/action', 'POST', dict(resize={flavorId=3}})) - res = req.get_response(fakes.wsgi_app()) + req = self.webreq('/1/action', 'POST', dict(resize=dict(flavorId=3))) self.resize_called = False - def resize_mock(context, inst_id, flavor): + def resize_mock(*args): self.resize_called = True - self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) + self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) + + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) self.assertEqual(self.resize_called, True) - def test_resize_bad_param_fails(self): - req = self.webreq('/1/action', 'POST', dict('ferp')) - res = req.get_response(fakes.wsgi_app()) + def test_resize_bad_flavor_fails(self): + req = self.webreq('/1/action', 'POST', dict(resize=dict(derp=3))) self.resize_called = False - def resize_mock(context, inst_id, flavor): + def resize_mock(*args): self.resize_called = True - self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) - self.assertEqual(res.status_int, 422) - self.assertEqual(self.resize_called, False) + self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) - def test_resize_bad_flavor_fails(self): - req = self.webreq('/1/action', 'POST', dict(resize={derp=3})) res = req.get_response(fakes.wsgi_app()) - - self.resize_called = False - def resize_mock(context, inst_id, flavor): - self.resize_called = True - - self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) self.assertEqual(res.status_int, 422) self.assertEqual(self.resize_called, False) def test_resize_raises_fails(self): - req = self.webreq('/1/action', 'POST', dict(resize={flavorId=3})) - res = req.get_response(fakes.wsgi_app()) + req = self.webreq('/1/action', 'POST', dict(resize=dict(flavorId=3))) - def resize_mock(context, inst_id, flavor): + def resize_mock(*args): raise Exception, 'hurr durr' - self.stubs.Set(nova.compute.api, 'resize_instance', resize_mock) - self.assertEqual(res.status_int, 500) + self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) def test_confirm_resize_server(self): req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) - res = req.get_response(fakes.wsgi_app()) self.resize_called = False - def confirm_resize_mock(context, inst_id): + def confirm_resize_mock(*args): self.resize_called = True - self.stubs.Set(nova.compute.api, 'confirm_resize', + self.stubs.Set(nova.compute.api.API, 'confirm_resize', confirm_resize_mock) - self.assertEqual(res.status_int, 202) + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 204) self.assertEqual(self.resize_called, True) def test_confirm_resize_server_fails(self): req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) - res = req.get_response(fakes.wsgi_app()) - def confirm_resize_mock(context, inst_id): + def confirm_resize_mock(*args): raise Exception, 'hurr durr' - self.stubs.Set(nova.compute.api, 'confirm_resize', + self.stubs.Set(nova.compute.api.API, 'confirm_resize', confirm_resize_mock) - self.assertEqual(res.status_int, 500) + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) def test_revert_resize_server(self): req = self.webreq('/1/action', 'POST', dict(revertResize=None)) - res = req.get_response(fakes.wsgi_app()) self.resize_called = False - def revert_resize_mock(context, inst_id): + def revert_resize_mock(*args): self.resize_called = True - self.stubs.Set(nova.compute.api, 'revert_resize', - confirm_resize_mock) + self.stubs.Set(nova.compute.api.API, 'revert_resize', + revert_resize_mock) + + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) self.assertEqual(self.resize_called, True) def test_revert_resize_server_fails(self): - req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) - res = req.get_response(fakes.wsgi_app()) + req = self.webreq('/1/action', 'POST', dict(revertResize=None)) - def revert_resize_mock(context, inst_id): + def revert_resize_mock(*args): raise Exception, 'hurr durr' - self.stubs.Set(nova.compute.api, 'revert_resize', - confirm_resize_mock) - self.assertEqual(res.status_int, 500) + self.stubs.Set(nova.compute.api.API, 'revert_resize', + revert_resize_mock) + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) if __name__ == "__main__": unittest.main() -- cgit From 4b51ec3e9bca7421c66816c77c43396e51e68ea6 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 17 Feb 2011 23:09:06 -0600 Subject: Tests --- nova/tests/api/openstack/common.py | 8 +++++--- nova/tests/test_compute.py | 7 +++++++ nova/virt/fake.py | 13 +++++++++++++ nova/virt/xenapi_conn.py | 8 -------- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py index b55d3087b..66207cddc 100644 --- a/nova/tests/api/openstack/common.py +++ b/nova/tests/api/openstack/common.py @@ -21,10 +21,12 @@ import webob def webob_factory(url): base_url = url - def web_request(url, method, body=None): + def web_request(url, method=None, body=None): req = webob.Request.blank("%s%s" % (base_url, url)) - req.method = method - req.body = json.dumps(body) + if method: + req.method = method + if body: + req.body = json.dumps(body) return req return web_request diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 2aa0690e7..e27e08827 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -258,3 +258,10 @@ class ComputeTestCase(test.TestCase): self.assertEqual(ret_val, None) self.compute.terminate_instance(self.context, instance_id) + + def test_resize_instance(self): + """Ensure instance can be migrated/resized""" + instance_id = self._create_instance() + self.compute.run_instnce(self.context, instance_id) + self.compute.prep_resize(self.context, instance_id) + diff --git a/nova/virt/fake.py b/nova/virt/fake.py index ff5e22603..da86df6d4 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -176,6 +176,19 @@ class FakeConnection(object): """ pass + def migrate_disk_and_power_off(self, instance, dest): + """ + Transfers the disk of a running instance in multiple phases, turning + off the instance before the end. + """ + pass + + def attach_disk(self, instance, disk_info): + """ + Attaches the disk to an instance given the metadata disk_info + """ + pass + def pause(self, instance, callback): """ Pause the specified instance. diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index aafd836e2..be018b47f 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -184,14 +184,6 @@ class XenAPIConnection(object): """Unpause paused VM instance""" self._vmops.unpause(instance, callback) - def power_off(self, instance): - """Shuts down a running VM instance""" - self._vmops._shutdown(instance, method='clean') - - def power_on(self, instance): - """powers on a powered off VM instance""" - self._vmops.power_on(instance) - def migrate_disk_and_power_off(self, instance, dest): """Transfers the VHD of a running instance to another host, then shuts off the instance copies over the COW disk""" -- cgit From 671766cb4ada59b0e575b395b5afff82950ddb76 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Fri, 18 Feb 2011 06:03:15 +0000 Subject: Resize compute tests --- nova/tests/test_compute.py | 24 +++++++++++++++++++++--- nova/virt/fake.py | 6 ++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index e27e08827..3f2e64c87 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -56,7 +56,7 @@ class ComputeTestCase(test.TestCase): self.manager.delete_project(self.project) super(ComputeTestCase, self).tearDown() - def _create_instance(self): + def _create_instance(self, params={}): """Create a test instance""" inst = {} inst['image_id'] = 'ami-test' @@ -67,6 +67,7 @@ class ComputeTestCase(test.TestCase): inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() inst['ami_launch_index'] = 0 + inst.update(params) return db.instance_create(self.context, inst)['id'] def _create_group(self): @@ -262,6 +263,23 @@ class ComputeTestCase(test.TestCase): def test_resize_instance(self): """Ensure instance can be migrated/resized""" instance_id = self._create_instance() - self.compute.run_instnce(self.context, instance_id) - self.compute.prep_resize(self.context, instance_id) + context = self.context.elevated() + self.compute.run_instance(self.context, instance_id) + db.instance_update(self.context, instance_id, {'host':'foo'}) + + self.compute.prep_resize(context, instance_id) + migration_ref = db.migration_get_by_instance_and_status(context, + instance_id, 'pre-migrating') + self.compute.resize_instance(context, instance_id, + migration_ref['id']) + self.compute.terminate_instance(context, instance_id) + + def test_resize_same_source_fails(self): + """Ensure instance fails to migrate when source and destination are + the same host""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.assertRaises(exception.Error, self.compute.prep_resize, + self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index da86df6d4..9106ebf03 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -139,6 +139,12 @@ class FakeConnection(object): """ pass + def get_host_ip_addr(self): + """ + Retrieves the IP address of the dom0 + """ + pass + def resize(self, instance, flavor): """ Resizes/Migrates the specified instance. -- cgit From c884064e7a9af04b2ebdbbb9ee32318a00716412 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Feb 2011 12:08:35 -0400 Subject: fixups backed on merge comments --- nova/api/openstack/zones.py | 30 +++++---------------- nova/flags.py | 5 ++-- nova/scheduler/api.py | 49 ++++++++++++++++++++++++++++++++++ nova/scheduler/zone_manager.py | 15 ++++++----- nova/tests/api/openstack/test_zones.py | 33 +++++++++++------------ 5 files changed, 82 insertions(+), 50 deletions(-) create mode 100644 nova/scheduler/api.py diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index f75176824..24a4444f7 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -1,4 +1,4 @@ -# Copyright 2010 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -20,6 +20,7 @@ from nova import flags from nova import wsgi from nova import db from nova import rpc +from nova.scheduler.api import API FLAGS = flags.FLAGS @@ -49,31 +50,14 @@ class Controller(wsgi.Controller): "attributes": { "zone": ["id", "api_url", "name", "capabilities"]}}} - def _call_scheduler(self, method, context, params=None): - """Generic handler for RPC calls to the scheduler. - - :param params: Optional dictionary of arguments to be passed to the - scheduler worker - - :retval: Result returned by scheduler worker - """ - if not params: - params = {} - queue = FLAGS.scheduler_topic - kwargs = {'method': method, 'args': params} - return rpc.call(context, queue, kwargs) - def index(self, req): """Return all zones in brief""" - # Ask the ZoneManager in the Scheduler for most recent data. - items = self._call_scheduler('get_zone_list', - req.environ['nova.context']) - for item in items: - item['api_url'] = item['api_url'].replace('\\/', '/') - - # Or fall-back to the database ... - if len(items) == 0: + # Ask the ZoneManager in the Scheduler for most recent data, + # or fall-back to the database ... + items = API().get_zone_list(req.environ['nova.context']) + if not items: items = db.zone_get_all(req.environ['nova.context']) + items = common.limited(items, req) items = [_exclude_keys(item, ['username', 'password']) for item in items] diff --git a/nova/flags.py b/nova/flags.py index 7e4919d6e..41f01fcd7 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -316,6 +316,5 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'xen, linux', - 'comma-delimited list of tags which represent boolean' - ' capabilities of this zone') +DEFINE_string('zone_capabilities', 'kypervisor:xenserver;os:linux', + 'Key/Value tags which represent capabilities of this zone') diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py new file mode 100644 index 000000000..8491bf3a9 --- /dev/null +++ b/nova/scheduler/api.py @@ -0,0 +1,49 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Handles all requests relating to schedulers. +""" + +from nova import flags +from nova import log as logging +from nova import rpc + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.scheduler.api') + + +class API: + """API for interacting with the scheduler.""" + + def _call_scheduler(self, method, context, params=None): + """Generic handler for RPC calls to the scheduler. + + :param params: Optional dictionary of arguments to be passed to the + scheduler worker + + :retval: Result returned by scheduler worker + """ + if not params: + params = {} + queue = FLAGS.scheduler_topic + kwargs = {'method': method, 'args': params} + return rpc.call(context, queue, kwargs) + + def get_zone_list(self, context): + items = self._call_scheduler('get_zone_list', context) + for item in items: + item['api_url'] = item['api_url'].replace('\\/', '/') + return items diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 3e7c1eba8..783783d06 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010 Openstack, LLC. +# Copyright (c) 2011 Openstack, LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -87,8 +87,8 @@ class ZoneState(object): def _call_novatools(zone): """Call novatools. Broken out for testing purposes.""" - os = novatools.OpenStack(zone.username, zone.password, zone.api_url) - return os.zones.info()._info + client = novatools.OpenStack(zone.username, zone.password, zone.api_url) + return client.zones.info()._info def _poll_zone(zone): @@ -105,6 +105,7 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} + self.green_pool = GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" @@ -123,20 +124,20 @@ class ZoneManager(object): self.zone_states[zone.id].update_credentials(zone) # Cleanup zones removed from db ... - for zone_id in self.zone_states.keys(): + keys = self.zone_states.keys() # since we're deleting + for zone_id in keys: if zone_id not in db_keys: del self.zone_states[zone_id] def _poll_zones(self, context): """Try to connect to each child zone and get update.""" - green_pool = GreenPool() - green_pool.imap(_poll_zone, self.zone_states.values()) + self.green_pool.imap(_poll_zone, self.zone_states.values()) def ping(self, context=None): """Ping should be called periodically to update zone status.""" diff = datetime.now() - self.last_zone_db_check if diff.seconds >= FLAGS.zone_db_check_interval: - logging.debug("Updating zone cache from db.") + logging.debug(_("Updating zone cache from db.")) self.last_zone_db_check = datetime.now() self._refresh_from_db(context) self._poll_zones(context) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 65cc1c023..4df7c7feb 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -1,4 +1,4 @@ -# Copyright 2010 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -24,6 +24,7 @@ from nova import context from nova import flags from nova.api.openstack import zones from nova.tests.api.openstack import fakes +from nova.scheduler.api import API FLAGS = flags.FLAGS @@ -31,7 +32,7 @@ FLAGS.verbose = True def zone_get(context, zone_id): - return dict(id=1, api_url='http://foo.com', username='bob', + return dict(id=1, api_url='http://example.com', username='bob', password='xxx') @@ -42,7 +43,7 @@ def zone_create(context, values): def zone_update(context, zone_id, values): - zone = dict(id=zone_id, api_url='http://foo.com', username='bob', + zone = dict(id=zone_id, api_url='http://example.com', username='bob', password='xxx') zone.update(values) return zone @@ -52,24 +53,24 @@ def zone_delete(context, zone_id): pass -def zone_get_all_scheduler(x, y, z): +def zone_get_all_scheduler(*args): return [ - dict(id=1, api_url='http://foo.com', username='bob', + dict(id=1, api_url='http://example.com', username='bob', password='xxx'), - dict(id=2, api_url='http://blah.com', username='alice', + dict(id=2, api_url='http://example.org', username='alice', password='qwerty') ] -def zone_get_all_scheduler_empty(x, y, z): +def zone_get_all_scheduler_empty(*args): return [] def zone_get_all_db(context): return [ - dict(id=1, api_url='http://foo.com', username='bob', + dict(id=1, api_url='http://example.com', username='bob', password='xxx'), - dict(id=2, api_url='http://blah.com', username='alice', + dict(id=2, api_url='http://example.org', username='alice', password='qwerty') ] @@ -96,8 +97,7 @@ class ZonesTest(unittest.TestCase): FLAGS.allow_admin_api = self.allow_admin def test_get_zone_list_scheduler(self): - self.stubs.Set(zones.Controller, '_call_scheduler', - zone_get_all_scheduler) + self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -106,8 +106,7 @@ class ZonesTest(unittest.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_list_db(self): - self.stubs.Set(zones.Controller, '_call_scheduler', - zone_get_all_scheduler_empty) + self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler_empty) self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) @@ -122,7 +121,7 @@ class ZonesTest(unittest.TestCase): res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) - self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') + self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('password' in res_dict['zone']) self.assertEqual(res.status_int, 200) @@ -133,7 +132,7 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res.status_int, 200) def test_zone_create(self): - body = dict(zone=dict(api_url='http://blah.zoo', username='fred', + body = dict(zone=dict(api_url='http://example.com', username='fred', password='fubar')) req = webob.Request.blank('/v1.0/zones') req.method = 'POST' @@ -144,7 +143,7 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res_dict['zone']['id'], 1) - self.assertEqual(res_dict['zone']['api_url'], 'http://blah.zoo') + self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('username' in res_dict['zone']) def test_zone_update(self): @@ -158,7 +157,7 @@ class ZonesTest(unittest.TestCase): self.assertEqual(res.status_int, 200) self.assertEqual(res_dict['zone']['id'], 1) - self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') + self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('username' in res_dict['zone']) -- cgit From bb5624258200f027320327a38c524c389979c97a Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Fri, 18 Feb 2011 19:04:57 +0000 Subject: Resize compute tests --- nova/tests/test_xenapi.py | 34 ++++++++++++++++++++++++++++++++++ nova/tests/xenapi/stubs.py | 25 +++++++++++++++++++++++-- nova/virt/xenapi/fake.py | 2 +- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 6b8efc9d8..ee4c68e6c 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -336,3 +336,37 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): def tearDown(self): super(XenAPIDiffieHellmanTestCase, self).tearDown() + +class XenAPIMigrateInstance(test.TestCase): + """ + Unit test for verifying migration-related actions + """ + def setUp(self): + super(XenAPIMigrateInstance, self).setUp() + self.stubs = stubout.StubOutForTesting() + FLAGS.target_host = '127.0.0.1' + FLAGS.xenapi_connection_url = 'test_url' + FLAGS.xenapi_connection_password = 'test_pass' + db_fakes.stub_out_db_instance_api(self.stubs) + stubs.stub_out_get_target(self.stubs) + xenapi_fake.reset() + self.values = {'name': 1, 'id': 1, + 'project_id': 'fake', + 'user_id': 'fake', + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } + stubs.stub_out_migration_methods(self.stubs) + + def test_migrate_disk_and_power_off(self): + FLAGS.target_host = '127.0.0.1' + FLAGS.xenapi_connection_url = 'test_url' + FLAGS.xenapi_connection_password = 'test_pass' + destination = '127.0.0.1' + instance = db.instance_create(self.values) + stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) + conn = xenapi_conn.get_connection(False) + conn.migrate_disk_and_power_off(instance, destination) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 624995ada..d1c367475 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -20,6 +20,7 @@ from nova.virt import xenapi_conn from nova.virt.xenapi import fake from nova.virt.xenapi import volume_utils from nova.virt.xenapi import vm_utils +from nova.virt.xenapi import vmops def stubout_instance_snapshot(stubs): @@ -170,8 +171,8 @@ class FakeSessionForVMTests(fake.SessionBase): def VM_destroy(self, session_ref, vm_ref): fake.destroy_vm(vm_ref) - - + + class FakeSessionForVolumeTests(fake.SessionBase): """ Stubs out a XenAPISession for Volume tests """ def __init__(self, uri): @@ -205,3 +206,23 @@ class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): def SR_forget(self, _1, ref): pass + +class FakeSessionForMigrationTests(fake.SessionBase): + """ Stubs out a XenAPISession for Migration tests """ + def __init__(self, uri): + super(FakeSessionForMigrationTests, self).__init__(uri) + + +class FakeSnapshot(vmops.VMOps): + def __getattr__(self, key): + return 'fake' + + def __exit__(self, type, value, traceback) + pass + +def fake_get_snapshot(self, instance): + return FakeSnapshot() + +def stub_out_migration_methods(stubs): + stubs.Set(vmops.VMOps, '_get_snapshot', + fake_get_snapshot) diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index 018d0dcd3..e1ae03e70 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -401,7 +401,7 @@ class SessionBase(object): field in _db_content[cls][ref]): return _db_content[cls][ref][field] - LOG.debuug(_('Raising NotImplemented')) + LOG.debug(_('Raising NotImplemented')) raise NotImplementedError( _('xenapi.fake does not have an implementation for %s or it has ' 'been called with the wrong number of arguments') % name) -- cgit From 5812a95736b9a16733b99700e8664dd29ae34def Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 18 Feb 2011 22:10:06 +0100 Subject: Introduce IptablesManager in linux_net. Port every use of iptables in linux_net to it. --- nova/network/linux_net.py | 287 ++++++++++++++++++++++++++++----------------- nova/tests/test_network.py | 59 ++++++++++ nova/utils.py | 61 ++++++---- nova/virt/libvirt_conn.py | 28 +++-- 4 files changed, 287 insertions(+), 148 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c1cbff7d8..3d267d941 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -17,6 +17,7 @@ Implements vlans, bridges, and iptables rules using linux utilities. """ +import inspect import os from nova import db @@ -25,7 +26,6 @@ from nova import flags from nova import log as logging from nova import utils - LOG = logging.getLogger("nova.linux_net") @@ -63,73 +63,168 @@ flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', 'dmz range that should be accepted') +binary_name = os.path.basename(inspect.stack()[-1][1]) + + +class IptablesRule(object): + def __init__(self, chain, rule, wrap=True): + self.chain = chain + self.rule = rule + self.wrap = wrap + + def __eq__(self, other): + return ((self.chain == other.chain) and + (self.rule == other.rule) and + (self.wrap == other.wrap)) + + def __ne__(self, other): + return ((self.chain != other.chain) or + (self.rule != other.rule) or + (self.wrap != other.wrap)) + + def __str__(self): + if self.wrap: + chain = '%s-%s' % (binary_name, self.chain) + else: + chain = self.chain + return '-A %s %s' % (chain, self.rule) + + +class IptablesTable(object): + def __init__(self): + self.rules = [] + self.chains = set() + + def add_chain(self, name): + self.chains.add(name) + + def remove_chain(self, name): + self.chains.remove(name) + + def add_rule(self, chain, rule, wrap=True): + if wrap and chain not in self.chains: + raise ValueError(_("Unknown chain: %r") % chain) + + self.rules.append(IptablesRule(chain, rule, wrap)) + + def remove_rule(self, chain, rule): + self.rules.remove(IptablesRule(chain, rule)) + +class IptablesManager(object): + def __init__(self, execute=None): + if not execute: + if FLAGS.fake_network: + self.execute = lambda *args, **kwargs: ('', '') + else: + self.execute = utils.execute + else: + self.execute = execute + + self.ipv4 = { 'filter': IptablesTable(), + 'nat': IptablesTable() } + self.ipv6 = { 'filter': IptablesTable(), + 'nat': IptablesTable() } + + self.ipv4['nat'].add_chain('SNATTING') + self.ipv4['nat'].add_rule('POSTROUTING', + '-j %s-SNATTING' % (binary_name,), + wrap=False) + + self.ipv4['filter'].add_chain('local') + self.ipv4['filter'].add_rule('FORWARD', + '-j %s-local' % (binary_name,), + wrap=False) + + # Wrap the builtin chains + builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'INPUT', + 'OUTPUT', 'POSTROUTING']} + + for table, chains in builtin_chains.iteritems(): + for chain in chains: + self.ipv4[table].add_chain(chain) + self.ipv4[table].add_rule(chain, + '-j %s-%s' % (binary_name, chain), + wrap=False) + + + def apply(self): + s = [('iptables', self.ipv4)] + if FLAGS.use_ipv6: + s += [('ip6tables', self.ipv6)] + + for cmd, tables in s: + for table in tables: + current_filter, _ = self.execute('sudo %s-save -t %s' % + (cmd, table), attempts=5) + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, tables[table]) + self.execute('sudo %s-restore' % (cmd,), + process_input='\n'.join(new_filter), + attempts=5) + + def modify_rules(self, current_lines, table, binary=None): + + chains = table.chains + rules = table.rules + + # Remove any trace of our rules + new_filter = filter(lambda l: '%s' % binary not in l, current_lines) + + seen_chains = False + for rules_index in range(len(new_filter)): + if not seen_chains: + if new_filter[rules_index].startswith(':'): + seen_chains = True + elif seen_chains == 1: + if not new_filter[rules_index].startswith(':'): + break + + new_filter[rules_index:rules_index] = [str(rule) for rule in rules] + new_filter[rules_index:rules_index] = [':%s-%s - [0:0]' % \ + (binary_name, name,) \ + for name in chains] + + return new_filter + + +iptables_manager = IptablesManager() + + def metadata_forward(): """Create forwarding rule for metadata""" - _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " - "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + iptables_manager.ipv4['nat'].add_rule("PREROUTING", + "-s 0.0.0.0/0 -d 169.254.169.254/32 " + "-p tcp -m tcp --dport 80 -j DNAT " + "--to-destination %s:%s" % \ + (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + iptables_manager.apply() def init_host(): """Basic networking setup goes here""" - if FLAGS.use_nova_chains: - _execute("sudo iptables -N nova_input", check_exit_code=False) - _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, - check_exit_code=False) - _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) - - _execute("sudo iptables -N nova_forward", check_exit_code=False) - _execute("sudo iptables -D FORWARD -j nova_forward", - check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _execute("sudo iptables -N nova_output", check_exit_code=False) - _execute("sudo iptables -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -A OUTPUT -j nova_output") - - _execute("sudo iptables -t nat -N nova_prerouting", - check_exit_code=False) - _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", - check_exit_code=False) - _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") - - _execute("sudo iptables -t nat -N nova_postrouting", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") - - _execute("sudo iptables -t nat -N nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") - - _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) - _execute("sudo iptables -t nat -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -t nat -A OUTPUT -j nova_output") - else: - # NOTE(vish): This makes it easy to ensure snatting rules always - # come after the accept rules in the postrouting chain - _execute("sudo iptables -t nat -N SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") - # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("SNATTING", "-t nat -s %s " - "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) + iptables_manager.ipv4['nat'].add_rule("SNATTING", + "-s %s -j SNAT --to-source %s" % \ + (FLAGS.fixed_range, + FLAGS.routing_source_ip)) + + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %s -j SNAT --to-source %s" % \ + (FLAGS.fixed_range, + FLAGS.routing_source_ip)) - _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % - (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % - {'range': FLAGS.fixed_range}) + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %s -d %s -j ACCEPT" % \ + (FLAGS.fixed_range, FLAGS.dmz_cidr)) + + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %(range)s -d %(range)s " + "-j ACCEPT" % \ + {'range': FLAGS.fixed_range}) + iptables_manager.apply() def bind_floating_ip(floating_ip, check_exit_code=True): @@ -147,32 +242,33 @@ def unbind_floating_ip(floating_ip): def ensure_vlan_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan""" - _confirm_rule("FORWARD", "-d %s -p udp --dport 1194 -j ACCEPT" % - private_ip) - _confirm_rule("PREROUTING", - "-t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (public_ip, port, private_ip)) + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "-d %s -p udp " + "--dport 1194 " + "-j ACCEPT" % private_ip) + iptables_manager.ipv4['nat'].add_rule("PREROUTING", + "-d %s -p udp " + "--dport %s -j DNAT --to %s:1194" % + (public_ip, port, private_ip)) + iptables_manager.apply() def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" - _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) - + for chain, rule in floating_forward_rules(floating_ip, fixed_ip): + iptables_manager.ipv4['nat'].add_rule(chain, rule) + iptables_manager.apply() def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" - _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + for chain, rule in floating_forward_rules(floating_ip, fixed_ip): + iptables_manager.ipv4['nat'].remove_rule(chain, rule) + iptables_manager.apply() +def floating_forward_rules(floating_ip, fixed_ip): + return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), + ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), + ("SNATTING", "-d %s -j DNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): """Create a vlan and bridge unless they already exist""" @@ -258,19 +354,12 @@ def ensure_bridge(bridge, interface, net_attrs=None): "enslave it to bridge %s.\n" % (interface, bridge)): raise exception.Error("Failed to add interface: %s" % err) - if FLAGS.use_nova_chains: - (out, err) = _execute("sudo iptables -N nova_forward", - check_exit_code=False) - if err != 'iptables: Chain already exists.\n': - # NOTE(vish): chain didn't exist link chain - _execute("sudo iptables -D FORWARD -j nova_forward", - check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) - _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) - _execute("sudo iptables -N nova-local", check_exit_code=False) - _confirm_rule("FORWARD", "-j nova-local") + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "--in-interface %s -j ACCEPT" % \ + bridge) + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "--out-interface %s -j ACCEPT" % \ + bridge) def get_dhcp_hosts(context, network_id): @@ -390,26 +479,6 @@ def _device_exists(device): return not err -def _confirm_rule(chain, cmd, append=False): - """Delete and re-add iptables rule""" - if FLAGS.use_nova_chains: - chain = "nova_%s" % chain.lower() - if append: - loc = "-A" - else: - loc = "-I" - _execute("sudo iptables --delete %s %s" % (chain, cmd), - check_exit_code=False) - _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) - - -def _remove_rule(chain, cmd): - """Remove iptables rule""" - if FLAGS.use_nova_chains: - chain = "%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd)) - - def _dnsmasq_cmd(net): """Builds dnsmasq command""" cmd = ['sudo -E dnsmasq', diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..b28d64245 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -29,11 +29,70 @@ from nova import log as logging from nova import test from nova import utils from nova.auth import manager +from nova.network import linux_net FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.network') +class IptablesManagerTestCase(test.TestCase): + sample_filter = """# Completed on Fri Feb 18 15:17:05 2011 +# Generated by iptables-save v1.4.10 on Fri Feb 18 15:17:05 2011 +*filter +:INPUT ACCEPT [2223527:305688874] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [2172501:140856656] +-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT +-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT +-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT +-A FORWARD -i virbr0 -o virbr0 -j ACCEPT +-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable +COMMIT +# Completed on Fri Feb 18 15:17:05 2011""" + + def setUp(self): + super(IptablesManagerTestCase, self).setUp() + self.manager = linux_net.IptablesManager() + + def test_rules_are_wrapped(self): + current_lines = self.sample_filter.split('\n') + + table = self.manager.ipv4['filter'] + table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager.modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' in new_lines) + + table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager.modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' not in new_lines) + + def test_wrapper_rules_in_place(self): + current_lines = self.sample_filter.split('\n') + + # TODO(soren): Add stuff for ipv6 + check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'INPUT', + 'OUTPUT', 'POSTROUTING']} } + + for ip_version in check_matrix: + ip = getattr(self.manager, 'ipv%d' % ip_version) + for table_name in ip: + table = ip[table_name] + new_lines = self.manager.modify_rules(current_lines, table) + for chain in check_matrix[ip_version][table_name]: + self.assertTrue(':run_tests.py-%s - [0:0]' % \ + (chain,) in new_lines) + self.assertTrue('-A %s -j run_tests.py-%s' % \ + (chain, chain) in new_lines) + print '\n'.join(new_lines) + + class NetworkTestCase(test.TestCase): """Test cases for network code""" def setUp(self): diff --git a/nova/utils.py b/nova/utils.py index ba71ebf39..bf3a4b098 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -124,32 +124,41 @@ def fetchfile(url, target): execute("curl --fail %s -o %s" % (url, target)) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) - env = os.environ.copy() - if addl_env: - env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) - result = None - if process_input != None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=obj.returncode, - stdout=stdout, - stderr=stderr, - cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one - greenthread.sleep(0) - return result +def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attempts=1): + while attempts > 0: + attempts -= 1 + try: + LOG.debug(_("Running cmd (subprocess): %s"), cmd) + env = os.environ.copy() + if addl_env: + env.update(addl_env) + obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + result = None + if process_input != None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() + if obj.returncode: + LOG.debug(_("Result was %s") % obj.returncode) + if check_exit_code and obj.returncode != 0: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=cmd) + # NOTE(termie): this appears to be necessary to let the subprocess call + # clean something up in between calls, without it two + # execute calls in a row hangs the second one + greenthread.sleep(0) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + greenthread.sleep(random.randint(50,300)/100) + pass def ssh_execute(ssh, cmd, process_input=None, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7548fff63..11b3acbf6 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -64,7 +64,6 @@ from nova.compute import power_state from nova.virt import disk from nova.virt import images -libvirt_semaphore = semaphore.Semaphore() libvirt = None libxml2 = None Template = None @@ -1239,19 +1238,22 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - with libvirt_semaphore: - current_filter, _ = self.execute('sudo iptables-save -t filter') + current_filter, _ = self.execute('sudo iptables-save -t filter', + attempts=5) + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, 4) + self.execute('sudo iptables-restore', + process_input='\n'.join(new_filter), + attempts=5) + if(FLAGS.use_ipv6): + current_filter, _ = self.execute('sudo ip6tables-save ' + '-t filter', + attempts=5) current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter)) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save ' - '-t filter') - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', - process_input='\n'.join(new_filter)) + new_filter = self.modify_rules(current_lines, 6) + self.execute('sudo ip6tables-restore', + process_input='\n'.join(new_filter), + attempts=5) def modify_rules(self, current_lines, ip_version=4): ctxt = context.get_admin_context() -- cgit From 62b3eb71384581e900b061e65caa6418c4452fa9 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Fri, 18 Feb 2011 21:37:57 +0000 Subject: XenAPI tests --- nova/tests/test_xenapi.py | 12 +++++++----- nova/tests/xenapi/stubs.py | 38 +++++++++++++++++++++++++++++--------- nova/virt/xenapi/fake.py | 3 +++ 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index ee4c68e6c..3cbc01e5c 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -362,11 +362,13 @@ class XenAPIMigrateInstance(test.TestCase): stubs.stub_out_migration_methods(self.stubs) def test_migrate_disk_and_power_off(self): - FLAGS.target_host = '127.0.0.1' - FLAGS.xenapi_connection_url = 'test_url' - FLAGS.xenapi_connection_password = 'test_pass' - destination = '127.0.0.1' instance = db.instance_create(self.values) stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) conn = xenapi_conn.get_connection(False) - conn.migrate_disk_and_power_off(instance, destination) + conn.migrate_disk_and_power_off(instance, '127.0.0.1') + + def test_attach_disk(self): + instance = db.instance_create(self.values) + stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) + conn = xenapi_conn.get_connection(False) + conn.attach_disk(instance, {'base_copy': 'hurr', 'cow': 'durr'}) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index d1c367475..054fc434b 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -213,16 +213,36 @@ class FakeSessionForMigrationTests(fake.SessionBase): super(FakeSessionForMigrationTests, self).__init__(uri) -class FakeSnapshot(vmops.VMOps): - def __getattr__(self, key): - return 'fake' +def stub_out_migration_methods(stubs): + class FakeSnapshot(object): + def __getattr__(self, key): + return str(key) + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + pass + + def fake_get_snapshot(self, instance): + return FakeSnapshot() - def __exit__(self, type, value, traceback) + @classmethod + def fake_get_vdi(cls, session, vm_ref): + vdi_ref = fake.create_vdi(name_label='derp', read_only=False, + sr_ref='herp', sharable=False) + vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) + return vdi_ref, {'uuid': vdi_rec['uuid']} + + def fake_shutdown(self, inst, vm, method='clean'): pass -def fake_get_snapshot(self, instance): - return FakeSnapshot() + @classmethod + def fake_scan_sr(cls, session): + pass -def stub_out_migration_methods(stubs): - stubs.Set(vmops.VMOps, '_get_snapshot', - fake_get_snapshot) + stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_scan_sr) + stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot) + stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi) + stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x,y,z: None) + stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown) diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index e1ae03e70..ba12d4d3a 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -290,6 +290,9 @@ class SessionBase(object): #Always return 12GB available return 12 * 1024 * 1024 * 1024 + def host_call_plugin(*args): + return 'herp' + def xenapi_request(self, methodname, params): if methodname.startswith('login'): self._login(methodname, params) -- cgit From 18e573a14414838f11e772edca3eb5510f852c94 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Feb 2011 17:45:57 -0400 Subject: sandy y u no read hacking guide and import classes? --- nova/api/openstack/zones.py | 5 ++--- nova/scheduler/manager.py | 4 ++-- nova/scheduler/zone_manager.py | 4 ++-- nova/tests/api/openstack/test_zones.py | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 24a4444f7..99be0ba02 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -19,8 +19,7 @@ import logging from nova import flags from nova import wsgi from nova import db -from nova import rpc -from nova.scheduler.api import API +from nova.scheduler import api FLAGS = flags.FLAGS @@ -54,7 +53,7 @@ class Controller(wsgi.Controller): """Return all zones in brief""" # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... - items = API().get_zone_list(req.environ['nova.context']) + items = api.API().get_zone_list(req.environ['nova.context']) if not items: items = db.zone_get_all(req.environ['nova.context']) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 7ced33b9c..c94397210 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -29,7 +29,7 @@ from nova import log as logging from nova import manager from nova import rpc from nova import utils -from nova.scheduler.zone_manager import ZoneManager +from nova.scheduler import zone_manager LOG = logging.getLogger('nova.scheduler.manager') FLAGS = flags.FLAGS @@ -44,7 +44,7 @@ class SchedulerManager(manager.Manager): if not scheduler_driver: scheduler_driver = FLAGS.scheduler_driver self.driver = utils.import_object(scheduler_driver) - self.zone_manager = ZoneManager() + self.zone_manager = zone_manager.ZoneManager() super(SchedulerManager, self).__init__(*args, **kwargs) def __getattr__(self, key): diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 783783d06..758c5e3db 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -22,7 +22,7 @@ import thread import traceback from datetime import datetime -from eventlet.greenpool import GreenPool +from eventlet import greenpool from nova import db from nova import flags @@ -105,7 +105,7 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} - self.green_pool = GreenPool() + self.green_pool = greenpool.GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 4df7c7feb..4374cd457 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -24,7 +24,7 @@ from nova import context from nova import flags from nova.api.openstack import zones from nova.tests.api.openstack import fakes -from nova.scheduler.api import API +from nova.scheduler import api FLAGS = flags.FLAGS @@ -97,7 +97,7 @@ class ZonesTest(unittest.TestCase): FLAGS.allow_admin_api = self.allow_admin def test_get_zone_list_scheduler(self): - self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler) + self.stubs.Set(api.API, '_call_scheduler', zone_get_all_scheduler) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -106,7 +106,8 @@ class ZonesTest(unittest.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_list_db(self): - self.stubs.Set(API, '_call_scheduler', zone_get_all_scheduler_empty) + self.stubs.Set(api.API, '_call_scheduler', + zone_get_all_scheduler_empty) self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) -- cgit From a43c5929de7ebf58eb9ecb8416ce3cf4194c176a Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 18 Feb 2011 16:13:34 -0600 Subject: Pep8 cleanup --- nova/api/openstack/servers.py | 13 ++-- nova/compute/api.py | 12 ++- nova/compute/manager.py | 85 +++++++++++----------- nova/db/api.py | 5 +- nova/db/sqlalchemy/api.py | 6 +- .../versions/004_add_instance_migrations.py | 3 +- nova/db/sqlalchemy/models.py | 5 +- nova/tests/api/openstack/common.py | 7 +- nova/tests/api/openstack/test_servers.py | 24 +++--- nova/tests/test_compute.py | 6 +- nova/tests/test_xenapi.py | 1 + nova/tests/xenapi/stubs.py | 11 +-- nova/virt/xenapi/vm_utils.py | 13 ++-- nova/virt/xenapi/vmops.py | 5 +- nova/virt/xenapi_conn.py | 6 +- 15 files changed, 104 insertions(+), 98 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a719f5e15..f68c97323 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -209,12 +209,12 @@ class Controller(wsgi.Controller): resize a server """ actions = { - 'reboot':self._action_reboot, - 'resize':self._action_resize, - 'confirmResize':self._action_confirm_resize, - 'revertResize':self._action_revert_resize, - 'rebuild':self._action_rebuild - } + 'reboot': self._action_reboot, + 'resize': self._action_resize, + 'confirmResize': self._action_confirm_resize, + 'revertResize': self._action_revert_resize, + 'rebuild': self._action_rebuild, + } input_dict = self._deserialize(req.body, req) for key in actions.keys(): @@ -256,7 +256,6 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPBadRequest()) return faults.Fault(exc.HTTPAccepted()) - def _action_reboot(self, input_dict, req, id): try: reboot_type = input_dict['reboot']['type'] diff --git a/nova/compute/api.py b/nova/compute/api.py index 371cbae5f..2eb0e0743 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -408,7 +408,7 @@ class API(base.Base): raise exception.Error(_("No finished migrations found for \ instance")) - params = { 'migration_id': migration_ref['id'] } + params = {'migration_id': migration_ref['id']} self._cast_compute_message('revert_resize', context, instance_id, migration_ref['dest_compute'], params=params) @@ -422,14 +422,12 @@ class API(base.Base): raise exception.Error(_("No finished migrations found for \ instance")) instance_ref = self.db.instance_get(context, instance_id) - - params = { 'migration_id': migration_ref['id'] } + params = {'migration_id': migration_ref['id']} self._cast_compute_message('confirm_resize', context, instance_id, migration_ref['source_compute'], params=params) - self.db.migration_update(context, migration_id, - { 'status': 'confirmed' }) - + self.db.migration_update(context, migration_id, + {'status': 'confirmed'}) self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) @@ -439,7 +437,7 @@ class API(base.Base): {"method": "prep_resize", "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, }},) - + def pause(self, context, instance_id): """Pause the given instance.""" self._cast_compute_message('pause_instance', context, instance_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 1e1c44663..3f6c359ba 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -379,7 +379,7 @@ class ComputeManager(manager.Manager): def _update_state_callback(self, context, instance_id, result): """Update instance state when async task completes.""" self._update_state(context, instance_id) - + @exception.wrap_exception @checks_instance_lock def confirm_resize(self, context, instance_id, migration_id): @@ -392,8 +392,8 @@ class ComputeManager(manager.Manager): @exception.wrap_exception @checks_instance_lock def revert_resize(self, context, instance_id, migration_id): - """Destroys the new instance on the destination machine, - reverts the model changes, and powers on the old + """Destroys the new instance on the destination machine, + reverts the model changes, and powers on the old instance on the source machine""" instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_get(context, migration_id) @@ -401,24 +401,23 @@ class ComputeManager(manager.Manager): #TODO(mdietz): we may want to split these into separate methods. if migration_ref['source_compute'] == FLAGS.host: self.driver.power_on(instance_ref) - self.db.migration_update(context, migration_id, - { 'status': 'reverted' }) + self.db.migration_update(context, migration_id, + {'status': 'reverted'}) else: self.driver.destroy(instance_ref) - topic = self.db.queue_get_for(context, FLAGS.compute_topic, + topic = self.db.queue_get_for(context, FLAGS.compute_topic, instance_ref['host']) - rpc.cast(context, topic, - { 'method': 'revert_resize', - 'args': { - 'migration_id': migration_ref['id'], - 'instance_id': instance_id, - }, + rpc.cast(context, topic, + {'method': 'revert_resize', + 'args': { + 'migration_id': migration_ref['id'], + 'instance_id': instance_id, }, }) @exception.wrap_exception @checks_instance_lock def prep_resize(self, context, instance_id): - """Initiates the process of moving a running instance to another + """Initiates the process of moving a running instance to another host, possibly changing the RAM and disk size in the process""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -426,21 +425,21 @@ class ComputeManager(manager.Manager): raise exception.Error(_( 'Migration error: destination same as source!')) - migration_ref = self.db.migration_create(context, - { 'instance_id': instance_id, - 'source_compute': instance_ref['host'], - 'dest_compute': FLAGS.host, - 'dest_host': self.driver.get_host_ip_addr(), - 'status': 'pre-migrating' }) - LOG.audit(_('instance %s: migrating to '), instance_id, context=context) - topic = self.db.queue_get_for(context, FLAGS.compute_topic, + migration_ref = self.db.migration_create(context, + {'instance_id': instance_id, + 'source_compute': instance_ref['host'], + 'dest_compute': FLAGS.host, + 'dest_host': self.driver.get_host_ip_addr(), + 'status': 'pre-migrating'}) + LOG.audit(_('instance %s: migrating to '), instance_id, + context=context) + topic = self.db.queue_get_for(context, FLAGS.compute_topic, instance_ref['host']) - rpc.cast(context, topic, - { 'method': 'resize_instance', - 'args': { - 'migration_id': migration_ref['id'], - 'instance_id': instance_id, - }, + rpc.cast(context, topic, + {'method': 'resize_instance', + 'args': { + 'migration_id': migration_ref['id'], + 'instance_id': instance_id, }, }) @exception.wrap_exception @@ -449,28 +448,26 @@ class ComputeManager(manager.Manager): """Starts the migration of a running instance to another host""" migration_ref = self.db.migration_get(context, migration_id) instance_ref = self.db.instance_get(context, instance_id) - self.db.migration_update(context, migration_id, - { 'status': 'migrating', }) + self.db.migration_update(context, migration_id, + {'status': 'migrating', }) - disk_info = self.driver.migrate_disk_and_power_off(instance_ref, + disk_info = self.driver.migrate_disk_and_power_off(instance_ref, migration_ref['dest_host']) - - self.db.migration_update(context, migration_id, - { 'status': 'post-migrating', }) + self.db.migration_update(context, migration_id, + {'status': 'post-migrating', }) - #TODO(mdietz): This is where we would update the VM record + #TODO(mdietz): This is where we would update the VM record #after resizing service = self.db.service_get_by_host_and_topic(context, migration_ref['dest_compute'], FLAGS.compute_topic) - topic = self.db.queue_get_for(context, FLAGS.compute_topic, + topic = self.db.queue_get_for(context, FLAGS.compute_topic, migration_ref['dest_compute']) - rpc.cast(context, topic, - { 'method': 'finish_resize', - 'args': { - 'migration_id': migration_id, - 'instance_id': instance_id, - 'disk_info': disk_info, - }, + rpc.cast(context, topic, + {'method': 'finish_resize', + 'args': { + 'migration_id': migration_id, + 'instance_id': instance_id, + 'disk_info': disk_info, }, }) @exception.wrap_exception @@ -486,8 +483,8 @@ class ComputeManager(manager.Manager): new_disk_info = self.driver.attach_disk(instance_ref, disk_info) self.driver.spawn(instance_ref, disk=new_disk_info) - self.db.migration_update(context, migration_id, - {'status': 'finished', }) + self.db.migration_update(context, migration_id, + {'status': 'finished', }) @exception.wrap_exception @checks_instance_lock diff --git a/nova/db/api.py b/nova/db/api.py index 5a9d49374..8706ef3d6 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -260,17 +260,20 @@ def floating_ip_get_by_address(context, address): #################### def migration_update(context, id, values): - """Update a migration instance""" + """Update a migration instance""" return IMPL.migration_update(context, id, values) + def migration_create(context, values): """Create a migration record""" return IMPL.migration_create(context, values) + def migration_get(context, migration_id): """Finds a migration by the id""" return IMPL.migration_get(context, migration_id) + def migration_get_by_instance_and_status(context, instance_id, status): """Finds a migration by the instance id its migrating""" return IMPL.migration_get_by_instance_and_status(context, instance_id, diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 210b53296..facb46b8b 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -156,6 +156,7 @@ def service_get_all_by_topic(context, topic): filter_by(topic=topic).\ all() + @require_admin_context def service_get_by_host_and_topic(context, host, topic): session = get_session() @@ -166,6 +167,7 @@ def service_get_by_host_and_topic(context, host, topic): filter_by(topic=topic).\ first() + @require_admin_context def service_get_all_by_host(context, host): session = get_session() @@ -1996,7 +1998,7 @@ def migration_get(context, id, session=None): result = session.query(models.Migration).\ filter_by(id=id).first() if not result: - raise exception.NotFound(_("No migration found with id %s") + raise exception.NotFound(_("No migration found with id %s") % migration_id) return result @@ -2008,7 +2010,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(instance_id=instance_id).\ filter_by(status=status).first() if not result: - raise exception.NotFound(_("No migration found with instance id %s") + raise exception.NotFound(_("No migration found with instance id %s") % migration_id) return result diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py index 4aab5bdc6..4fda525f1 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py @@ -44,9 +44,10 @@ migrations = Table('migrations', meta, Column('dest_host', String(255)), Column('instance_id', Integer, ForeignKey('instances.id'), nullable=True), - Column('status', String(255)) + Column('status', String(255)), ) + def upgrade(migrate_engine): # Upgrade operations go here. Don't create your own engine; # bind migrate_engine to your metadata diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 0140fbeab..b05f134b7 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -374,7 +374,8 @@ class Migration(BASE, NovaBase): dest_compute = Column(String(255)) dest_host = Column(String(255)) instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True) - status = Column(String(255)) #TODO(_cerberus_): enum + #TODO(_cerberus_): enum + status = Column(String(255)) class Network(BASE, NovaBase): @@ -559,7 +560,7 @@ def register_models(): Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, - Project, Certificate, ConsolePool, Console, + Project, Certificate, ConsolePool, Console, Migration) # , Image, Host engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py index 66207cddc..3f9c7d3cf 100644 --- a/nova/tests/api/openstack/common.py +++ b/nova/tests/api/openstack/common.py @@ -19,14 +19,17 @@ import json import webob + def webob_factory(url): + """Factory for removing duplicate webob code from tests""" + base_url = url + def web_request(url, method=None, body=None): - req = webob.Request.blank("%s%s" % (base_url, url)) + req = webob.Request.blank("%s%s" % (base_url, url)) if method: req.method = method if body: req.body = json.dumps(body) return req return web_request - diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 665551c55..4eb4a3c62 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -419,8 +419,9 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(resize=dict(flavorId=3))) self.resize_called = False + def resize_mock(*args): - self.resize_called = True + self.resize_called = True self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) @@ -432,8 +433,9 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(resize=dict(derp=3))) self.resize_called = False + def resize_mock(*args): - self.resize_called = True + self.resize_called = True self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) @@ -445,7 +447,7 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(resize=dict(flavorId=3))) def resize_mock(*args): - raise Exception, 'hurr durr' + raise Exception('hurr durr') self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) @@ -456,10 +458,11 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) self.resize_called = False + def confirm_resize_mock(*args): - self.resize_called = True + self.resize_called = True - self.stubs.Set(nova.compute.api.API, 'confirm_resize', + self.stubs.Set(nova.compute.api.API, 'confirm_resize', confirm_resize_mock) res = req.get_response(fakes.wsgi_app()) @@ -470,9 +473,9 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(confirmResize=None)) def confirm_resize_mock(*args): - raise Exception, 'hurr durr' + raise Exception('hurr durr') - self.stubs.Set(nova.compute.api.API, 'confirm_resize', + self.stubs.Set(nova.compute.api.API, 'confirm_resize', confirm_resize_mock) res = req.get_response(fakes.wsgi_app()) @@ -482,10 +485,11 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(revertResize=None)) self.resize_called = False + def revert_resize_mock(*args): self.resize_called = True - self.stubs.Set(nova.compute.api.API, 'revert_resize', + self.stubs.Set(nova.compute.api.API, 'revert_resize', revert_resize_mock) res = req.get_response(fakes.wsgi_app()) @@ -496,9 +500,9 @@ class ServersTest(unittest.TestCase): req = self.webreq('/1/action', 'POST', dict(revertResize=None)) def revert_resize_mock(*args): - raise Exception, 'hurr durr' + raise Exception('hurr durr') - self.stubs.Set(nova.compute.api.API, 'revert_resize', + self.stubs.Set(nova.compute.api.API, 'revert_resize', revert_resize_mock) res = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 3f2e64c87..5fd1ddaec 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -265,8 +265,7 @@ class ComputeTestCase(test.TestCase): instance_id = self._create_instance() context = self.context.elevated() self.compute.run_instance(self.context, instance_id) - db.instance_update(self.context, instance_id, {'host':'foo'}) - + db.instance_update(self.context, instance_id, {'host': 'foo'}) self.compute.prep_resize(context, instance_id) migration_ref = db.migration_get_by_instance_and_status(context, instance_id, 'pre-migrating') @@ -279,7 +278,6 @@ class ComputeTestCase(test.TestCase): the same host""" instance_id = self._create_instance() self.compute.run_instance(self.context, instance_id) - self.assertRaises(exception.Error, self.compute.prep_resize, + self.assertRaises(exception.Error, self.compute.prep_resize, self.context, instance_id) - self.compute.terminate_instance(self.context, instance_id) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 3cbc01e5c..cb9b6620a 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -337,6 +337,7 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): def tearDown(self): super(XenAPIDiffieHellmanTestCase, self).tearDown() + class XenAPIMigrateInstance(test.TestCase): """ Unit test for verifying migration-related actions diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 054fc434b..303c37eb9 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -171,8 +171,8 @@ class FakeSessionForVMTests(fake.SessionBase): def VM_destroy(self, session_ref, vm_ref): fake.destroy_vm(vm_ref) - - + + class FakeSessionForVolumeTests(fake.SessionBase): """ Stubs out a XenAPISession for Volume tests """ def __init__(self, uri): @@ -207,6 +207,7 @@ class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): def SR_forget(self, _1, ref): pass + class FakeSessionForMigrationTests(fake.SessionBase): """ Stubs out a XenAPISession for Migration tests """ def __init__(self, uri): @@ -232,8 +233,8 @@ def stub_out_migration_methods(stubs): vdi_ref = fake.create_vdi(name_label='derp', read_only=False, sr_ref='herp', sharable=False) vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) - return vdi_ref, {'uuid': vdi_rec['uuid']} - + return vdi_ref, {'uuid': vdi_rec['uuid'], } + def fake_shutdown(self, inst, vm, method='clean'): pass @@ -244,5 +245,5 @@ def stub_out_migration_methods(stubs): stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_scan_sr) stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot) stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi) - stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x,y,z: None) + stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None) stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 91e7339b1..436c88023 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -256,21 +256,18 @@ class VMHelper(HelperBase): else: num_vdis = len(vdi_refs) if num_vdis != 1: - raise Exception(_("Unexpected number of VDIs (%(num_vdis)s) found" + raise Exception( + _("Unexpected number of VDIs (%(num_vdis)s) found" " for VM %(vm_ref)s") % locals()) vdi_ref = vdi_refs[0] vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) return vdi_ref, vdi_rec - - - @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): """ Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI, - Snapshot VHD - """ + Snapshot VHD """ #TODO(sirp): Add quiesce and VSS locking support when Windows support # is added LOG.debug(_("Snapshotting VM %(vm_ref)s with label '%(label)s'...") @@ -284,7 +281,7 @@ class VMHelper(HelperBase): task = session.call_xenapi('Async.VM.snapshot', vm_ref, label) template_vm_ref = session.wait_for_task(instance_id, task) - template_vdi_rec = cls.get_vdi_for_vm_safely(session, + template_vdi_rec = cls.get_vdi_for_vm_safely(session, template_vm_ref)[1] template_vdi_uuid = template_vdi_rec["uuid"] @@ -299,7 +296,7 @@ class VMHelper(HelperBase): @classmethod def get_sr(cls, session, sr_label='slices'): - """ Finds the SR named by the given name label and returns + """ Finds the SR named by the given name label and returns the UUID """ return session.call_xenapi('SR.get_by_name_label', sr_label)[0] diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 552c2ddd1..d457f2e3f 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -345,7 +345,7 @@ class VMOps(object): 'new_base_copy_uuid': new_base_copy_uuid, 'new_cow_uuid': new_cow_uuid, } - task = self._session.async_call_plugin('migration', + task = self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) self._session.wait_for_task(instance.id, task) @@ -469,7 +469,8 @@ class VMOps(object): vm = VMHelper.lookup(self._session, instance.name) return self._destroy(instance, vm, shutdown=True) - def _destroy(self, instance, vm, shutdown=True, destroy_kernel_ramdisk=True): + def _destroy(self, instance, vm, shutdown=True, + destroy_kernel_ramdisk=True): """ Destroys VM instance by performing: diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index be018b47f..e1c5dcc7c 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -192,7 +192,7 @@ class XenAPIConnection(object): def attach_disk(self, instance, disk_info): """Moves the copied VDIs into the SR""" return self._vmops.attach_disk(instance, disk_info) - + def suspend(self, instance, callback): """suspend the specified instance""" self._vmops.suspend(instance, callback) @@ -220,9 +220,9 @@ class XenAPIConnection(object): def get_ajax_console(self, instance): """Return link to instance's ajax console""" return self._vmops.get_ajax_console(instance) - + def get_host_ip_addr(self): - xs_url = urlparse.urlparse(FLAGS.xenapi_connection_url) + xs_url = urlparse.urlparse(FLAGS.xenapi_connection_url) return xs_url.netloc def attach_volume(self, instance_name, device_path, mountpoint): -- cgit From cfd6d4e403dcb2405cd7ff48bad3083a02159d2c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 00:14:08 +0100 Subject: Port libvirt_conn.IptablesDriver over to use linux_net.IptablesManager --- nova/network/linux_net.py | 17 +++- nova/tests/test_virt.py | 55 ++++++++---- nova/virt/libvirt_conn.py | 215 ++++++++++++++++++++-------------------------- 3 files changed, 145 insertions(+), 142 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 3d267d941..c11d34922 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -100,13 +100,23 @@ class IptablesTable(object): def remove_chain(self, name): self.chains.remove(name) + self.rules = filter(lambda r: r.chain != name, self.rules) def add_rule(self, chain, rule, wrap=True): if wrap and chain not in self.chains: raise ValueError(_("Unknown chain: %r") % chain) + if '$' in rule: + rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) + + print 'Adding rule: %r' % rule self.rules.append(IptablesRule(chain, rule, wrap)) + def _wrap_target_chain(self, s): + if s.startswith('$'): + return '%s-%s' % (binary_name, s[1:]) + return s + def remove_rule(self, chain, rule): self.rules.remove(IptablesRule(chain, rule)) @@ -122,8 +132,7 @@ class IptablesManager(object): self.ipv4 = { 'filter': IptablesTable(), 'nat': IptablesTable() } - self.ipv6 = { 'filter': IptablesTable(), - 'nat': IptablesTable() } + self.ipv6 = { 'filter': IptablesTable() } self.ipv4['nat'].add_chain('SNATTING') self.ipv4['nat'].add_rule('POSTROUTING', @@ -135,6 +144,10 @@ class IptablesManager(object): '-j %s-local' % (binary_name,), wrap=False) + self.ipv4['filter'].add_rule('OUTPUT', + '-j %s-local' % (binary_name,), + wrap=False) + # Wrap the builtin chains builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], 'nat': ['PREROUTING', 'INPUT', diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..a88e01818 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import re from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom @@ -233,16 +234,22 @@ class IptablesFirewallTestCase(test.TestCase): self.manager.delete_user(self.user) super(IptablesFirewallTestCase, self).tearDown() - in_rules = [ + in_nat_rules = [ + '# Generated by iptables-save v1.4.10 on Sat Feb 19 00:03:19 2011', + '*nat', + ':PREROUTING ACCEPT [1170:189210]', + ':INPUT ACCEPT [844:71028]', + ':OUTPUT ACCEPT [5149:405186]', + ':POSTROUTING ACCEPT [5063:386098]' + ] + + in_filter_rules = [ '# Generated by iptables-save v1.4.4 on Mon Dec 6 11:54:13 2010', '*filter', ':INPUT ACCEPT [969615:281627771]', ':FORWARD ACCEPT [0:0]', ':OUTPUT ACCEPT [915599:63811649]', ':nova-block-ipv4 - [0:0]', - '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', '-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED' ',ESTABLISHED -j ACCEPT ', @@ -254,7 +261,7 @@ class IptablesFirewallTestCase(test.TestCase): '# Completed on Mon Dec 6 11:54:13 2010', ] - in6_rules = [ + in6_filter_rules = [ '# Generated by ip6tables-save v1.4.4 on Tue Jan 18 23:47:56 2011', '*filter', ':INPUT ACCEPT [349155:75810423]', @@ -314,23 +321,31 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): + def fake_iptables_execute(cmd, process_input=None, attempts=5): if cmd == 'sudo ip6tables-save -t filter': - return '\n'.join(self.in6_rules), None + return '\n'.join(self.in6_filter_rules), None if cmd == 'sudo iptables-save -t filter': - return '\n'.join(self.in_rules), None + return '\n'.join(self.in_filter_rules), None + if cmd == 'sudo iptables-save -t nat': + return '\n'.join(self.in_nat_rules), None if cmd == 'sudo iptables-restore': - self.out_rules = process_input.split('\n') + lines = process_input.split('\n') + if '*filter' in lines: + self.out_rules = lines return '', '' if cmd == 'sudo ip6tables-restore': - self.out6_rules = process_input.split('\n') + lines = process_input.split('\n') + if '*filter' in lines: + self.out6_rules = lines return '', '' - self.fw.execute = fake_iptables_execute + + from nova.network import linux_net + linux_net.iptables_manager.execute = fake_iptables_execute self.fw.prepare_instance_filter(instance_ref) self.fw.apply_instance_filter(instance_ref) - in_rules = filter(lambda l: not l.startswith('#'), self.in_rules) + in_rules = filter(lambda l: not l.startswith('#'), self.in_filter_rules) for rule in in_rules: if not 'nova' in rule: self.assertTrue(rule in self.out_rules, @@ -338,6 +353,7 @@ class IptablesFirewallTestCase(test.TestCase): instance_chain = None for rule in self.out_rules: + print rule # This is pretty crude, but it'll do for now if '-d 10.11.12.13 -j' in rule: instance_chain = rule.split(' ')[-1] @@ -353,17 +369,18 @@ class IptablesFirewallTestCase(test.TestCase): self.assertTrue(security_group_chain, "The security group chain wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -j ACCEPT' % \ - security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP acceptance rule wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type ' - '8 -j ACCEPT' % security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -m icmp ' + '--icmp-type 8 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP Echo Request acceptance rule wasn't added") - self.assertTrue('-A %s -p tcp -s 192.168.10.0/24 -m multiport ' - '--dports 80:81 -j ACCEPT' % security_group_chain \ - in self.out_rules, + regex = re.compile('-A .* -p tcp -s 192.168.10.0/24 -m multiport ' + '--dports 80:81 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "TCP port 80/81 acceptance rule wasn't added") diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 11b3acbf6..976ccaca5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -57,7 +57,6 @@ from nova import exception from nova import flags from nova import log as logging from nova import utils -#from nova.api import context from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state @@ -1207,10 +1206,14 @@ class NWFilterFirewall(FirewallDriver): class IptablesFirewallDriver(FirewallDriver): def __init__(self, execute=None, **kwargs): - self.execute = execute or utils.execute + from nova.network import linux_net + self.iptables = linux_net.iptables_manager self.instances = {} self.nwfilter = NWFilterFirewall(kwargs['get_connection']) + self.iptables.ipv4['filter'].add_chain('sg-fallback') + self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') + def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" return self.nwfilter.setup_basic_filtering(instance) @@ -1226,124 +1229,92 @@ class IptablesFirewallDriver(FirewallDriver): LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) - def add_instance(self, instance): + def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance + chain_name = self._instance_chain_name(instance) + + self.iptables.ipv4['filter'].add_chain(chain_name) + ipv4_address = self._ip_for_instance(instance) + self.iptables.ipv4['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv4_address, chain_name)) + + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain(chain_name) + ipv6_address = self._ip_for_instance_v6(instance) + self.iptables.ipv4['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv6_address, + chain_name)) + + ipv4_rules, ipv6_rules = self.instance_rules(instance) + + for rule in ipv4_rules: + self.iptables.ipv4['filter'].add_rule(chain_name, rule) + + if FLAGS.use_ipv6: + for rule in ipv6_rules: + self.iptables.ipv6['filter'].add_rule(chain_name, rule) + + self.iptables.apply() + def unfilter_instance(self, instance): - self.remove_instance(instance) - self.apply_ruleset() + chain_name = self._instance_chain_name(instance) - def prepare_instance_filter(self, instance): - self.add_instance(instance) - self.apply_ruleset() + self.iptables.ipv4['filter'].remove_chain(chain_name) + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].remove_chain(chain_name) - def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter', - attempts=5) - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter), - attempts=5) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save ' - '-t filter', - attempts=5) - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', - process_input='\n'.join(new_filter), - attempts=5) - - def modify_rules(self, current_lines, ip_version=4): + self.iptables.apply() + + + def instance_rules(self, instance): ctxt = context.get_admin_context() - # Remove any trace of nova rules. - new_filter = filter(lambda l: 'nova-' not in l, current_lines) - - seen_chains = False - for rules_index in range(len(new_filter)): - if not seen_chains: - if new_filter[rules_index].startswith(':'): - seen_chains = True - elif seen_chains == 1: - if not new_filter[rules_index].startswith(':'): - break - our_chains = [':nova-fallback - [0:0]'] - our_rules = ['-A nova-fallback -j DROP'] - - our_chains += [':nova-local - [0:0]'] - our_rules += ['-A FORWARD -j nova-local'] - our_rules += ['-A OUTPUT -j nova-local'] - - security_groups = {} - # Add our chains - # First, we add instance chains and rules - for instance_id in self.instances: - instance = self.instances[instance_id] - chain_name = self._instance_chain_name(instance) - if(ip_version == 4): - ip_address = self._ip_for_instance(instance) - elif(ip_version == 6): - ip_address = self._ip_for_instance_v6(instance) - - our_chains += [':%s - [0:0]' % chain_name] - - # Jump to the per-instance chain - our_rules += ['-A nova-local -d %s -j %s' % (ip_address, - chain_name)] - - # Always drop invalid packets - our_rules += ['-A %s -m state --state ' - 'INVALID -j DROP' % (chain_name,)] - - # Allow established connections - our_rules += ['-A %s -m state --state ' - 'ESTABLISHED,RELATED -j ACCEPT' % (chain_name,)] - - # Jump to each security group chain in turn - for security_group in \ - db.security_group_get_by_instance(ctxt, - instance['id']): - security_groups[security_group['id']] = security_group - - sg_chain_name = self._security_group_chain_name( - security_group['id']) + ipv4_rules = [] + ipv6_rules = [] - our_rules += ['-A %s -j %s' % (chain_name, sg_chain_name)] - - if(ip_version == 4): - # Allow DHCP responses - dhcp_server = self._dhcp_server_for_instance(instance) - our_rules += ['-A %s -s %s -p udp --sport 67 --dport 68 ' - '-j ACCEPT ' % (chain_name, dhcp_server)] - #Allow project network traffic - if (FLAGS.allow_project_net_traffic): - cidr = self._project_cidr_for_instance(instance) - our_rules += ['-A %s -s %s -j ACCEPT' % (chain_name, cidr)] - elif(ip_version == 6): - # Allow RA responses - ra_server = self._ra_server_for_instance(instance) - if ra_server: - our_rules += ['-A %s -s %s -p icmpv6 -j ACCEPT' % - (chain_name, ra_server + "/128")] - #Allow project network traffic - if (FLAGS.allow_project_net_traffic): - cidrv6 = self._project_cidrv6_for_instance(instance) - our_rules += ['-A %s -s %s -j ACCEPT' % - (chain_name, cidrv6)] - - # If nothing matches, jump to the fallback chain - our_rules += ['-A %s -j nova-fallback' % (chain_name,)] + # Always drop invalid packets + ipv4_rules += ['-m state --state ' 'INVALID -j DROP'] + ipv6_rules += ['-m state --state ' 'INVALID -j DROP'] - # then, security group chains and rules - for security_group_id in security_groups: - chain_name = self._security_group_chain_name(security_group_id) - our_chains += [':%s - [0:0]' % chain_name] + # Allow established connections + ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + + dhcp_server = self._dhcp_server_for_instance(instance) + ipv4_rules += ['-s %s -p udp --sport 67 --dport 68 ' + '-j ACCEPT' % (dhcp_server,)] + + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidr = self._project_cidr_for_instance(instance) + ipv4_rules += ['-s %s -j ACCEPT' % (cidr,)] + + # We wrap these in FLAGS.use_ipv6 because they might cause + # a DB lookup. The other ones are just list operations, so + # they're not worth the clutter. + if FLAGS.use_ipv6: + # Allow RA responses + ra_server = self._ra_server_for_instance(instance) + if ra_server: + ipv6_rules += ['-s %s/128 -p icmpv6 -j ACCEPT' % (ra_server,)] - rules = \ - db.security_group_rule_get_by_security_group(ctxt, - security_group_id) + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidrv6 = self._project_cidrv6_for_instance(instance) + ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] + + + security_groups = db.security_group_get_by_instance(ctxt, + instance['id']) + + + # then, security group chains and rules + for security_group in security_groups: + rules = db.security_group_rule_get_by_security_group(ctxt, + security_group['id']) for rule in rules: logging.info('%r', rule) @@ -1354,14 +1325,16 @@ class IptablesFirewallDriver(FirewallDriver): continue version = _get_ip_version(rule.cidr) - if version != ip_version: - continue + if version == 4: + rules = ipv4_rules + else: + rules = ipv6_rules protocol = rule.protocol if version == 6 and rule.protocol == 'icmp': protocol = 'icmpv6' - args = ['-A', chain_name, '-p', protocol, '-s', rule.cidr] + args = ['-p', protocol, '-s', rule.cidr] if rule.protocol in ['udp', 'tcp']: if rule.from_port == rule.to_port: @@ -1382,20 +1355,20 @@ class IptablesFirewallDriver(FirewallDriver): icmp_type_arg += '/%s' % icmp_code if icmp_type_arg: - if(ip_version == 4): + if version == 4: args += ['-m', 'icmp', '--icmp-type', icmp_type_arg] - elif(ip_version == 6): + elif version == 6: args += ['-m', 'icmp6', '--icmpv6-type', icmp_type_arg] args += ['-j ACCEPT'] - our_rules += [' '.join(args)] + rules += [' '.join(args)] + + ipv4_rules += ['-j $fallback'] + ipv6_rules += ['-j $fallback'] - new_filter[rules_index:rules_index] = our_rules - new_filter[rules_index:rules_index] = our_chains - logging.info('new_filter: %s', '\n'.join(new_filter)) - return new_filter + return ipv4_rules, ipv6_rules def refresh_security_group_members(self, security_group): pass @@ -1407,7 +1380,7 @@ class IptablesFirewallDriver(FirewallDriver): return 'nova-sg-%s' % (security_group_id,) def _instance_chain_name(self, instance): - return 'nova-inst-%s' % (instance['id'],) + return 'inst-%s' % (instance['id'],) def _ip_for_instance(self, instance): return db.instance_get_fixed_address(context.get_admin_context(), -- cgit From 99760bd7a51371b29cf0f76134187dc81e7545d0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 00:30:44 +0100 Subject: Rename a few things for more clarity. --- nova/network/linux_net.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c11d34922..b657ab4bc 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -109,7 +109,6 @@ class IptablesTable(object): if '$' in rule: rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) - print 'Adding rule: %r' % rule self.rules.append(IptablesRule(chain, rule, wrap)) def _wrap_target_chain(self, s): @@ -168,9 +167,9 @@ class IptablesManager(object): for cmd, tables in s: for table in tables: - current_filter, _ = self.execute('sudo %s-save -t %s' % - (cmd, table), attempts=5) - current_lines = current_filter.split('\n') + current_table, _ = self.execute('sudo %s-save -t %s' % + (cmd, table), attempts=5) + current_lines = current_table.split('\n') new_filter = self.modify_rules(current_lines, tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), @@ -182,7 +181,7 @@ class IptablesManager(object): rules = table.rules # Remove any trace of our rules - new_filter = filter(lambda l: '%s' % binary not in l, current_lines) + new_filter = filter(lambda l: binary_name not in l, current_lines) seen_chains = False for rules_index in range(len(new_filter)): -- cgit From 23729c543350ce4ce563077522f18d0bedd1e61b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 00:36:34 +0100 Subject: Security group fallback is named sg-fallback. --- nova/virt/libvirt_conn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 976ccaca5..0ddf889a1 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1365,8 +1365,8 @@ class IptablesFirewallDriver(FirewallDriver): args += ['-j ACCEPT'] rules += [' '.join(args)] - ipv4_rules += ['-j $fallback'] - ipv6_rules += ['-j $fallback'] + ipv4_rules += ['-j $sg-fallback'] + ipv6_rules += ['-j $sg-fallback'] return ipv4_rules, ipv6_rules -- cgit From d0733621758985bdd621a05c7c8a53fe27aa62f2 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 01:28:26 +0100 Subject: Wrap iptables calls in a semaphore. --- nova/network/linux_net.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b81a704bf..ecda450bf 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -20,6 +20,8 @@ Implements vlans, bridges, and iptables rules using linux utilities. import inspect import os +from eventlet import semaphore + from nova import db from nova import exception from nova import flags @@ -149,8 +151,7 @@ class IptablesManager(object): # Wrap the builtin chains builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'INPUT', - 'OUTPUT', 'POSTROUTING']} + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']} for table, chains in builtin_chains.iteritems(): for chain in chains: @@ -158,22 +159,24 @@ class IptablesManager(object): self.ipv4[table].add_rule(chain, '-j %s-%s' % (binary_name, chain), wrap=False) + self.semaphore = semaphore.Semaphore() def apply(self): - s = [('iptables', self.ipv4)] - if FLAGS.use_ipv6: - s += [('ip6tables', self.ipv6)] - - for cmd, tables in s: - for table in tables: - current_table, _ = self.execute('sudo %s-save -t %s' % - (cmd, table), attempts=5) - current_lines = current_table.split('\n') - new_filter = self.modify_rules(current_lines, tables[table]) - self.execute('sudo %s-restore' % (cmd,), - process_input='\n'.join(new_filter), - attempts=5) + with self.semaphore: + s = [('iptables', self.ipv4)] + if FLAGS.use_ipv6: + s += [('ip6tables', self.ipv6)] + + for cmd, tables in s: + for table in tables: + current_table, _ = self.execute('sudo %s-save -t %s' % + (cmd, table), attempts=5) + current_lines = current_table.split('\n') + new_filter = self.modify_rules(current_lines, tables[table]) + self.execute('sudo %s-restore' % (cmd,), + process_input='\n'.join(new_filter), + attempts=5) def modify_rules(self, current_lines, table, binary=None): -- cgit From e729c49543c5acf354b154a3e2d9fd76a2f7da35 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 09:17:33 +0100 Subject: Fix refresh sec groups. --- nova/virt/libvirt_conn.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0ddf889a1..3faf01f4b 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1231,7 +1231,10 @@ class IptablesFirewallDriver(FirewallDriver): def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance + self.add_filters_for_instance(instance) + self.iptables.apply() + def add_filters_for_instance(self, instance): chain_name = self._instance_chain_name(instance) self.iptables.ipv4['filter'].add_chain(chain_name) @@ -1257,18 +1260,17 @@ class IptablesFirewallDriver(FirewallDriver): for rule in ipv6_rules: self.iptables.ipv6['filter'].add_rule(chain_name, rule) + def unfilter_instance(self, instance): + self.remove_filters_for_instance(instance) self.iptables.apply() - def unfilter_instance(self, instance): + def remove_filters_for_instance(self, instance): chain_name = self._instance_chain_name(instance) self.iptables.ipv4['filter'].remove_chain(chain_name) if FLAGS.use_ipv6: self.iptables.ipv6['filter'].remove_chain(chain_name) - self.iptables.apply() - - def instance_rules(self, instance): ctxt = context.get_admin_context() @@ -1374,7 +1376,10 @@ class IptablesFirewallDriver(FirewallDriver): pass def refresh_security_group_rules(self, security_group): - self.apply_ruleset() + for instance in self.instances: + self.remove_filters_for_instance(instance) + self.add_filters_for_instance(instance) + self.iptables.apply() def _security_group_chain_name(self, security_group_id): return 'nova-sg-%s' % (security_group_id,) -- cgit From cbb0402efac4ededdda0ac2097ec087216e23931 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 10:18:43 +0100 Subject: Also remove rules that jump to deleted chains. --- nova/network/linux_net.py | 5 ++++- nova/virt/libvirt_conn.py | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ecda450bf..1f96a4d55 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -104,6 +104,9 @@ class IptablesTable(object): self.chains.remove(name) self.rules = filter(lambda r: r.chain != name, self.rules) + jump_snippet = '-j %s-%s' % (binary_name, name) + self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) + def add_rule(self, chain, rule, wrap=True): if wrap and chain not in self.chains: raise ValueError(_("Unknown chain: %r") % chain) @@ -283,7 +286,7 @@ def remove_floating_forward(floating_ip, fixed_ip): def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), - ("SNATTING", "-d %s -j DNAT --to %s" % (fixed_ip, floating_ip))] + ("SNATTING", "-d %s -j SNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): """Create a vlan and bridge unless they already exist""" diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 3faf01f4b..daf8f0ed7 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -44,9 +44,6 @@ import uuid from xml.dom import minidom -from eventlet import greenthread -from eventlet import event -from eventlet import semaphore from eventlet import tpool import IPy @@ -1246,7 +1243,7 @@ class IptablesFirewallDriver(FirewallDriver): if FLAGS.use_ipv6: self.iptables.ipv6['filter'].add_chain(chain_name) ipv6_address = self._ip_for_instance_v6(instance) - self.iptables.ipv4['filter'].add_rule('local', + self.iptables.ipv6['filter'].add_rule('local', '-d %s -j $%s' % (ipv6_address, chain_name)) @@ -1376,7 +1373,7 @@ class IptablesFirewallDriver(FirewallDriver): pass def refresh_security_group_rules(self, security_group): - for instance in self.instances: + for instance in self.instances.values(): self.remove_filters_for_instance(instance) self.add_filters_for_instance(instance) self.iptables.apply() -- cgit From 9eebe4317f86ae13ffeaca1622e9fc555bc28ebc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 10:42:59 +0100 Subject: Unfilter instance correctly on termination. --- nova/network/linux_net.py | 4 ++++ nova/virt/libvirt_conn.py | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 1f96a4d55..1145bfa7a 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -101,6 +101,10 @@ class IptablesTable(object): self.chains.add(name) def remove_chain(self, name): + if name not in self.chain: + LOG.debug(_("Attempted to remove chain %s which doesn't exist"), + name) + return self.chains.remove(name) self.rules = filter(lambda r: r.chain != name, self.rules) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index daf8f0ed7..0c355e48e 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1219,9 +1219,11 @@ class IptablesFirewallDriver(FirewallDriver): """No-op. Everything is done in prepare_instance_filter""" pass - def remove_instance(self, instance): + def unfilter_instance(self, instance): if instance['id'] in self.instances: del self.instances[instance['id']] + self.remove_filters_for_instance(instance) + self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) @@ -1257,10 +1259,6 @@ class IptablesFirewallDriver(FirewallDriver): for rule in ipv6_rules: self.iptables.ipv6['filter'].add_rule(chain_name, rule) - def unfilter_instance(self, instance): - self.remove_filters_for_instance(instance) - self.iptables.apply() - def remove_filters_for_instance(self, instance): chain_name = self._instance_chain_name(instance) -- cgit From 384a4525e9d6de54158cd170487fce95df814b52 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 11:01:27 +0100 Subject: Fix typo --- nova/network/linux_net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 1145bfa7a..c5779f85f 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -101,7 +101,7 @@ class IptablesTable(object): self.chains.add(name) def remove_chain(self, name): - if name not in self.chain: + if name not in self.chains: LOG.debug(_("Attempted to remove chain %s which doesn't exist"), name) return -- cgit From 15203c9ceaa94f0cd5bad96622ee93af7662bcce Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 12:22:29 +0100 Subject: Allow non-existing rules to be removed. --- nova/network/linux_net.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c5779f85f..d4cfbbde9 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -125,8 +125,12 @@ class IptablesTable(object): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, chain, rule): - self.rules.remove(IptablesRule(chain, rule)) + def remove_rule(self, *args, **kwargs): + try: + self.rules.remove(IptablesRule(*args, **kwargs)) + except ValueError: + LOG.debug(_("Tried to remove rule that wasn't there: %r %r"), + args, kwargs) class IptablesManager(object): def __init__(self, execute=None): -- cgit From a57dffb5fdfbfac59b9ddbe7b33d6f03b7b748ba Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:16:42 +0100 Subject: PEP-8 fixes --- nova/network/linux_net.py | 28 ++++++++++++++++++++-------- nova/tests/test_network.py | 21 +++++++++------------ nova/tests/test_virt.py | 3 ++- nova/utils.py | 12 ++++++------ nova/virt/libvirt_conn.py | 2 -- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index d4cfbbde9..b5d1323a1 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -129,8 +129,10 @@ class IptablesTable(object): try: self.rules.remove(IptablesRule(*args, **kwargs)) except ValueError: - LOG.debug(_("Tried to remove rule that wasn't there: %r %r"), - args, kwargs) + LOG.debug(_("Tried to remove rule that wasn't there:" + " %(args)r %(kwargs)r"), {'args': args, + 'kwargs': kwargs}) + class IptablesManager(object): def __init__(self, execute=None): @@ -142,9 +144,9 @@ class IptablesManager(object): else: self.execute = execute - self.ipv4 = { 'filter': IptablesTable(), - 'nat': IptablesTable() } - self.ipv6 = { 'filter': IptablesTable() } + self.ipv4 = {'filter': IptablesTable(), + 'nat': IptablesTable()} + self.ipv6 = {'filter': IptablesTable()} self.ipv4['nat'].add_chain('SNATTING') self.ipv4['nat'].add_rule('POSTROUTING', @@ -155,11 +157,18 @@ class IptablesManager(object): self.ipv4['filter'].add_rule('FORWARD', '-j %s-local' % (binary_name,), wrap=False) - self.ipv4['filter'].add_rule('OUTPUT', '-j %s-local' % (binary_name,), wrap=False) + self.ipv6['filter'].add_chain('local') + self.ipv6['filter'].add_rule('FORWARD', + '-j %s-local' % (binary_name,), + wrap=False) + self.ipv6['filter'].add_rule('OUTPUT', + '-j %s-local' % (binary_name,), + wrap=False) + # Wrap the builtin chains builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']} @@ -172,7 +181,6 @@ class IptablesManager(object): wrap=False) self.semaphore = semaphore.Semaphore() - def apply(self): with self.semaphore: s = [('iptables', self.ipv4)] @@ -184,7 +192,8 @@ class IptablesManager(object): current_table, _ = self.execute('sudo %s-save -t %s' % (cmd, table), attempts=5) current_lines = current_table.split('\n') - new_filter = self.modify_rules(current_lines, tables[table]) + new_filter = self.modify_rules(current_lines, + tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), attempts=5) @@ -285,17 +294,20 @@ def ensure_floating_forward(floating_ip, fixed_ip): iptables_manager.ipv4['nat'].add_rule(chain, rule) iptables_manager.apply() + def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" for chain, rule in floating_forward_rules(floating_ip, fixed_ip): iptables_manager.ipv4['nat'].remove_rule(chain, rule) iptables_manager.apply() + def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("SNATTING", "-d %s -j SNAT --to %s" % (fixed_ip, floating_ip))] + def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): """Create a vlan and bridge unless they already exist""" interface = ensure_vlan(vlan_num) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index b28d64245..c9a62a391 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -42,15 +42,14 @@ class IptablesManagerTestCase(test.TestCase): :INPUT ACCEPT [2223527:305688874] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [2172501:140856656] --A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT --A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT --A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT --A FORWARD -i virbr0 -o virbr0 -j ACCEPT --A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable --A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable +-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT +-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT +-A FORWARD -i virbr0 -o virbr0 -j ACCEPT +-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable COMMIT # Completed on Fri Feb 18 15:17:05 2011""" @@ -77,8 +76,7 @@ COMMIT # TODO(soren): Add stuff for ipv6 check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'INPUT', - 'OUTPUT', 'POSTROUTING']} } + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}} for ip_version in check_matrix: ip = getattr(self.manager, 'ipv%d' % ip_version) @@ -90,7 +88,6 @@ COMMIT (chain,) in new_lines) self.assertTrue('-A %s -j run_tests.py-%s' % \ (chain, chain) in new_lines) - print '\n'.join(new_lines) class NetworkTestCase(test.TestCase): diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index a88e01818..11201788c 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -345,7 +345,8 @@ class IptablesFirewallTestCase(test.TestCase): self.fw.prepare_instance_filter(instance_ref) self.fw.apply_instance_filter(instance_ref) - in_rules = filter(lambda l: not l.startswith('#'), self.in_filter_rules) + in_rules = filter(lambda l: not l.startswith('#'), + self.in_filter_rules) for rule in in_rules: if not 'nova' in rule: self.assertTrue(rule in self.out_rules, diff --git a/nova/utils.py b/nova/utils.py index 644bf18fd..5b44bccb5 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -126,7 +126,8 @@ def fetchfile(url, target): execute("curl --fail %s -o %s" % (url, target)) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attempts=1): +def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, + attempts=1): while attempts > 0: attempts -= 1 try: @@ -150,17 +151,16 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attemp stdout=stdout, stderr=stderr, cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one greenthread.sleep(0) return result except ProcessExecutionError: if not attempts: raise else: - greenthread.sleep(random.randint(50,300)/100) - pass + greenthread.sleep(random.randint(20, 200) / 100.0) def ssh_execute(ssh, cmd, process_input=None, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0c355e48e..7f74e3505 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1303,11 +1303,9 @@ class IptablesFirewallDriver(FirewallDriver): cidrv6 = self._project_cidrv6_for_instance(instance) ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] - security_groups = db.security_group_get_by_instance(ctxt, instance['id']) - # then, security group chains and rules for security_group in security_groups: rules = db.security_group_rule_get_by_security_group(ctxt, -- cgit From 3d2ec0f594e02018a32c8d0d7a8cc46f7ab4c849 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:39:02 +0100 Subject: Wrap ipv6 rules, too --- nova/network/linux_net.py | 26 +++++++++++++++++--------- nova/tests/test_network.py | 3 ++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b5d1323a1..f47219b2e 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -170,15 +170,23 @@ class IptablesManager(object): wrap=False) # Wrap the builtin chains - builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']} - - for table, chains in builtin_chains.iteritems(): - for chain in chains: - self.ipv4[table].add_chain(chain) - self.ipv4[table].add_rule(chain, - '-j %s-%s' % (binary_name, chain), - wrap=False) + builtin_chains = { 4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} + + for ip_version in builtin_chains: + if ip_version == 4: + tables = self.ipv4 + elif ip_version == 6: + tables = self.ipv6 + + for table, chains in builtin_chains[ip_version].iteritems(): + for chain in chains: + tables[table].add_chain(chain) + tables[table].add_rule(chain, + '-j %s-%s' % (binary_name, chain), + wrap=False) + self.semaphore = semaphore.Semaphore() def apply(self): diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index c9a62a391..f1d4fe133 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -76,7 +76,8 @@ COMMIT # TODO(soren): Add stuff for ipv6 check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}} + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} for ip_version in check_matrix: ip = getattr(self.manager, 'ipv%d' % ip_version) -- cgit From 1ed4df93a246a06518f2c216cd0fc60ca67eb5c4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:39:38 +0100 Subject: More PEP-8 --- nova/network/linux_net.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index f47219b2e..d7a3075cb 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -170,9 +170,9 @@ class IptablesManager(object): wrap=False) # Wrap the builtin chains - builtin_chains = { 4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, - 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} + builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} for ip_version in builtin_chains: if ip_version == 4: -- cgit From fc0ea52d9379649d28de88d4fa1628e455533842 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 23:08:26 +0100 Subject: Add a bunch of docs for the new iptables hotness. --- nova/network/linux_net.py | 72 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index d7a3075cb..3b6ec9338 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -69,6 +69,11 @@ binary_name = os.path.basename(inspect.stack()[-1][1]) class IptablesRule(object): + """An iptables rule + + You shouldn't need to use this class directly, it's only used by + IptablesManager + """ def __init__(self, chain, rule, wrap=True): self.chain = chain self.rule = rule @@ -93,14 +98,33 @@ class IptablesRule(object): class IptablesTable(object): + """An iptables table""" + def __init__(self): self.rules = [] self.chains = set() def add_chain(self, name): + """Adds a named chain to the table + + The chain name is wrapped to be unique for the component creating + it, so different components of Nova can safely create identically + named chains without interfering with one another. + + At the moment, its wrapped name is -, + so if nova-compute creates a chain named "OUTPUT", it'll actually + end up named "nova-compute-OUTPUT". + """ self.chains.add(name) def remove_chain(self, name): + """Remove named chain + + This removal "cascades". All rule in the chain are removed, as are + all rules in other chains that jump to it. + + If the chain is not found, this is merely logged. + """ if name not in self.chains: LOG.debug(_("Attempted to remove chain %s which doesn't exist"), name) @@ -112,6 +136,15 @@ class IptablesTable(object): self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) def add_rule(self, chain, rule, wrap=True): + """Add a rule to the table + + This is just like what you'd feed to iptables, just without + the "-A " bit at the start. + + However, if you need to jump to one of your wrapped chains, + prepend its name with a '$' which will ensure the wrapping + is applied correctly. + """ if wrap and chain not in self.chains: raise ValueError(_("Unknown chain: %r") % chain) @@ -125,7 +158,13 @@ class IptablesTable(object): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, *args, **kwargs): + def remove_rule(self, chain, rule, wrap=True): + """Remove a rule from a chain + + Note: The rule must be exactly identical to the one that was added. + You cannot switch arguments around like you can with the iptables + CLI tool. + """ try: self.rules.remove(IptablesRule(*args, **kwargs)) except ValueError: @@ -135,6 +174,22 @@ class IptablesTable(object): class IptablesManager(object): + """Wrapper for iptables + + See IptablesTable for some usage docs + + A number of chains are set up to begin with. + + For ipv4, the filter table has a INPUT, OUTPUT, FORWARD, and local chains + already set up, while the NAT chain has PREROUTING, OUTPUT, POSTROUTING, + and SNATTING. Except for "local" and "SNATTING" these are all set up so + that they are applied at the beginning of their non-wrapped counterparts. + "SNATTING" is jumped to from nat/POSTROUTING and "local" is jumped to from + filter/OUTPUT and filter/FORWARD. + + For ipv6, the filter table has INPUT, OUTPUT, FORWARD, and local. "local" + has the same semantics as for ipv4. + """ def __init__(self, execute=None): if not execute: if FLAGS.fake_network: @@ -190,6 +245,16 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() def apply(self): + """Apply the current in-memory set of iptables rules + + This will blow away any rules left over from previous runs of the + same component of Nova, and replace them with our current set of + rules. This happens atomically, thanks to iptables-restore. + + We wrap the call in a semaphore lock, so that we don't race with + ourselves. In the event of a race with another component running + an iptables-* command at the same time, we retry up to 5 times. + """ with self.semaphore: s = [('iptables', self.ipv4)] if FLAGS.use_ipv6: @@ -200,14 +265,13 @@ class IptablesManager(object): current_table, _ = self.execute('sudo %s-save -t %s' % (cmd, table), attempts=5) current_lines = current_table.split('\n') - new_filter = self.modify_rules(current_lines, + new_filter = self._modify_rules(current_lines, tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), attempts=5) - def modify_rules(self, current_lines, table, binary=None): - + def _modify_rules(self, current_lines, table, binary=None): chains = table.chains rules = table.rules -- cgit From c53bb1718a9b5900d09637d0ee966dadbf073900 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 09:00:29 +0100 Subject: Address some review comments. --- nova/network/linux_net.py | 16 ++++++++-------- nova/tests/test_virt.py | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 3b6ec9338..42af73e74 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -54,8 +54,6 @@ flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') flags.DEFINE_string('routing_source_ip', '$my_ip', 'Public IP of network host') -flags.DEFINE_bool('use_nova_chains', False, - 'use the nova_ routing chains instead of default') flags.DEFINE_string('input_chain', 'INPUT', 'chain to add nova_input to') @@ -266,7 +264,7 @@ class IptablesManager(object): (cmd, table), attempts=5) current_lines = current_table.split('\n') new_filter = self._modify_rules(current_lines, - tables[table]) + tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), attempts=5) @@ -276,15 +274,17 @@ class IptablesManager(object): rules = table.rules # Remove any trace of our rules - new_filter = filter(lambda l: binary_name not in l, current_lines) + new_filter = filter(lambda line: binary_name not in line, + current_lines) seen_chains = False - for rules_index in range(len(new_filter)): + rules_index = 0 + for rules_index, rule in enumerate(new_filter): if not seen_chains: - if new_filter[rules_index].startswith(':'): + if rule.startswith(':'): seen_chains = True - elif seen_chains == 1: - if not new_filter[rules_index].startswith(':'): + else: + if not rule.startswith(':'): break new_filter[rules_index:rules_index] = [str(rule) for rule in rules] diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 11201788c..c2c7c8337 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -354,7 +354,6 @@ class IptablesFirewallTestCase(test.TestCase): instance_chain = None for rule in self.out_rules: - print rule # This is pretty crude, but it'll do for now if '-d 10.11.12.13 -j' in rule: instance_chain = rule.split(' ')[-1] -- cgit From 70d526dc44299b6bd54a5757d013ca3109887747 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 09:28:02 +0100 Subject: Add a new chain, floating-ip-snat, at the top of SNATTING, so that SNATting for floating ips gets applied before the default SNAT rule. --- nova/network/linux_net.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 42af73e74..6b0735b5c 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -201,26 +201,13 @@ class IptablesManager(object): 'nat': IptablesTable()} self.ipv6 = {'filter': IptablesTable()} - self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('POSTROUTING', - '-j %s-SNATTING' % (binary_name,), - wrap=False) - self.ipv4['filter'].add_chain('local') - self.ipv4['filter'].add_rule('FORWARD', - '-j %s-local' % (binary_name,), - wrap=False) - self.ipv4['filter'].add_rule('OUTPUT', - '-j %s-local' % (binary_name,), - wrap=False) + self.ipv4['filter'].add_rule('FORWARD', '-j $local', wrap=False) + self.ipv4['filter'].add_rule('OUTPUT', '-j $local', wrap=False) self.ipv6['filter'].add_chain('local') - self.ipv6['filter'].add_rule('FORWARD', - '-j %s-local' % (binary_name,), - wrap=False) - self.ipv6['filter'].add_rule('OUTPUT', - '-j %s-local' % (binary_name,), - wrap=False) + self.ipv6['filter'].add_rule('FORWARD', '-j $local', wrap=False) + self.ipv6['filter'].add_rule('OUTPUT', '-j $local', wrap=False) # Wrap the builtin chains builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], @@ -236,12 +223,22 @@ class IptablesManager(object): for table, chains in builtin_chains[ip_version].iteritems(): for chain in chains: tables[table].add_chain(chain) - tables[table].add_rule(chain, - '-j %s-%s' % (binary_name, chain), + tables[table].add_rule(chain, '-j $%s' % (chain,), wrap=False) + # We add a SNATTING chain after our (wrapped) POSTROUTING chain + # so that rules added there will be applied after whatever we have in + # (the wrapped) POSTROUTING. + self.ipv4['nat'].add_chain('SNATTING') + self.ipv4['nat'].add_rule('POSTROUTING', '-j $SNATTING', wrap=False) + + self.ipv4['nat'].add_chain('floating-ip-snat') + self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') + self.semaphore = semaphore.Semaphore() + iptables_manager.apply() + def apply(self): """Apply the current in-memory set of iptables rules @@ -318,11 +315,6 @@ def init_host(): (FLAGS.fixed_range, FLAGS.routing_source_ip)) - iptables_manager.ipv4['nat'].add_rule("POSTROUTING", - "-s %s -j SNAT --to-source %s" % \ - (FLAGS.fixed_range, - FLAGS.routing_source_ip)) - iptables_manager.ipv4['nat'].add_rule("POSTROUTING", "-s %s -d %s -j ACCEPT" % \ (FLAGS.fixed_range, FLAGS.dmz_cidr)) @@ -377,7 +369,8 @@ def remove_floating_forward(floating_ip, fixed_ip): def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), - ("SNATTING", "-d %s -j SNAT --to %s" % (fixed_ip, floating_ip))] + ("floating-ip-snat", + "-s %s -j SNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): -- cgit From bf37fb0ab5503a077a3d9e4109990d252e27cb15 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 11:29:58 +0100 Subject: Add a bunch of tests for everything. Add a 'head' kwarg to add_rule that lets the rule bubble to the top. This is needed for nova-filter-top to end up at the top. --- nova/network/linux_net.py | 120 ++++++++++++++++++++++++++++++------------ nova/tests/test_network.py | 128 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 192 insertions(+), 56 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 6b0735b5c..2ccb5969e 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -72,19 +72,22 @@ class IptablesRule(object): You shouldn't need to use this class directly, it's only used by IptablesManager """ - def __init__(self, chain, rule, wrap=True): + def __init__(self, chain, rule, wrap=True, head=False): self.chain = chain self.rule = rule self.wrap = wrap + self.head = head def __eq__(self, other): return ((self.chain == other.chain) and (self.rule == other.rule) and + (self.head == other.head) and (self.wrap == other.wrap)) def __ne__(self, other): return ((self.chain != other.chain) or (self.rule != other.rule) or + (self.head != other.head) or (self.wrap != other.wrap)) def __str__(self): @@ -101,8 +104,9 @@ class IptablesTable(object): def __init__(self): self.rules = [] self.chains = set() + self.unwrapped_chains = set() - def add_chain(self, name): + def add_chain(self, name, wrap=True): """Adds a named chain to the table The chain name is wrapped to be unique for the component creating @@ -113,9 +117,12 @@ class IptablesTable(object): so if nova-compute creates a chain named "OUTPUT", it'll actually end up named "nova-compute-OUTPUT". """ - self.chains.add(name) + if wrap: + self.chains.add(name) + else: + self.unwrapped_chains.add(name) - def remove_chain(self, name): + def remove_chain(self, name, wrap=True): """Remove named chain This removal "cascades". All rule in the chain are removed, as are @@ -123,17 +130,27 @@ class IptablesTable(object): If the chain is not found, this is merely logged. """ - if name not in self.chains: + if wrap: + chain_set = self.chains + else: + chain_set = self.unwrapped_chains + + if name not in chain_set: LOG.debug(_("Attempted to remove chain %s which doesn't exist"), name) return - self.chains.remove(name) + + chain_set.remove(name) self.rules = filter(lambda r: r.chain != name, self.rules) - jump_snippet = '-j %s-%s' % (binary_name, name) + if wrap: + jump_snippet = '-j %s-%s' % (binary_name, name) + else: + jump_snippet = '-j %s' % (name,) + self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) - def add_rule(self, chain, rule, wrap=True): + def add_rule(self, chain, rule, wrap=True, head=False): """Add a rule to the table This is just like what you'd feed to iptables, just without @@ -149,14 +166,14 @@ class IptablesTable(object): if '$' in rule: rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) - self.rules.append(IptablesRule(chain, rule, wrap)) + self.rules.append(IptablesRule(chain, rule, wrap, head)) def _wrap_target_chain(self, s): if s.startswith('$'): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, chain, rule, wrap=True): + def remove_rule(self, chain, rule, wrap=True, head=False): """Remove a rule from a chain Note: The rule must be exactly identical to the one that was added. @@ -164,11 +181,12 @@ class IptablesTable(object): CLI tool. """ try: - self.rules.remove(IptablesRule(*args, **kwargs)) + self.rules.remove(IptablesRule(chain, rule, wrap, head)) except ValueError: LOG.debug(_("Tried to remove rule that wasn't there:" - " %(args)r %(kwargs)r"), {'args': args, - 'kwargs': kwargs}) + " %(chain)r %(rule)r %(wrap)r %(head)r"), + {'chain': chain, 'rule': rule, + 'head': head, 'wrap': wrap}) class IptablesManager(object): @@ -178,15 +196,19 @@ class IptablesManager(object): A number of chains are set up to begin with. - For ipv4, the filter table has a INPUT, OUTPUT, FORWARD, and local chains - already set up, while the NAT chain has PREROUTING, OUTPUT, POSTROUTING, - and SNATTING. Except for "local" and "SNATTING" these are all set up so - that they are applied at the beginning of their non-wrapped counterparts. - "SNATTING" is jumped to from nat/POSTROUTING and "local" is jumped to from - filter/OUTPUT and filter/FORWARD. + First, nova-filter-top. It's added at the top of FORWARD and OUTPUT. Its + name is not wrapped, so it's shared between the various nova workers. It's + intended for rules that need to live at the top of the FORWARD and OUTPUT + chains. It's in both the ipv4 and ipv6 set of tables. - For ipv6, the filter table has INPUT, OUTPUT, FORWARD, and local. "local" - has the same semantics as for ipv4. + For ipv4 and ipv6, the builtin INPUT, OUTPUT, and FORWARD filter chains are + wrapped, meaning that the "real" INPUT chain has a rule that jumps to the + wrapped INPUT chain, etc. Additionally, there's a wrapped chain named + "local" which is jumped to from nova-filter-top. + + For ipv4, the builtin PREROUTING, OUTPUT, and POSTROUTING nat chains are + wrapped in the same was as the builtin filter chains. Additionally, there's + a SNATTING chain that is applied after the POSTROUTING chain. """ def __init__(self, execute=None): if not execute: @@ -201,13 +223,19 @@ class IptablesManager(object): 'nat': IptablesTable()} self.ipv6 = {'filter': IptablesTable()} - self.ipv4['filter'].add_chain('local') - self.ipv4['filter'].add_rule('FORWARD', '-j $local', wrap=False) - self.ipv4['filter'].add_rule('OUTPUT', '-j $local', wrap=False) + # Add a nova-filter-top chain. It's intended to be shared + # among the various nova components. It sits at the very top + # of FORWARD and OUTPUT. + for tables in [self.ipv4, self.ipv6]: + tables['filter'].add_chain('nova-filter-top', wrap=False) + tables['filter'].add_rule('FORWARD', '-j nova-filter-top', + wrap=False, head=True) + tables['filter'].add_rule('OUTPUT', '-j nova-filter-top', + wrap=False, head=True) - self.ipv6['filter'].add_chain('local') - self.ipv6['filter'].add_rule('FORWARD', '-j $local', wrap=False) - self.ipv6['filter'].add_rule('OUTPUT', '-j $local', wrap=False) + tables['filter'].add_chain('local') + tables['filter'].add_rule('nova-filter-top', '-j $local', + wrap=False) # Wrap the builtin chains builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], @@ -226,18 +254,27 @@ class IptablesManager(object): tables[table].add_rule(chain, '-j $%s' % (chain,), wrap=False) - # We add a SNATTING chain after our (wrapped) POSTROUTING chain - # so that rules added there will be applied after whatever we have in - # (the wrapped) POSTROUTING. + # Add a nova-postrouting-bottom chain. It's intended to be shared + # among the various nova components. We set it as the last chain + # of POSTROUTING chain. + self.ipv4['nat'].add_chain('nova-postrouting-bottom', wrap=False) + self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', + wrap=False) + + + # We add a SNATTING chain to the shared nova-postrouting-bottom chain + # so that it's applied last. self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('POSTROUTING', '-j $SNATTING', wrap=False) + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', wrap=False) + # And then we add a floating-ip-snat chain and jump to first thing in the SNATTING + # chain. self.ipv4['nat'].add_chain('floating-ip-snat') self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') self.semaphore = semaphore.Semaphore() - iptables_manager.apply() + self.apply() def apply(self): """Apply the current in-memory set of iptables rules @@ -245,7 +282,7 @@ class IptablesManager(object): This will blow away any rules left over from previous runs of the same component of Nova, and replace them with our current set of rules. This happens atomically, thanks to iptables-restore. - + We wrap the call in a semaphore lock, so that we don't race with ourselves. In the event of a race with another component running an iptables-* command at the same time, we retry up to 5 times. @@ -267,6 +304,7 @@ class IptablesManager(object): attempts=5) def _modify_rules(self, current_lines, table, binary=None): + unwrapped_chains = table.unwrapped_chains chains = table.chains rules = table.rules @@ -284,11 +322,25 @@ class IptablesManager(object): if not rule.startswith(':'): break - new_filter[rules_index:rules_index] = [str(rule) for rule in rules] + new_filter[rules_index:rules_index] = [str(rule) for rule in rules + if rule.head or + str(rule) not in new_filter] + new_filter[rules_index:rules_index] = [':%s - [0:0]' % \ + (name,) \ + for name in unwrapped_chains] new_filter[rules_index:rules_index] = [':%s-%s - [0:0]' % \ (binary_name, name,) \ for name in chains] + seen_lines = set() + def _weed_out_duplicates(line): + if line in seen_lines: + return False + else: + seen_lines.add(line) + return True + + new_filter = filter(_weed_out_duplicates, new_filter) return new_filter diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index f1d4fe133..afd38272d 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -36,12 +36,22 @@ LOG = logging.getLogger('nova.tests.network') class IptablesManagerTestCase(test.TestCase): - sample_filter = """# Completed on Fri Feb 18 15:17:05 2011 -# Generated by iptables-save v1.4.10 on Fri Feb 18 15:17:05 2011 + sample_filter = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 *filter :INPUT ACCEPT [2223527:305688874] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [2172501:140856656] +:nova-compute-FORWARD - [0:0] +:nova-compute-INPUT - [0:0] +:nova-compute-local - [0:0] +:nova-compute-OUTPUT - [0:0] +:nova-filter-top - [0:0] +-A FORWARD -j nova-filter-top +-A OUTPUT -j nova-filter-top +-A nova-filter-top -j nova-compute-local +-A INPUT -j nova-compute-INPUT +-A OUTPUT -j nova-compute-OUTPUT +-A FORWARD -j nova-compute-FORWARD -A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT -A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT -A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT @@ -53,42 +63,116 @@ class IptablesManagerTestCase(test.TestCase): COMMIT # Completed on Fri Feb 18 15:17:05 2011""" + sample_nat = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 +*nat +:PREROUTING ACCEPT [3936:762355] +:INPUT ACCEPT [2447:225266] +:OUTPUT ACCEPT [63491:4191863] +:POSTROUTING ACCEPT [63112:4108641] +:nova-compute-OUTPUT - [0:0] +:nova-compute-floating-ip-snat - [0:0] +:nova-compute-SNATTING - [0:0] +:nova-compute-PREROUTING - [0:0] +:nova-compute-POSTROUTING - [0:0] +:nova-postrouting-bottom - [0:0] +-A PREROUTING -j nova-compute-PREROUTING +-A OUTPUT -j nova-compute-OUTPUT +-A POSTROUTING -j nova-compute-POSTROUTING +-A POSTROUTING -j nova-postrouting-bottom +-A nova-postrouting-bottom -j nova-compute-SNATTING +-A nova-compute-SNATTING -j nova-compute-floating-ip-snat +COMMIT +# Completed on Fri Feb 18 15:17:05 2011 +""" + def setUp(self): super(IptablesManagerTestCase, self).setUp() self.manager = linux_net.IptablesManager() - def test_rules_are_wrapped(self): + + def test_filter_rules_are_wrapped(self): current_lines = self.sample_filter.split('\n') table = self.manager.ipv4['filter'] table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') - new_lines = self.manager.modify_rules(current_lines, table) + new_lines = self.manager._modify_rules(current_lines, table) self.assertTrue('-A run_tests.py-FORWARD ' '-s 1.2.3.4/5 -j DROP' in new_lines) table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') - new_lines = self.manager.modify_rules(current_lines, table) + new_lines = self.manager._modify_rules(current_lines, table) self.assertTrue('-A run_tests.py-FORWARD ' '-s 1.2.3.4/5 -j DROP' not in new_lines) - def test_wrapper_rules_in_place(self): - current_lines = self.sample_filter.split('\n') + def test_nat_rules(self): + current_lines = self.sample_nat.split('\n') + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['nat']) + + for line in [':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains " + "went missing.") + + seen_lines = set() + for line in new_lines: + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) - # TODO(soren): Add stuff for ipv6 - check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, - 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} - - for ip_version in check_matrix: - ip = getattr(self.manager, 'ipv%d' % ip_version) - for table_name in ip: - table = ip[table_name] - new_lines = self.manager.modify_rules(current_lines, table) - for chain in check_matrix[ip_version][table_name]: - self.assertTrue(':run_tests.py-%s - [0:0]' % \ - (chain,) in new_lines) - self.assertTrue('-A %s -j run_tests.py-%s' % \ - (chain, chain) in new_lines) + last_postrouting_line = '' + + for line in new_lines: + if line.startswith('-A POSTROUTING'): + last_postrouting_line = line + + self.assertTrue('-j nova-postrouting-bottom' in last_postrouting_line, + "Last POSTROUTING rule does not jump to " + "nova-postouting-bottom: %s" % last_postrouting_line) + + for chain in ['POSTROUTING', 'PREROUTING', 'OUTPUT']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) + + + def test_filter_rules(self): + current_lines = self.sample_filter.split('\n') + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['filter']) + + for line in [':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains" + " went missing.") + + seen_lines = set() + for line in new_lines: + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) + + for chain in ['FORWARD', 'OUTPUT']: + for line in new_lines: + if line.startswith('-A %s' % chain): + self.assertTrue('-j nova-filter-top' in line, + "First %s rule does not " + "jump to nova-filter-top" % chain) + break + + self.assertTrue('-A nova-filter-top ' + '-j run_tests.py-local' in new_lines, + "nova-filter-top does not jump to wrapped local chain") + + for chain in ['INPUT', 'OUTPUT', 'FORWARD']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) class NetworkTestCase(test.TestCase): -- cgit From 54f2362d09393ad6ccdfee5689d4f547c69b3f42 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 11:47:46 +0100 Subject: Remove leftover from debugging. --- nova/network/linux_net.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 2ccb5969e..5f480f633 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -274,8 +274,6 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() - self.apply() - def apply(self): """Apply the current in-memory set of iptables rules -- cgit From b5e6601f76d64a96d6c7de5e9acdf5a8cf0fe8e9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 12:21:29 +0100 Subject: PEP8 adjustments. --- nova/network/linux_net.py | 9 +++++---- nova/tests/test_network.py | 10 ++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 5f480f633..7c4c16810 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -261,14 +261,14 @@ class IptablesManager(object): self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', wrap=False) - # We add a SNATTING chain to the shared nova-postrouting-bottom chain # so that it's applied last. self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', wrap=False) + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', + wrap=False) - # And then we add a floating-ip-snat chain and jump to first thing in the SNATTING - # chain. + # And then we add a floating-ip-snat chain and jump to first thing in + # the SNATTING chain. self.ipv4['nat'].add_chain('floating-ip-snat') self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') @@ -331,6 +331,7 @@ class IptablesManager(object): for name in chains] seen_lines = set() + def _weed_out_duplicates(line): if line in seen_lines: return False diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index afd38272d..2bdf3709e 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -89,7 +89,6 @@ COMMIT super(IptablesManagerTestCase, self).setUp() self.manager = linux_net.IptablesManager() - def test_filter_rules_are_wrapped(self): current_lines = self.sample_filter.split('\n') @@ -114,8 +113,8 @@ COMMIT ':nova-compute-SNATTING - [0:0]', ':nova-compute-PREROUTING - [0:0]', ':nova-compute-POSTROUTING - [0:0]']: - self.assertTrue(line in new_lines, "One of nova-compute's chains " - "went missing.") + self.assertTrue(line in new_lines, "One of nova-compute's chains " + "went missing.") seen_lines = set() for line in new_lines: @@ -138,7 +137,6 @@ COMMIT % (chain, chain) in new_lines, "Built-in chain %s not wrapped" % (chain,)) - def test_filter_rules(self): current_lines = self.sample_filter.split('\n') new_lines = self.manager._modify_rules(current_lines, @@ -148,8 +146,8 @@ COMMIT ':nova-compute-INPUT - [0:0]', ':nova-compute-local - [0:0]', ':nova-compute-OUTPUT - [0:0]']: - self.assertTrue(line in new_lines, "One of nova-compute's chains" - " went missing.") + self.assertTrue(line in new_lines, "One of nova-compute's chains" + " went missing.") seen_lines = set() for line in new_lines: -- cgit From 92a693b04feddad2420a835a12f53520c5529d8f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 12:42:38 +0100 Subject: floating-ip-snat was too long. Use floating-snat instead. --- nova/network/linux_net.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 7c4c16810..5e89a1df6 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -267,10 +267,10 @@ class IptablesManager(object): self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', wrap=False) - # And then we add a floating-ip-snat chain and jump to first thing in + # And then we add a floating-snat chain and jump to first thing in # the SNATTING chain. - self.ipv4['nat'].add_chain('floating-ip-snat') - self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') + self.ipv4['nat'].add_chain('floating-snat') + self.ipv4['nat'].add_rule('SNATTING', '-j $floating-snat') self.semaphore = semaphore.Semaphore() @@ -420,7 +420,7 @@ def remove_floating_forward(floating_ip, fixed_ip): def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), - ("floating-ip-snat", + ("floating-snat", "-s %s -j SNAT --to %s" % (fixed_ip, floating_ip))] -- cgit From 443f01ef7d977ba6ff86508b908f66733bdd989e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 14:32:20 +0100 Subject: Account for the fact that iptables-save outputs rules with a space at the end. Reverse the rule deduplication so that the last one takes precedence. --- nova/network/linux_net.py | 45 ++++++++++++++++++++++++++++++--------------- nova/tests/test_network.py | 42 ++++++++++++++++++++++-------------------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 5e89a1df6..849181641 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -72,22 +72,22 @@ class IptablesRule(object): You shouldn't need to use this class directly, it's only used by IptablesManager """ - def __init__(self, chain, rule, wrap=True, head=False): + def __init__(self, chain, rule, wrap=True, top=False): self.chain = chain self.rule = rule self.wrap = wrap - self.head = head + self.top = top def __eq__(self, other): return ((self.chain == other.chain) and (self.rule == other.rule) and - (self.head == other.head) and + (self.top == other.top) and (self.wrap == other.wrap)) def __ne__(self, other): return ((self.chain != other.chain) or (self.rule != other.rule) or - (self.head != other.head) or + (self.top != other.top) or (self.wrap != other.wrap)) def __str__(self): @@ -150,7 +150,7 @@ class IptablesTable(object): self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) - def add_rule(self, chain, rule, wrap=True, head=False): + def add_rule(self, chain, rule, wrap=True, top=False): """Add a rule to the table This is just like what you'd feed to iptables, just without @@ -166,14 +166,14 @@ class IptablesTable(object): if '$' in rule: rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) - self.rules.append(IptablesRule(chain, rule, wrap, head)) + self.rules.append(IptablesRule(chain, rule, wrap, top)) def _wrap_target_chain(self, s): if s.startswith('$'): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, chain, rule, wrap=True, head=False): + def remove_rule(self, chain, rule, wrap=True, top=False): """Remove a rule from a chain Note: The rule must be exactly identical to the one that was added. @@ -181,12 +181,12 @@ class IptablesTable(object): CLI tool. """ try: - self.rules.remove(IptablesRule(chain, rule, wrap, head)) + self.rules.remove(IptablesRule(chain, rule, wrap, top)) except ValueError: LOG.debug(_("Tried to remove rule that wasn't there:" - " %(chain)r %(rule)r %(wrap)r %(head)r"), + " %(chain)r %(rule)r %(wrap)r %(top)r"), {'chain': chain, 'rule': rule, - 'head': head, 'wrap': wrap}) + 'top': top, 'wrap': wrap}) class IptablesManager(object): @@ -229,9 +229,9 @@ class IptablesManager(object): for tables in [self.ipv4, self.ipv6]: tables['filter'].add_chain('nova-filter-top', wrap=False) tables['filter'].add_rule('FORWARD', '-j nova-filter-top', - wrap=False, head=True) + wrap=False, top=True) tables['filter'].add_rule('OUTPUT', '-j nova-filter-top', - wrap=False, head=True) + wrap=False, top=True) tables['filter'].add_chain('local') tables['filter'].add_rule('nova-filter-top', '-j $local', @@ -320,9 +320,19 @@ class IptablesManager(object): if not rule.startswith(':'): break - new_filter[rules_index:rules_index] = [str(rule) for rule in rules - if rule.head or - str(rule) not in new_filter] + our_rules = [] + for rule in rules: + rule_str = str(rule) + if rule.top: + # rule.top == True means we want this rule to be at the top. + # Further down, we weed out duplicates from the bottom of the + # list, so here we remove the dupes ahead of time. + new_filter = filter(lambda s: s.strip() != rule_str.strip(), + new_filter) + our_rules += [rule_str] + + new_filter[rules_index:rules_index] = our_rules + new_filter[rules_index:rules_index] = [':%s - [0:0]' % \ (name,) \ for name in unwrapped_chains] @@ -333,13 +343,18 @@ class IptablesManager(object): seen_lines = set() def _weed_out_duplicates(line): + line = line.strip() if line in seen_lines: return False else: seen_lines.add(line) return True + # We filter duplicates, letting the *last* occurrence take + # precendence. + new_filter.reverse() new_filter = filter(_weed_out_duplicates, new_filter) + new_filter.reverse() return new_filter diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 2bdf3709e..b1d70e323 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -46,20 +46,20 @@ class IptablesManagerTestCase(test.TestCase): :nova-compute-local - [0:0] :nova-compute-OUTPUT - [0:0] :nova-filter-top - [0:0] --A FORWARD -j nova-filter-top --A OUTPUT -j nova-filter-top --A nova-filter-top -j nova-compute-local --A INPUT -j nova-compute-INPUT --A OUTPUT -j nova-compute-OUTPUT --A FORWARD -j nova-compute-FORWARD --A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT --A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT --A FORWARD -i virbr0 -o virbr0 -j ACCEPT --A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable --A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -j nova-filter-top +-A OUTPUT -j nova-filter-top +-A nova-filter-top -j nova-compute-local +-A INPUT -j nova-compute-INPUT +-A OUTPUT -j nova-compute-OUTPUT +-A FORWARD -j nova-compute-FORWARD +-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT +-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT +-A FORWARD -i virbr0 -o virbr0 -j ACCEPT +-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable COMMIT # Completed on Fri Feb 18 15:17:05 2011""" @@ -75,12 +75,12 @@ COMMIT :nova-compute-PREROUTING - [0:0] :nova-compute-POSTROUTING - [0:0] :nova-postrouting-bottom - [0:0] --A PREROUTING -j nova-compute-PREROUTING --A OUTPUT -j nova-compute-OUTPUT --A POSTROUTING -j nova-compute-POSTROUTING --A POSTROUTING -j nova-postrouting-bottom --A nova-postrouting-bottom -j nova-compute-SNATTING --A nova-compute-SNATTING -j nova-compute-floating-ip-snat +-A PREROUTING -j nova-compute-PREROUTING +-A OUTPUT -j nova-compute-OUTPUT +-A POSTROUTING -j nova-compute-POSTROUTING +-A POSTROUTING -j nova-postrouting-bottom +-A nova-postrouting-bottom -j nova-compute-SNATTING +-A nova-compute-SNATTING -j nova-compute-floating-ip-snat COMMIT # Completed on Fri Feb 18 15:17:05 2011 """ @@ -118,6 +118,7 @@ COMMIT seen_lines = set() for line in new_lines: + line = line.strip() self.assertTrue(line not in seen_lines, "Duplicate line: %s" % line) seen_lines.add(line) @@ -151,6 +152,7 @@ COMMIT seen_lines = set() for line in new_lines: + line = line.strip() self.assertTrue(line not in seen_lines, "Duplicate line: %s" % line) seen_lines.add(line) -- cgit From 7eee81ee6480a36b179ae26be88ebad9415c4b62 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 14:40:00 +0100 Subject: PEP8 again --- nova/tests/test_network.py | 103 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index b1d70e323..d3a23abf4 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -36,61 +36,62 @@ LOG = logging.getLogger('nova.tests.network') class IptablesManagerTestCase(test.TestCase): - sample_filter = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 -*filter -:INPUT ACCEPT [2223527:305688874] -:FORWARD ACCEPT [0:0] -:OUTPUT ACCEPT [2172501:140856656] -:nova-compute-FORWARD - [0:0] -:nova-compute-INPUT - [0:0] -:nova-compute-local - [0:0] -:nova-compute-OUTPUT - [0:0] -:nova-filter-top - [0:0] --A FORWARD -j nova-filter-top --A OUTPUT -j nova-filter-top --A nova-filter-top -j nova-compute-local --A INPUT -j nova-compute-INPUT --A OUTPUT -j nova-compute-OUTPUT --A FORWARD -j nova-compute-FORWARD --A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT --A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT --A FORWARD -i virbr0 -o virbr0 -j ACCEPT --A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable --A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable -COMMIT -# Completed on Fri Feb 18 15:17:05 2011""" - - sample_nat = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 -*nat -:PREROUTING ACCEPT [3936:762355] -:INPUT ACCEPT [2447:225266] -:OUTPUT ACCEPT [63491:4191863] -:POSTROUTING ACCEPT [63112:4108641] -:nova-compute-OUTPUT - [0:0] -:nova-compute-floating-ip-snat - [0:0] -:nova-compute-SNATTING - [0:0] -:nova-compute-PREROUTING - [0:0] -:nova-compute-POSTROUTING - [0:0] -:nova-postrouting-bottom - [0:0] --A PREROUTING -j nova-compute-PREROUTING --A OUTPUT -j nova-compute-OUTPUT --A POSTROUTING -j nova-compute-POSTROUTING --A POSTROUTING -j nova-postrouting-bottom --A nova-postrouting-bottom -j nova-compute-SNATTING --A nova-compute-SNATTING -j nova-compute-floating-ip-snat -COMMIT -# Completed on Fri Feb 18 15:17:05 2011 -""" + sample_filter = ['#Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*filter', + ':INPUT ACCEPT [2223527:305688874]', + ':FORWARD ACCEPT [0:0]', + ':OUTPUT ACCEPT [2172501:140856656]', + ':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-filter-top - [0:0]', + '-A FORWARD -j nova-filter-top ', + '-A OUTPUT -j nova-filter-top ', + '-A nova-filter-top -j nova-compute-local ', + '-A INPUT -j nova-compute-INPUT ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A FORWARD -j nova-compute-FORWARD ', + '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', + '-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT ', + '-A FORWARD -i virbr0 -o virbr0 -j ACCEPT ', + '-A FORWARD -o virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + '-A FORWARD -i virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] + + sample_nat = ['# Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*nat', + ':PREROUTING ACCEPT [3936:762355]', + ':INPUT ACCEPT [2447:225266]', + ':OUTPUT ACCEPT [63491:4191863]', + ':POSTROUTING ACCEPT [63112:4108641]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]', + ':nova-postrouting-bottom - [0:0]', + '-A PREROUTING -j nova-compute-PREROUTING ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A POSTROUTING -j nova-compute-POSTROUTING ', + '-A POSTROUTING -j nova-postrouting-bottom ', + '-A nova-postrouting-bottom -j nova-compute-SNATTING ', + '-A nova-compute-SNATTING -j nova-compute-floating-ip-snat ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] def setUp(self): super(IptablesManagerTestCase, self).setUp() self.manager = linux_net.IptablesManager() def test_filter_rules_are_wrapped(self): - current_lines = self.sample_filter.split('\n') + current_lines = self.sample_filter table = self.manager.ipv4['filter'] table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') @@ -104,7 +105,7 @@ COMMIT '-s 1.2.3.4/5 -j DROP' not in new_lines) def test_nat_rules(self): - current_lines = self.sample_nat.split('\n') + current_lines = self.sample_nat new_lines = self.manager._modify_rules(current_lines, self.manager.ipv4['nat']) @@ -139,7 +140,7 @@ COMMIT "Built-in chain %s not wrapped" % (chain,)) def test_filter_rules(self): - current_lines = self.sample_filter.split('\n') + current_lines = self.sample_filter new_lines = self.manager._modify_rules(current_lines, self.manager.ipv4['filter']) -- cgit From 8b0e8b155eab313e0caece48eee609d12df5e5d4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 20:45:36 +0100 Subject: Rename "SNATTING" chain to "snat". --- nova/network/linux_net.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 849181641..34337901a 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -208,7 +208,7 @@ class IptablesManager(object): For ipv4, the builtin PREROUTING, OUTPUT, and POSTROUTING nat chains are wrapped in the same was as the builtin filter chains. Additionally, there's - a SNATTING chain that is applied after the POSTROUTING chain. + a snat chain that is applied after the POSTROUTING chain. """ def __init__(self, execute=None): if not execute: @@ -261,16 +261,16 @@ class IptablesManager(object): self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', wrap=False) - # We add a SNATTING chain to the shared nova-postrouting-bottom chain + # We add a snat chain to the shared nova-postrouting-bottom chain # so that it's applied last. - self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', + self.ipv4['nat'].add_chain('snat') + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $snat', wrap=False) # And then we add a floating-snat chain and jump to first thing in - # the SNATTING chain. + # the snat chain. self.ipv4['nat'].add_chain('floating-snat') - self.ipv4['nat'].add_rule('SNATTING', '-j $floating-snat') + self.ipv4['nat'].add_rule('snat', '-j $floating-snat') self.semaphore = semaphore.Semaphore() @@ -376,7 +376,7 @@ def init_host(): # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - iptables_manager.ipv4['nat'].add_rule("SNATTING", + iptables_manager.ipv4['nat'].add_rule("snat", "-s %s -j SNAT --to-source %s" % \ (FLAGS.fixed_range, FLAGS.routing_source_ip)) -- cgit From f4289df0e58080d6d9fa381915bbd0d29f3b9751 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 15:05:48 -0800 Subject: We're not using prefix matching on AMQP, so fakerabbit shouldn't be doing it! --- nova/fakerabbit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/fakerabbit.py b/nova/fakerabbit.py index dd82a9366..a7dee8caf 100644 --- a/nova/fakerabbit.py +++ b/nova/fakerabbit.py @@ -48,7 +48,6 @@ class Exchange(object): nm = self.name LOG.debug(_('(%(nm)s) publish (key: %(routing_key)s)' ' %(message)s') % locals()) - routing_key = routing_key.split('.')[0] if routing_key in self._routes: for f in self._routes[routing_key]: LOG.debug(_('Publishing to route %s'), f) -- cgit From e8a3f461319c1ee20a18f3a375af5e1e958af05e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 15:50:42 -0800 Subject: Missing import for nova.exceptions (!) --- nova/network/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/network/api.py b/nova/network/api.py index bf43acb51..4ee1148cb 100644 --- a/nova/network/api.py +++ b/nova/network/api.py @@ -21,6 +21,7 @@ Handles all requests relating to instances (guest vms). """ from nova import db +from nova import exception from nova import flags from nova import log as logging from nova import quota -- cgit From ab6b11b0399655ccdd9619be00470eda464cf2a7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 16:09:41 -0800 Subject: Don't blindly concatenate queue name if second portiion is None --- nova/db/sqlalchemy/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..009ed1f06 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1209,7 +1209,9 @@ def project_get_network_v6(context, project_id): def queue_get_for(_context, topic, physical_node_id): # FIXME(ja): this should be servername? - return "%s.%s" % (topic, physical_node_id) + if physical_node_id: + return "%s.%s" % (topic, physical_node_id) + return topic ################### -- cgit From b6254db80ca9841361775a92b85f88db7251f857 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 17:45:38 -0800 Subject: Refactoring nova-api to be a service, so that we can reuse it in tests --- bin/nova-api | 50 ++++---------------------------- nova/apiservice.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 45 deletions(-) create mode 100644 nova/apiservice.py diff --git a/bin/nova-api b/bin/nova-api index d5efb4687..99417e6c6 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -36,57 +36,17 @@ gettext.install('nova', unicode=1) from nova import flags from nova import log as logging -from nova import version +from nova import apiservice +from nova import utils from nova import wsgi -LOG = logging.getLogger('nova.api') - FLAGS = flags.FLAGS -flags.DEFINE_string('ec2_listen', "0.0.0.0", - 'IP address for EC2 API to listen') -flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') -flags.DEFINE_string('osapi_listen', "0.0.0.0", - 'IP address for OpenStack API to listen') -flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') - -API_ENDPOINTS = ['ec2', 'osapi'] - - -def run_app(paste_config_file): - LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) - apps = [] - for api in API_ENDPOINTS: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - LOG.debug(_("No paste configuration for app: %s"), api) - continue - LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) - LOG.info(_("Running %s API"), api) - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_listen_port" % api), - getattr(FLAGS, "%s_listen" % api))) - if len(apps) == 0: - LOG.error(_("No known API applications configured in %s."), - paste_config_file) - return - - server = wsgi.Server() - for app in apps: - server.start(*app) - server.wait() - if __name__ == '__main__': FLAGS(sys.argv) logging.setup() - LOG.audit(_("Starting nova-api node (version %s)"), - version.version_string_with_vcs()) - LOG.debug(_("Full set of FLAGS:")) - for flag in FLAGS: - flag_get = FLAGS.get(flag, None) - LOG.debug("%(flag)s : %(flag_get)s" % locals()) conf = wsgi.paste_config_file('nova-api.conf') - if conf: - run_app(conf) - else: + if not conf: LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') + else: + apiservice.serve(conf) diff --git a/nova/apiservice.py b/nova/apiservice.py new file mode 100644 index 000000000..7b453e19f --- /dev/null +++ b/nova/apiservice.py @@ -0,0 +1,85 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Wrapper for API service, makes it look more like the non-WSGI services +""" + +from nova import flags +from nova import log as logging +from nova import version +from nova import wsgi + + +LOG = logging.getLogger('nova.api') + +FLAGS = flags.FLAGS +flags.DEFINE_string('ec2_listen', "0.0.0.0", + 'IP address for EC2 API to listen') +flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') +flags.DEFINE_string('osapi_listen', "0.0.0.0", + 'IP address for OpenStack API to listen') +flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') + +API_ENDPOINTS = ['ec2', 'osapi'] + + +def _run_app(paste_config_file): + LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) + apps = [] + for api in API_ENDPOINTS: + config = wsgi.load_paste_configuration(paste_config_file, api) + if config is None: + LOG.debug(_("No paste configuration for app: %s"), api) + continue + LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) + LOG.info(_("Running %s API"), api) + app = wsgi.load_paste_app(paste_config_file, api) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) + if len(apps) == 0: + LOG.error(_("No known API applications configured in %s."), + paste_config_file) + return + + server = wsgi.Server() + for app in apps: + server.start(*app) + server.wait() + + +class ApiService(object): + """Base class for workers that run on hosts.""" + + def __init__(self, conf): + self.conf = conf + + def start(self): + _run_app(self.conf) + + +def serve(conf): + LOG.audit(_("Starting nova-api node (version %s)"), + version.version_string_with_vcs()) + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) + + service = ApiService(conf) + service.start() -- cgit From 79f6c437b486262bab3faacb59197a5cae30b2bd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 17:50:26 -0800 Subject: Added create static method to ApiService --- nova/apiservice.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nova/apiservice.py b/nova/apiservice.py index 7b453e19f..1914b9e59 100644 --- a/nova/apiservice.py +++ b/nova/apiservice.py @@ -72,6 +72,11 @@ class ApiService(object): def start(self): _run_app(self.conf) + @staticmethod + def create(): + conf = wsgi.paste_config_file('nova-api.conf') + return serve(conf) + def serve(conf): LOG.audit(_("Starting nova-api node (version %s)"), @@ -83,3 +88,5 @@ def serve(conf): service = ApiService(conf) service.start() + + return service -- cgit From e37e7b91a9fb5664ad50c1ff38e95f1a2d655c06 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 17:58:01 -0800 Subject: Support service-like wait behaviour for API service --- bin/nova-api | 3 ++- nova/apiservice.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 99417e6c6..ccb7701ae 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -49,4 +49,5 @@ if __name__ == '__main__': if not conf: LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') else: - apiservice.serve(conf) + service = apiservice.serve(conf) + service.wait() diff --git a/nova/apiservice.py b/nova/apiservice.py index 1914b9e59..14239f196 100644 --- a/nova/apiservice.py +++ b/nova/apiservice.py @@ -60,7 +60,7 @@ def _run_app(paste_config_file): server = wsgi.Server() for app in apps: server.start(*app) - server.wait() + return server class ApiService(object): @@ -68,9 +68,13 @@ class ApiService(object): def __init__(self, conf): self.conf = conf + self.wsgi_app = None def start(self): - _run_app(self.conf) + self.wsgi_app = _run_app(self.conf) + + def wait(self): + self.wsgi_app.wait() @staticmethod def create(): -- cgit From 7c8c325f475926724f3243344803841e24d5cb84 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:15:29 -0800 Subject: Make static create method behave more like other services --- nova/apiservice.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/apiservice.py b/nova/apiservice.py index 14239f196..693bc9a63 100644 --- a/nova/apiservice.py +++ b/nova/apiservice.py @@ -79,7 +79,10 @@ class ApiService(object): @staticmethod def create(): conf = wsgi.paste_config_file('nova-api.conf') - return serve(conf) + LOG.audit(_("Starting nova-api node (version %s)"), + version.version_string_with_vcs()) + service = ApiService(conf) + return service def serve(conf): -- cgit From dd6b9c21d3ad493051f25ce632fb327ed7fc7b73 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:57:04 -0800 Subject: Exit with exit code 1 if conf cannot be read --- bin/nova-api | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-api b/bin/nova-api index ccb7701ae..d03be85e3 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -48,6 +48,7 @@ if __name__ == '__main__': conf = wsgi.paste_config_file('nova-api.conf') if not conf: LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') + sys.exit(1) else: service = apiservice.serve(conf) service.wait() -- cgit From 50e71cef14c3bd079fbc2d2c203b0e0f76ee869e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:59:23 -0800 Subject: Removed unused import & formatting cleanups --- bin/nova-api | 1 - nova/apiservice.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index d03be85e3..96c784624 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -37,7 +37,6 @@ gettext.install('nova', unicode=1) from nova import flags from nova import log as logging from nova import apiservice -from nova import utils from nova import wsgi FLAGS = flags.FLAGS diff --git a/nova/apiservice.py b/nova/apiservice.py index 693bc9a63..6340e9b9b 100644 --- a/nova/apiservice.py +++ b/nova/apiservice.py @@ -16,9 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Wrapper for API service, makes it look more like the non-WSGI services -""" +"""Wrapper for API service, makes it look more like the non-WSGI services""" from nova import flags from nova import log as logging @@ -28,6 +26,7 @@ from nova import wsgi LOG = logging.getLogger('nova.api') + FLAGS = flags.FLAGS flags.DEFINE_string('ec2_listen', "0.0.0.0", 'IP address for EC2 API to listen') @@ -36,6 +35,7 @@ flags.DEFINE_string('osapi_listen', "0.0.0.0", 'IP address for OpenStack API to listen') flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') + API_ENDPOINTS = ['ec2', 'osapi'] -- cgit From fbfc2b21657a2878ab97138c133a253f7c88303e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 15:17:32 -0800 Subject: Alphabetize imports --- bin/nova-api | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 96c784624..933202dc8 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -34,9 +34,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import apiservice from nova import flags from nova import log as logging -from nova import apiservice from nova import wsgi FLAGS = flags.FLAGS -- cgit From 861a7f2b53f02af2ef196411171182394edd7e17 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 15:31:40 -0800 Subject: Changed create from a @staticmethod to a @classmethod --- nova/apiservice.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/apiservice.py b/nova/apiservice.py index 6340e9b9b..03aa781fb 100644 --- a/nova/apiservice.py +++ b/nova/apiservice.py @@ -76,12 +76,12 @@ class ApiService(object): def wait(self): self.wsgi_app.wait() - @staticmethod - def create(): + @classmethod + def create(cls): conf = wsgi.paste_config_file('nova-api.conf') LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) - service = ApiService(conf) + service = cls(conf) return service -- cgit From 503fe37427247b2728051426d3c40266de69bd71 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 15:59:54 -0800 Subject: Reverted bad-fix to sqlalchemy code --- nova/db/sqlalchemy/api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 53498fbc5..d8751bef4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1209,9 +1209,7 @@ def project_get_network_v6(context, project_id): def queue_get_for(_context, topic, physical_node_id): # FIXME(ja): this should be servername? - if physical_node_id: - return "%s.%s" % (topic, physical_node_id) - return topic + return "%s.%s" % (topic, physical_node_id) ################### -- cgit From 7c6638cc5effc7d727087ed4354c180d75476d1a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 25 Feb 2011 13:40:15 -0800 Subject: API changed to new style class --- nova/scheduler/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 8491bf3a9..2405f1343 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -25,7 +25,7 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.api') -class API: +class API(object): """API for interacting with the scheduler.""" def _call_scheduler(self, method, context, params=None): -- cgit From 4229990fa77d6edb73b88e92750a8779c478e40c Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sat, 26 Feb 2011 19:09:57 +0100 Subject: replaced ConnectionFailed with Exception in tools/euca-get-ajax-console was not working for me with euca2tools 1.2 (version 2007-10-10, release 31337) --- tools/euca-get-ajax-console | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/euca-get-ajax-console b/tools/euca-get-ajax-console index 37060e74f..e407dd566 100755 --- a/tools/euca-get-ajax-console +++ b/tools/euca-get-ajax-console @@ -35,7 +35,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): import boto import nova from boto.ec2.connection import EC2Connection -from euca2ools import Euca2ool, InstanceValidationError, Util, ConnectionFailed +from euca2ools import Euca2ool, InstanceValidationError, Util usage_string = """ Retrieves a url to an ajax console terminal @@ -147,7 +147,7 @@ def main(): try: euca_conn = euca.make_connection() - except ConnectionFailed, e: + except Exception, e: print e.message sys.exit(1) try: -- cgit From 38c21546ecc079300c575e5950bcb990eecee3a3 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:28:04 -0500 Subject: execute: shell=True removed. --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..40a8d8d8c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,7 +125,7 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl --fail %s -o %s" % (url, target)) + execute("curl","--fail",url,"-o",target) def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): @@ -133,7 +133,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip -f inet6 -o addr show %s" % interface) + if_str = execute("ip","-f","inet6","-o","addr","show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] -- cgit From 4f90783224025618661bf8814e016843ec237875 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:52:32 -0500 Subject: execvp --- nova/volume/driver.py | 69 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e3744c790..e73202b73 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -97,22 +97,21 @@ class VolumeDriver(object): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("sudo lvcreate -L %s -n %s %s" % - (sizestr, + self._try_execute('sudo','lvcreate','-L',sizestr,'-n', volume['name'], - FLAGS.volume_group)) + FLAGS.volume_group) def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute("sudo lvdisplay %s/%s" % + self._try_execute('sudo','lvdisplay','%s/%s" % (FLAGS.volume_group, volume['name'])) except Exception as e: # If the volume isn't present, then don't attempt to delete return True - self._try_execute("sudo lvremove -f %s/%s" % + self._try_execute('sudo','lvremove','-f',"%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -168,12 +167,13 @@ class AOEDriver(VolumeDriver): blade_id) = self.db.volume_allocate_shelf_and_blade(context, volume['id']) self._try_execute( - "sudo vblade-persist setup %s %s %s /dev/%s/%s" % - (shelf_id, + 'sudo','vblade-persist','setup', + shelf_id, blade_id, FLAGS.aoe_eth_dev, - FLAGS.volume_group, - volume['name'])) + "/dev/%s/%s" % + (FLAGS.volume_group, + volume['name'])) # NOTE(vish): The standard _try_execute does not work here # because these methods throw errors if other # volumes on this host are in the process of @@ -182,9 +182,9 @@ class AOEDriver(VolumeDriver): # just wait a bit for the current volume to # be ready and ignore any errors. time.sleep(2) - self._execute("sudo vblade-persist auto all", + self._execute('sudo','vblade-persist','auto','all', check_exit_code=False) - self._execute("sudo vblade-persist start all", + self._execute('sudo','vblade-persist','start','all', check_exit_code=False) def remove_export(self, context, volume): @@ -192,15 +192,15 @@ class AOEDriver(VolumeDriver): (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) - self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + self._try_execute('sudo','vblade-persist','stop', + shelf_id, blade_id) + self._try_execute('sudo','vblade-persist','destroy', + shelf_id, blade_id) def discover_volume(self, _volume): """Discover volume on a remote host.""" - self._execute("sudo aoe-discover") - self._execute("sudo aoe-stat", check_exit_code=False) + self._execute('sudo','aoe-discover') + self._execute('sudo','aoe-stat', check_exit_code=False) def undiscover_volume(self, _volume): """Undiscover volume on a remote host.""" @@ -252,13 +252,16 @@ class ISCSIDriver(VolumeDriver): iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - self._sync_exec("sudo ietadm --op new " - "--tid=%s --params Name=%s" % - (iscsi_target, iscsi_name), + self._sync_exec('sudo','ietadm','--op','new', + "--tid=%s" % iscsi_target, + '--params', + "Name=%s" % iscsi-name, check_exit_code=False) - self._sync_exec("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path), + self._sync_exec('sudo','ietadm','--op','new', + "--tid=%s" % iscsi_target, + '--lun=0', + '--params', + "Path=%s,Type=fileio" % volume_path, check_exit_code=False) def _ensure_iscsi_targets(self, context, host): @@ -490,16 +493,13 @@ class RBDDriver(VolumeDriver): size = 100 else: size = int(volume['size']) * 1024 - self._try_execute("rbd --pool %s --size %d create %s" % - (FLAGS.rbd_pool, - size, - volume['name'])) + self._try_execute('rbd','--pool',FLAGS.rbd_pool, + '--size', size,'create', volume['name']) def delete_volume(self, volume): """Deletes a logical volume.""" - self._try_execute("rbd --pool %s rm %s" % - (FLAGS.rbd_pool, - volume['name'])) + self._try_execute('rbd','--pool',FLAGS.rbd_pool, + 'rm', voluname['name']) def local_path(self, volume): """Returns the path of the rbd volume.""" @@ -534,7 +534,7 @@ class SheepdogDriver(VolumeDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" try: - (out, err) = self._execute("collie cluster info") + (out, err) = self._execute('collie','cluster','info') if not out.startswith('running'): raise exception.Error(_("Sheepdog is not working: %s") % out) except exception.ProcessExecutionError: @@ -546,12 +546,13 @@ class SheepdogDriver(VolumeDriver): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("qemu-img create sheepdog:%s %s" % - (volume['name'], sizestr)) + self._try_execute('qemu-img','create', + "sheepdog:%s" %s" % volume['name'], + sizestr) def delete_volume(self, volume): """Deletes a logical volume""" - self._try_execute("collie vdi delete %s" % volume['name']) + self._try_execute('collie','vdi','delete',volume['name']) def local_path(self, volume): return "sheepdog:%s" % volume['name'] -- cgit From 953efce36b74c18a32ef9c42e6b1a57190e3ff6e Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:53:53 -0500 Subject: execvp --- nova/crypto.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index a34b940f5..b240a3958 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,8 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen -q -b %d -N "" -f %s' % (bits, keyfile)) - (out, err) = utils.execute('ssh-keygen -q -l -f %s.pub' % (keyfile)) + utils.execute('ssh-keygen','-q','-b',"%d" % bits,'-N','""','-f',keyfile) + (out, err) = utils.execute('ssh-keygen','-q','-l','-f',"%s.pub" % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +118,7 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen -y -f /dev/stdin', private_key) + # public_key, err = execute('ssh-keygen','-y','-f','/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,8 +143,8 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute("openssl ca -config ./openssl.cnf -revoke '%s'" % file_name) - utils.execute("openssl ca -gencrl -config ./openssl.cnf -out '%s'" % + utils.execute('openssl','ca','-config','./openssl.cnf','-revoke',"'%s'" % file_name) + utils.execute('openssl','ca','-gencrl','-config','./openssl.cnf','-out',"'%s'" % FLAGS.crl_file) os.chdir(start) @@ -193,9 +193,8 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.execute("openssl req -new -key %s -out %s -batch -subj %s" % - (keyfile, csrfile, subject)) + utils.execute('openssl','genrsa','-out',keyfile,bits) + utils.execute('openssl','req','-new','-key',keyfile,'-out',csrfile,'-batch','-subj',subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -212,8 +211,7 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute("sh geninter.sh %s %s" % - (project_id, _project_cert_subject(project_id))) + utils.execute('sh','geninter.sh',project_id, _project_cert_subject(project_id)) os.chdir(start) @@ -228,8 +226,8 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute("sh genvpn.sh %s %s" % - (project_id, _vpn_cert_subject(project_id))) + utils.execute('sh','genvpn.sh', + project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() (serial, signed_csr) = sign_csr(csr_text, project_id) @@ -259,9 +257,9 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute("openssl ca -batch -out %s -config " - "./openssl.cnf -infiles %s" % (outbound, inbound)) - out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) + utils.execute('openssl','ca','-batch','-out',outbound,'-config' + './openssl.cnf','-infiles',inbound) + out, _err = utils.execute('openssl','x509','-in',outbound','-serial','-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: -- cgit From 90abcdc7ae9e3f855dadb1ccc88892a2cc7bab05 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:57:13 -0500 Subject: execvp --- nova/console/xvp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/console/xvp.py b/nova/console/xvp.py index cd257e0a6..271dffa54 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -133,10 +133,10 @@ class XVPConsoleProxy(object): return logging.debug(_("Starting xvp")) try: - utils.execute('xvp -p %s -c %s -l %s' % - (FLAGS.console_xvp_pid, - FLAGS.console_xvp_conf, - FLAGS.console_xvp_log)) + utils.execute('xvp', + '-p',FLAGS.console_xvp_pid, + '-c',FLAGS.console_xvp_conf, + '-l',FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) @@ -190,5 +190,5 @@ class XVPConsoleProxy(object): flag = '-x' #xvp will blow up on passwords that are too long (mdragon) password = password[:maxlen] - out, err = utils.execute('xvp %s' % flag, process_input=password) + out, err = utils.execute('xvp', flag, process_input=password) return out.strip() -- cgit From 8b3e9ad11c2f5c425701f1eb4abb7b3f577ae1cc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 12:37:02 +0100 Subject: Add utils.synchronized decorator to allow for synchronising method entrance across multiple workers on the same host. --- nova/tests/test_misc.py | 37 ++++++++++++++++++++++++++++++++++++- nova/utils.py | 11 +++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index e6da6112a..154b6fae6 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,10 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +from datetime import datetime +import errno import os +import select +import time from nova import test -from nova.utils import parse_mailmap, str_dict_replace +from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): @@ -55,3 +59,34 @@ class ProjectTestCase(test.TestCase): '%r not listed in Authors' % missing) finally: tree.unlock() + + +class LockTestCase(test.TestCase): + def test_synchronized(self): + rpipe, wpipe = os.pipe() + pid = os.fork() + if pid > 0: + os.close(wpipe) + + @synchronized('testlock') + def f(): + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + + f() + else: + os.close(rpipe) + + @synchronized('testlock') + def g(): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + g() + os._exit(0) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..cb1ea5a7d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -25,6 +25,7 @@ import base64 import datetime import inspect import json +import lockfile import os import random import socket @@ -491,6 +492,16 @@ def loads(s): return json.loads(s) +def synchronized(name): + def wrap(f): + def inner(*args, **kwargs): + lock = lockfile.FileLock('nova-%s.lock' % name) + with lock: + return f(*args, **kwargs) + return inner + return wrap + + def ensure_b64_encoding(val): """Safety method to ensure that values expected to be base64-encoded actually are. If they are, the value is returned unchanged. Otherwise, -- cgit From 0d3a1f9c7478ae3c4f042e1876d47359804e1973 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 15:29:42 +0100 Subject: Wrap IptablesManager.apply() calls in utils.synchronized to avoid having different workers step on each other's toes. --- nova/network/linux_net.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 34337901a..8832f7c68 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -274,6 +274,7 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() + @utils.synchronized('iptables') def apply(self): """Apply the current in-memory set of iptables rules -- cgit From 7a6daa8d92f4f11fe2fce8fb2f4b11d96cb98c2d Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Mon, 28 Feb 2011 17:27:19 +0000 Subject: Suppress stack traces unless --verbose is specified --- nova/log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/log.py b/nova/log.py index 87a21ddb4..d194ab8f0 100644 --- a/nova/log.py +++ b/nova/log.py @@ -266,7 +266,10 @@ class NovaRootLogger(NovaLogger): def handle_exception(type, value, tb): - logging.root.critical(str(value), exc_info=(type, value, tb)) + extra = {} + if FLAGS.verbose: + extra['exc_info'] = (type, value, tb) + logging.root.critical(str(value), **extra) def reset(): -- cgit From c1bcf1dead8734a02172b4ac20b24fbbb7dbb993 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 28 Feb 2011 11:40:22 -0600 Subject: Rename migration to coincide with latest trunk changes --- .../versions/004_add_instance_migrations.py | 61 ---------------------- .../versions/007_add_instance_migrations.py | 61 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 61 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py deleted file mode 100644 index 4fda525f1..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_migrations.py +++ /dev/null @@ -1,61 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License.from sqlalchemy import * - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -# Just for the ForeignKey and column creation to succeed, these are not the -# actual definitions of instances or services. -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -# -# New Tables -# - -migrations = Table('migrations', meta, - Column('created_at', DateTime(timezone=False)), - Column('updated_at', DateTime(timezone=False)), - Column('deleted_at', DateTime(timezone=False)), - Column('deleted', Boolean(create_constraint=True, name=None)), - Column('id', Integer(), primary_key=True, nullable=False), - Column('source_compute', String(255)), - Column('dest_compute', String(255)), - Column('dest_host', String(255)), - Column('instance_id', Integer, ForeignKey('instances.id'), - nullable=True), - Column('status', String(255)), - ) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - for table in (migrations, ): - try: - table.create() - except Exception: - logging.info(repr(table)) - logging.exception('Exception while creating table') - raise diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py new file mode 100644 index 000000000..4fda525f1 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py @@ -0,0 +1,61 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.from sqlalchemy import * + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +migrations = Table('migrations', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('source_compute', String(255)), + Column('dest_compute', String(255)), + Column('dest_host', String(255)), + Column('instance_id', Integer, ForeignKey('instances.id'), + nullable=True), + Column('status', String(255)), + ) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (migrations, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise -- cgit From 8da6796789767b1341cb5a650066b67ad3191c74 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 28 Feb 2011 12:30:02 -0600 Subject: Merge review fixes --- nova/virt/xenapi/vm_utils.py | 17 ++++-- nova/virt/xenapi/vmops.py | 27 ++++++---- .../xenserver/xenapi/etc/xapi.d/plugins/migration | 62 +++++----------------- 3 files changed, 45 insertions(+), 61 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index e7ad1f686..870660dea 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -300,6 +300,13 @@ class VMHelper(HelperBase): the UUID """ return session.call_xenapi('SR.get_by_name_label', sr_label)[0] + @classmethod + def get_sr_path(cls, session, sr_label='slices'): + """ Finds the SR and then coerces it into a path on the dom0 file + system """ + # TODO(mdietz): replace this with the flag once unified-images merges + return '/var/run/sr-mount/%s' % cls.get_sr(session, sr_label) + @classmethod def upload_image(cls, session, instance_id, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and @@ -508,13 +515,17 @@ class VMHelper(HelperBase): @classmethod def scan_sr(cls, session, instance_id=None, sr_ref=None): + """Scans the SR specified by sr_ref""" if sr_ref: LOG.debug(_("Re-scanning SR %s"), sr_ref) task = session.call_xenapi('Async.SR.scan', sr_ref) session.wait_for_task(instance_id, task) - else: - sr_ref = cls.get_sr(session) - session.call_xenapi('SR.scan', sr_ref) + + @classmethod + def scan_default_sr(cls, session): + """Looks for the system default SR and triggers a re-scan""" + sr_ref = cls.get_sr(session) + session.call_xenapi('SR.scan', sr_ref) def get_rrd(host, uuid): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index f5278ff07..b3e5627d8 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -298,8 +298,10 @@ class VMOps(object): VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) cow_uuid = vm_vdi_rec['uuid'] - params = {'host': dest, 'vdi_uuid': base_copy_uuid, - 'instance_id': instance.id, } + params = {'host': dest, + 'vdi_uuid': base_copy_uuid, + 'instance_id': instance.id, + 'sr_path': VMHelper.get_sr_path(self._session), } task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) @@ -308,8 +310,11 @@ class VMOps(object): # Now power down the instance and transfer the COW VHD self._shutdown(instance, vm_ref, method='clean') - params = {'host': dest, 'vdi_uuid': cow_uuid, - 'instance_id': instance.id, } + params = {'host': dest, + 'vdi_uuid': cow_uuid, + 'instance_id': instance.id, + 'sr_path': VMHelper.get_sr_path(self._session), } + task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) self._session.wait_for_task(instance.id, task) @@ -326,14 +331,15 @@ class VMOps(object): 'old_base_copy_uuid': disk_info['base_copy'], 'old_cow_uuid': disk_info['cow'], 'new_base_copy_uuid': new_base_copy_uuid, - 'new_cow_uuid': new_cow_uuid, } + 'new_cow_uuid': new_cow_uuid, + 'sr_path': VMHelper.get_sr_path(self._session), } task = self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) self._session.wait_for_task(instance.id, task) # Now we rescan the SR so we find the VHDs - VMHelper.scan_sr(self._session) + VMHelper.scan_default_sr(self._session) return new_cow_uuid @@ -411,7 +417,7 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def _shutdown(self, instance, vm, method='hard'): + def _shutdown(self, instance, vm, hard=True): """Shutdown an instance """ state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: @@ -421,10 +427,11 @@ class VMOps(object): try: task = None - if method == 'clean': - task = self._session.call_xenapi('Async.VM.clean_shutdown', vm) - else: + if hard: task = self._session.call_xenapi('Async.VM.hard_shutdown', vm) + else: + task = self._session.call_xenapi('Async.VM.clean_shutdown', vm) + self._session.wait_for_task(instance.id, task) except self.XenAPI.Failure, exc: LOG.exception(exc) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 7a6eefda2..4aa89863a 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -1,7 +1,6 @@ #!/usr/bin/env python -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -31,38 +30,6 @@ import XenAPIPlugin from pluginlib_nova import * configure_logging('migration') -SSH_HOSTS = '/root/.ssh/known_hosts' -DEVNULL = '/dev/null' -KEYSCAN = '/usr/bin/ssh-keyscan' -RSYNC = '/usr/bin/rsync' -FILE_SR_PATH = '/var/run/sr-mount' -IMAGE_PATH = '/images/' -VHD_UTIL = '/usr/sbin/vhd-util' - -def get_sr_path(session): - sr_ref = find_sr(session) - - if sr_ref is None: - raise Exception('Cannot find SR to read VDI from') - - sr_rec = session.xenapi.SR.get_record(sr_ref) - sr_uuid = sr_rec["uuid"] - sr_path = os.path.join(FILE_SR_PATH, sr_uuid) - return sr_path - -def find_sr(session): - host = get_this_host(session) - srs = session.xenapi.SR.get_all() - for sr in srs: - sr_rec = session.xenapi.SR.get_record(sr) - if not ('i18n-key' in sr_rec['other_config'] and - sr_rec['other_config']['i18n-key'] == 'local-storage'): - continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.xenapi.PBD.get_record(pbd) - if pbd_rec['host'] == host: - return sr - return None def move_vhds_into_sr(session, args): """Moves the VHDs from their copied location to the SR""" @@ -75,13 +42,13 @@ def move_vhds_into_sr(session, args): new_base_copy_uuid = params['new_base_copy_uuid'] new_cow_uuid = params['new_cow_uuid'] - sr_path = get_sr_path(session) + sr_path = params['sr_path'] sr_temp_path = "%s/images/" % sr_path - # Discover the copied VHDs locally, and then set up paths to copy + # Discover the copied VHDs locally, and then set up paths to copy # them to under the SR - source_image_path = "%s/instance%d" % (IMAGE_PATH, instance_id) - source_base_copy_path = "%s/%s.vhd" % (source_image_path, + source_image_path = "%s/instance%d" % ('/images/', instance_id) + source_base_copy_path = "%s/%s.vhd" % (source_image_path, old_base_copy_uuid) source_cow_path = "%s/%s.vhd" % (source_image_path, old_cow_uuid) @@ -102,11 +69,10 @@ def move_vhds_into_sr(session, args): os.rmdir(source_image_path) # Link the COW to the base copy - logging.debug('Attaching COW to the base copy %s -> %s' % + logging.debug('Attaching COW to the base copy %s -> %s' % (new_cow_path, new_base_copy_path)) - subprocess.call([VHD_UTIL, 'modify', '-n', new_cow_path, '-p', - new_base_copy_path]) - + subprocess.call(shlex.split('/usr/sbin/vhd-util modify -n %s -p %s' % + (new_cow_path, new_base_copy_path))) logging.debug('Moving VHDs into SR %s' % sr_path) shutil.move("%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid), sr_path) shutil.move("%s/%s.vhd" % (temp_vhd_path, new_cow_uuid), sr_path) @@ -122,19 +88,19 @@ def transfer_vhd(session, args): instance_id = params['instance_id'] host = params['host'] vdi_uuid = params['vdi_uuid'] - sr_path = get_sr_path(session) + sr_path = params['sr_path'] vhd_path = "%s.vhd" % vdi_uuid source_path = "%s/%s" % (sr_path, vhd_path) - dest_path = '%s:%sinstance%d/' % (host, IMAGE_PATH, instance_id) + dest_path = '%s:%sinstance%d/' % (host, '/images/', instance_id) - logging.debug("Preparing to transmit %s to %s" % (source_path, + logging.debug("Preparing to transmit %s to %s" % (source_path, dest_path)) ssh_cmd = 'ssh -o StrictHostKeyChecking=no' - rsync_args = ['nohup', RSYNC, '-av', '--progress', '-e', ssh_cmd, - source_path, dest_path] + rsync_args = shlex.split('nohup /usr/bin/rsync -av --progress -e %s %s %s' + % (ssh_cmd, source_path, dest_path)) logging.debug('rsync %s' % (' '.join(rsync_args, ))) @@ -148,4 +114,4 @@ def transfer_vhd(session, args): if __name__ == '__main__': XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd, - 'move_vhds_into_sr':move_vhds_into_sr, }) + 'move_vhds_into_sr': move_vhds_into_sr, }) -- cgit From d5736e925f288462f6325130be0af49f0ace5884 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 23:31:09 +0100 Subject: Add a lock_path flag for lock files. --- nova/flags.py | 2 ++ nova/utils.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index 8cf199b2f..213d4d4e1 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -321,6 +321,8 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), "Top-level directory for maintaining nova's state") +DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'), + "Directory for lock files") DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') diff --git a/nova/utils.py b/nova/utils.py index cb1ea5a7d..48f12350f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -44,11 +44,13 @@ from eventlet.green import subprocess from nova import exception from nova.exception import ProcessExecutionError +from nova import flags from nova import log as logging LOG = logging.getLogger("nova.utils") TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +FLAGS = flags.FLAGS def import_class(import_str): @@ -495,7 +497,8 @@ def loads(s): def synchronized(name): def wrap(f): def inner(*args, **kwargs): - lock = lockfile.FileLock('nova-%s.lock' % name) + lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name)) with lock: return f(*args, **kwargs) return inner -- cgit From be9004ffa4c70358c8edda1f33ffe7ba7e1ae1ee Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 1 Mar 2011 20:49:46 +0100 Subject: Use functools.wraps to make sure wrapped method's metadata (docstring and name) doesn't get mangled. --- nova/tests/test_misc.py | 12 ++++++++++-- nova/utils.py | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 154b6fae6..9f572b58e 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,11 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -from datetime import datetime import errno import os import select -import time from nova import test from nova.utils import parse_mailmap, str_dict_replace, synchronized @@ -62,6 +60,16 @@ class ProjectTestCase(test.TestCase): class LockTestCase(test.TestCase): + def test_synchronized_wrapped_function_metadata(self): + @synchronized('whatever') + def foo(): + """Bar""" + pass + self.assertEquals(foo.__doc__, 'Bar', "Wrapped function's docstring " + "got lost") + self.assertEquals(foo.__name__, 'foo', "Wrapped function's name " + "got mangled") + def test_synchronized(self): rpipe, wpipe = os.pipe() pid = os.fork() diff --git a/nova/utils.py b/nova/utils.py index 48f12350f..9929e6fef 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -23,11 +23,14 @@ System-level utilities and helper functions. import base64 import datetime +import functools import inspect import json import lockfile +import netaddr import os import random +import re import socket import string import struct @@ -35,8 +38,6 @@ import sys import time import types from xml.sax import saxutils -import re -import netaddr from eventlet import event from eventlet import greenthread @@ -496,6 +497,7 @@ def loads(s): def synchronized(name): def wrap(f): + @functools.wraps(f) def inner(*args, **kwargs): lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name)) -- cgit From cdb1b16a6019fd68a7969666d754c4007607ae53 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Tue, 1 Mar 2011 23:18:37 +0000 Subject: * Added ability to launch XenServer instances with per-os vm-params. --- nova/compute/api.py | 5 +- .../versions/007_add_os_type_to_instances.py | 45 ++++++ nova/db/sqlalchemy/models.py | 2 + nova/virt/xenapi/vm_utils.py | 152 +++++++++++++++------ nova/virt/xenapi/vmops.py | 17 +-- 5 files changed, 164 insertions(+), 57 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py diff --git a/nova/compute/api.py b/nova/compute/api.py index 625778b66..8bdf712a0 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -125,6 +125,8 @@ class API(base.Base): raise quota.QuotaError(msg, "MetadataLimitExceeded") image = self.image_service.show(context, image_id) + os_type = image['properties'].get('os_type', 'linux') + if kernel_id is None: kernel_id = image.get('kernel_id', None) if ramdisk_id is None: @@ -180,7 +182,8 @@ class API(base.Base): 'key_data': key_data, 'locked': False, 'metadata': metadata, - 'availability_zone': availability_zone} + 'availability_zone': availability_zone, + 'os_type': os_type} elevated = context.elevated() instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py new file mode 100644 index 000000000..d6d964b95 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# FIXME(dubs) should this be not null? Maybe create as nullable, then +# populate all existing rows with 'linux', then adding not null constraint. +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 1882efeba..b78c95e40 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -188,6 +188,8 @@ class Instance(BASE, NovaBase): locked = Column(Boolean) + os_type = Column(String(255)) + # TODO(vish): see Ewan's email about state improvements, probably # should be in a driver base class or some such # vmstate_state = running, halted, suspended, paused diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 6d9aeb060..11f1fabe9 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -80,62 +80,80 @@ class VMHelper(HelperBase): """ @classmethod - def create_vm(cls, session, instance, kernel, ramdisk, pv_kernel=False): + def create_vm(cls, session, instance, kernel, ramdisk, use_pv_kernel=False): """Create a VM record. Returns a Deferred that gives the new VM reference. - the pv_kernel flag indicates whether the guest is HVM or PV + the use_pv_kernel flag indicates whether the guest is HVM or PV + + There are 3 scenarios: + + 1. Using paravirtualization, kernel passed in + + 2. Using paravirtualization, kernel within the image + + 3. Using hardware virtualization """ instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) rec = { - 'name_label': instance.name, - 'name_description': '', + 'actions_after_crash': 'destroy', + 'actions_after_reboot': 'restart', + 'actions_after_shutdown': 'destroy', + 'affinity': '', + 'blocked_operations': {}, + 'ha_always_run': False, + 'ha_restart_priority': '', + 'HVM_boot_params': {}, + 'HVM_boot_policy': '', 'is_a_template': False, - 'memory_static_min': '0', - 'memory_static_max': mem, 'memory_dynamic_min': mem, 'memory_dynamic_max': mem, - 'VCPUs_at_startup': vcpus, - 'VCPUs_max': vcpus, - 'VCPUs_params': {}, - 'actions_after_shutdown': 'destroy', - 'actions_after_reboot': 'restart', - 'actions_after_crash': 'destroy', - 'PV_bootloader': '', - 'PV_kernel': '', - 'PV_ramdisk': '', + 'memory_static_min': '0', + 'memory_static_max': mem, + 'memory_target': mem, + 'name_description': '', + 'name_label': instance.name, +# 'other_config': {'allowvssprovider': False}, + 'other_config': {}, + 'PCI_bus': '', + 'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true', + 'viridian': 'true', 'timeoffset': '0'}, 'PV_args': '', + 'PV_bootloader': '', 'PV_bootloader_args': '', + 'PV_kernel': '', 'PV_legacy_args': '', - 'HVM_boot_policy': '', - 'HVM_boot_params': {}, - 'platform': {}, - 'PCI_bus': '', + 'PV_ramdisk': '', 'recommendations': '', - 'affinity': '', + 'tags': [], 'user_version': '0', - 'other_config': {}, + 'VCPUs_at_startup': vcpus, + 'VCPUs_max': vcpus, + 'VCPUs_params': {}, + 'xenstore_data': {} } - #Complete VM configuration record according to the image type - #non-raw/raw with PV kernel/raw in HVM mode - if instance.kernel_id: - rec['PV_bootloader'] = '' - rec['PV_kernel'] = kernel - rec['PV_ramdisk'] = ramdisk - rec['PV_args'] = 'root=/dev/xvda1' - rec['PV_bootloader_args'] = '' - rec['PV_legacy_args'] = '' - else: - if pv_kernel: - rec['PV_args'] = 'noninteractive' - rec['PV_bootloader'] = 'pygrub' + + # Complete VM configuration record according to the image type + # non-raw/raw with PV kernel/raw in HVM mode + if use_pv_kernel: + rec['platform']['nx'] = 'false' + if instance.kernel_id: + # 1. Kernel explicitly passed in, use that + rec['PV_args'] = 'root=/dev/xvda1' + rec['PV_kernel'] = kernel + rec['PV_ramdisk'] = ramdisk else: - rec['HVM_boot_policy'] = 'BIOS order' - rec['HVM_boot_params'] = {'order': 'dc'} - rec['platform'] = {'acpi': 'true', 'apic': 'true', - 'pae': 'true', 'viridian': 'true'} + # 2. Use kernel within the image + rec['PV_args'] = 'clocksource=jiffies' + rec['PV_bootloader'] = 'pygrub' + else: + # 3. Using hardware virtualization + rec['platform']['nx'] = 'true' + rec['HVM_boot_params'] = {'order': 'dc'} + rec['HVM_boot_policy'] = 'BIOS order' + LOG.debug(_('Created VM %s...'), instance.name) vm_ref = session.call_xenapi('VM.create', rec) instance_name = instance.name @@ -497,17 +515,32 @@ class VMHelper(HelperBase): return uuid @classmethod - def lookup_image(cls, session, instance_id, vdi_ref): + def determine_is_pv(cls, session, instance_id, vdi_ref, disk_image_type, + os_type): """ - Determine if VDI is using a PV kernel + Determine whether the VM will use a paravirtualized kernel or if it + will use hardware virtualization. + + 1. Objectstore (any image type): + We use plugin to figure out whether the VDI uses PV + + 2. Glance (VHD): then we use `os_type`, raise if not set + + 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is + available + + 4. Glance (DISK): pv is assumed """ if FLAGS.xenapi_image_service == 'glance': - return cls._lookup_image_glance(session, vdi_ref) + # 2, 3, 4: Glance + return cls._determine_is_pv_glance( + session, vdi_ref, disk_image_type, os_type) else: - return cls._lookup_image_objectstore(session, instance_id, vdi_ref) + # 1. Objecstore + return cls._determine_is_pv_objectstore(session, instance_id, vdi_ref) @classmethod - def _lookup_image_objectstore(cls, session, instance_id, vdi_ref): + def _determine_is_pv_objectstore(cls, session, instance_id, vdi_ref): LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) fn = "is_vdi_pv" args = {} @@ -523,9 +556,38 @@ class VMHelper(HelperBase): return pv @classmethod - def _lookup_image_glance(cls, session, vdi_ref): + def _determine_is_pv_glance(cls, session, vdi_ref, disk_image_type, + os_type): + """ + For a Glance image, determine if we need paravirtualization. + + The relevant scenarios are: + 2. Glance (VHD): then we use `os_type`, raise if not set + + 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is + available + + 4. Glance (DISK): pv is assumed + """ + LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) - return with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + if disk_image_type == ImageType.DISK_VHD: + # 2. VHD + if os_type == 'windows': + is_pv = False + else: + is_pv = True + elif disk_image_type == ImageType.DISK_RAW: + # 3. RAW + is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + elif disk_image_type == ImageType.DISK: + # 4. Disk + is_pv = True + else: + raise exception.Error(_("Unknown image format %(disk_image_type)s") + % locals()) + + return is_pv @classmethod def lookup(cls, session, i): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index bc39aa140..abc1fb699 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -87,15 +87,7 @@ class VMOps(object): vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - pv_kernel = False - if disk_image_type == ImageType.DISK_RAW: - #Have a look at the VDI and see if it has a PV kernel - pv_kernel = VMHelper.lookup_image(self._session, instance.id, - vdi_ref) - elif disk_image_type == ImageType.DISK_VHD: - # TODO(sirp): Assuming PV for now; this will need to be - # configurable as Windows will use HVM. - pv_kernel = True + os_type = instance.get('os_type', 'linux') kernel = None if instance.kernel_id: @@ -107,8 +99,11 @@ class VMOps(object): ramdisk = VMHelper.fetch_image(self._session, instance.id, instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) - vm_ref = VMHelper.create_vm(self._session, - instance, kernel, ramdisk, pv_kernel) + use_pv_kernel = VMHelper.determine_is_pv( + self._session, instance.id, vdi_ref, disk_image_type, os_type) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, + use_pv_kernel) + VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) # inject_network_info and create vifs -- cgit From 6321c5047c082bba8edf10a660fdb6a56430cc44 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 2 Mar 2011 00:19:02 +0000 Subject: * Added first cut of migration for os_type on instances table * Track os_type when taking snapshots --- .../migrate_repo/versions/007_add_os_type_to_instances.py | 4 +++- nova/virt/xenapi/vm_utils.py | 9 ++++++--- nova/virt/xenapi/vmops.py | 4 ++-- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 11 +++++++---- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py index d6d964b95..21f21b040 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py @@ -34,7 +34,7 @@ instances_os_type = Column('os_type', String(length=255, convert_unicode=False, assert_unicode=None, unicode_error=None, _warn_on_bytestring=False), - nullable=True) + nullable=False) def upgrade(migrate_engine): @@ -43,3 +43,5 @@ def upgrade(migrate_engine): meta.bind = migrate_engine instances.create_column(instances_os_type) + + diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 11f1fabe9..9c0bb5579 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -303,7 +303,7 @@ class VMHelper(HelperBase): return template_vm_ref, template_vdi_uuids @classmethod - def upload_image(cls, session, instance_id, vdi_uuids, image_id): + def upload_image(cls, session, instance, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and push them into Glance using the specified human-friendly name. """ @@ -312,15 +312,18 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) + # TODO(dubs): os_type is currently defaulting to linux, we actually + # want to make this a NOT NULL column and require it to be specified. params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, - 'sr_path': get_sr_path(session)} + 'sr_path': get_sr_path(session), + 'os_type': instance.get('os_type', 'linux')} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) - session.wait_for_task(instance_id, task) + session.wait_for_task(instance.id, task) @classmethod def fetch_image(cls, session, instance_id, image, user, project, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index abc1fb699..1edf39c5b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -238,11 +238,11 @@ class VMOps(object): try: # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, image_id) + self._session, instance, template_vdi_uuids, image_id) finally: self._destroy(instance, template_vm_ref, shutdown=False, destroy_kernel_ramdisk=False) - + logging.debug(_("Finished snapshot and upload for VM %s"), instance) def reboot(self, instance): diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 7531af4ec..160bf482f 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -191,7 +191,7 @@ def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): os.link(source, link_name) -def _upload_tarball(staging_path, image_id, glance_host, glance_port): +def _upload_tarball(staging_path, image_id, glance_host, glance_port, os_type): """ Create a tarball of the image and then stream that into Glance using chunked-transfer-encoded HTTP. @@ -205,9 +205,10 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): headers = { 'content-type': 'application/octet-stream', 'transfer-encoding': 'chunked', - 'x-image-meta-is_public': 'True', + 'x-image-meta-is-public': 'True', 'x-image-meta-status': 'queued', - 'x-image-meta-type': 'vhd' + 'x-image-meta-type': 'vhd', + 'x-image-meta-property-os-type': os_type } for header, value in headers.iteritems(): conn.putheader(header, value) @@ -330,11 +331,13 @@ def upload_vhd(session, args): glance_host = params["glance_host"] glance_port = params["glance_port"] sr_path = params["sr_path"] + os_type = params["os_type"] staging_path = _make_staging_area(sr_path) try: _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids) - _upload_tarball(staging_path, image_id, glance_host, glance_port) + _upload_tarball(staging_path, image_id, glance_host, glance_port, + os_type) finally: _cleanup_staging_area(staging_path) -- cgit From 58ac632f8e08b248d234deffdb56fe3a33a25130 Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 3 Mar 2011 00:12:48 +0900 Subject: Port Todd's lp720157 fix to the current trunk, rev 752. --- nova/api/ec2/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5adc2c075..5a63dc8da 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -198,6 +198,12 @@ class Requestify(wsgi.Middleware): try: # Raise KeyError if omitted action = req.params['Action'] + # Fix bug lp:720157 for older (version 1) clients + version = req.params['SignatureVersion'] + if int(version) == 1: + non_args.remove('SignatureMethod') + if 'SignatureMethod' in args: + args.pop('SignatureMethod') for non_arg in non_args: # Remove, but raise KeyError if omitted args.pop(non_arg) -- cgit From 953fe68ce9b27322003200c464c121464761d1e2 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 2 Mar 2011 15:46:50 -0600 Subject: merge fixes --- nova/compute/api.py | 4 +- nova/virt/xenapi/vm_utils.py | 4 +- nova/virt/xenapi/vmops.py | 48 ++++++++++------------ plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 2 +- 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 52d30c7f7..fb7ef1aed 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -430,7 +430,7 @@ class API(base.Base): migration_ref = self.db.migration_get_by_instance_and_status(context, instance_id, 'finished') if not migration_ref: - raise exception.Error(_("No finished migrations found for \ + raise exception.NotFound(_("No finished migrations found for \ instance")) params = {'migration_id': migration_ref['id']} @@ -444,7 +444,7 @@ class API(base.Base): migration_ref = self.db.migration_get_by_instance_and_status(context, instance_id, 'finished') if not migration_ref: - raise exception.Error(_("No finished migrations found for \ + raise exception.NotFound(_("No finished migrations found for \ instance")) instance_ref = self.db.instance_get(context, instance_id) params = {'migration_id': migration_ref['id']} diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index eea9bb0b9..3ee963fa7 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -310,8 +310,7 @@ class VMHelper(HelperBase): def get_sr_path(cls, session, sr_label='slices'): """ Finds the SR and then coerces it into a path on the dom0 file system """ - # TODO(mdietz): replace this with the flag once unified-images merges - return '/var/run/sr-mount/%s' % cls.get_sr(session, sr_label) + return FLAGS.xenapi_sr_base_path + cls.get_sr(session, sr_label) @classmethod def upload_image(cls, session, instance_id, vdi_uuids, image_id): @@ -649,6 +648,7 @@ class VMHelper(HelperBase): @classmethod def scan_default_sr(cls, session): """Looks for the system default SR and triggers a re-scan""" + #FIXME(sirp/mdietz): refactor scan_default_sr in there sr_ref = cls.get_sr(session) session.call_xenapi('SR.scan', sr_ref) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 5157f18f1..b54084842 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -105,7 +105,7 @@ class VMOps(object): vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', disk) if disk_image_type == ImageType.DISK_RAW: - #Have a look at the VDI and see if it has a PV kernel + # Have a look at the VDI and see if it has a PV kernel pv_kernel = VMHelper.lookup_image(self._session, instance.id, vdi_ref) elif disk_image_type == ImageType.DISK_VHD: @@ -113,7 +113,6 @@ class VMOps(object): # configurable as Windows will use HVM. pv_kernel = True - #Have a look at the VDI and see if it has a PV kernel if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) @@ -240,29 +239,20 @@ class VMOps(object): that will bundle the VHDs together and then push the bundle into Glance. """ - - with self._get_snapshot(instance) as snapshot: + template_vm_ref = None + try: + template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, snapshot.vdi_uuids, image_id) + self._session, instance.id, template_vdi_uuids, image_id) + finally: + if template_vm_ref: + self.virt._destroy(self.instance, template_vm_ref, shutdown=False, + destroy_kernel_ramdisk=False) logging.debug(_("Finished snapshot and upload for VM %s"), instance) def _get_snapshot(self, instance): - class Snapshot(object): - def __init__(self, virt, instance, vm_ref, vdis): - self.instance = instance - self.vdi_uuids = vdis - self.virt = virt - self.vm_ref = vm_ref - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.virt._destroy(self.instance, self.vm_ref, shutdown=False, - destroy_kernel_ramdisk=False) - #TODO(sirp): Add quiesce and VSS locking support when Windows support # is added @@ -273,8 +263,7 @@ class VMOps(object): try: template_vm_ref, template_vdi_uuids = VMHelper.create_snapshot( self._session, instance.id, vm_ref, label) - return Snapshot(self, instance, template_vm_ref, - template_vdi_uuids) + return template_vm_ref, template_vdi_uuids except self.XenAPI.Failure, exc: logging.error(_("Unable to Snapshot %(vm_ref)s: %(exc)s") % locals()) @@ -294,9 +283,11 @@ class VMOps(object): # from the snapshot creation base_copy_uuid = cow_uuid = None - with self._get_snapshot(instance) as snapshot: + template_vdi_uuids = template_vm_ref = None + try: # transfer the base copy - base_copy_uuid = snapshot.vdi_uuids[1] + template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) + base_copy_uuid = template_vdi_uuids[1] vdi_ref, vm_vdi_rec = \ VMHelper.get_vdi_for_vm_safely(self._session, vm_ref) cow_uuid = vm_vdi_rec['uuid'] @@ -304,7 +295,7 @@ class VMOps(object): params = {'host': dest, 'vdi_uuid': base_copy_uuid, 'instance_id': instance.id, - 'sr_path': VMHelper.get_sr_path(self._session), } + 'sr_path': VMHelper.get_sr_path(self._session)} task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) @@ -322,6 +313,11 @@ class VMOps(object): {'params': pickle.dumps(params)}) self._session.wait_for_task(instance.id, task) + finally: + if template_vm_ref: + self.virt._destroy(self.instance, template_vm_ref, shutdown=False, + destroy_kernel_ramdisk=False) + # TODO(mdietz): we could also consider renaming these to something # sensible so we don't need to blindly pass around dictionaries return {'base_copy': base_copy_uuid, 'cow': cow_uuid} @@ -332,9 +328,9 @@ class VMOps(object): new_cow_uuid = str(uuid.uuid4()) params = {'instance_id': instance.id, 'old_base_copy_uuid': disk_info['base_copy'], - 'old_cow_uuid': disk_info['cow'], + 'old_cow_uuid': disk_info['cow'], 'new_base_copy_uuid': new_base_copy_uuid, - 'new_cow_uuid': new_cow_uuid, + 'new_cow_uuid': new_cow_uuid, 'sr_path': VMHelper.get_sr_path(self._session), } task = self._session.async_call_plugin('migration', diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index d08754c19..aa12d432a 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -207,7 +207,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): 'transfer-encoding': 'chunked', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'queued', - 'x-image-meta-type': 'vhd', } + 'x-image-meta-type': 'vhd'} for header, value in headers.iteritems(): conn.putheader(header, value) conn.endheaders() -- cgit From 6797c5acc47fb5111ef821d6b074cb635692a9fb Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 3 Mar 2011 15:41:45 +0000 Subject: Add in multi-tenant support in openstack api. --- bin/nova-manage | 3 + nova/api/openstack/__init__.py | 25 ++++ nova/api/openstack/accounts.py | 73 +++++++++++ nova/api/openstack/auth.py | 54 ++++++-- nova/api/openstack/backup_schedules.py | 6 +- nova/api/openstack/consoles.py | 10 +- nova/api/openstack/flavors.py | 6 +- nova/api/openstack/images.py | 12 +- nova/api/openstack/servers.py | 38 +++--- nova/api/openstack/shared_ip_groups.py | 12 +- nova/api/openstack/users.py | 93 ++++++++++++++ nova/api/openstack/zones.py | 12 +- nova/auth/novarc.template | 2 +- nova/db/sqlalchemy/api.py | 3 + nova/tests/api/openstack/fakes.py | 82 +++++++++++- nova/tests/api/openstack/test_accounts.py | 123 ++++++++++++++++++ nova/tests/api/openstack/test_adminapi.py | 6 +- nova/tests/api/openstack/test_auth.py | 17 ++- nova/tests/api/openstack/test_flavors.py | 4 +- nova/tests/api/openstack/test_images.py | 6 +- nova/tests/api/openstack/test_servers.py | 51 ++++---- nova/tests/api/openstack/test_users.py | 139 +++++++++++++++++++++ nova/tests/api/openstack/test_zones.py | 12 +- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 2 +- 24 files changed, 688 insertions(+), 103 deletions(-) create mode 100644 nova/api/openstack/accounts.py create mode 100644 nova/api/openstack/users.py create mode 100644 nova/tests/api/openstack/test_accounts.py create mode 100644 nova/tests/api/openstack/test_users.py diff --git a/bin/nova-manage b/bin/nova-manage index 89332f2af..8f8f0a6a8 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -433,6 +433,8 @@ class ProjectCommands(object): "been created.\nPlease create a database by running a " "nova-api server on this host.") +AccountCommands = ProjectCommands + class FixedIpCommands(object): """Class for managing fixed ip.""" @@ -663,6 +665,7 @@ class VolumeCommands(object): CATEGORIES = [ ('user', UserCommands), + ('account', AccountCommands), ('project', ProjectCommands), ('role', RoleCommands), ('shell', ShellCommands), diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index b1b38ed2d..73d52192e 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -27,6 +27,7 @@ import webob.exc from nova import flags from nova import log as logging from nova import wsgi +from nova.api.openstack import accounts from nova.api.openstack import faults from nova.api.openstack import backup_schedules from nova.api.openstack import consoles @@ -34,6 +35,7 @@ from nova.api.openstack import flavors from nova.api.openstack import images from nova.api.openstack import servers from nova.api.openstack import shared_ip_groups +from nova.api.openstack import users from nova.api.openstack import zones @@ -71,6 +73,18 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() + accounts_controller = accounts.Controller() + mapper.connect("account", "/{id}", + controller=accounts_controller, action="show", + conditions=dict(method=["GET"])) + if FLAGS.allow_admin_api: + mapper.connect("/{id}", + controller=accounts_controller, action="update", + conditions=dict(method=["PUT"])) + mapper.connect("/{id}", + controller=accounts_controller, action="delete", + conditions=dict(method=["DELETE"])) + server_members = {'action': 'POST'} if FLAGS.allow_admin_api: LOG.debug(_("Including admin operations in API.")) @@ -84,27 +98,38 @@ class APIRouter(wsgi.Router): server_members['inject_network_info'] = 'POST' mapper.resource("zone", "zones", controller=zones.Controller(), + path_prefix="{account_id}/", + collection={'detail': 'GET'}) + + mapper.resource("user", "users", controller=users.Controller(), + path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, + path_prefix="{account_id}/", member=server_members) mapper.resource("backup_schedule", "backup_schedule", controller=backup_schedules.Controller(), + path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("console", "consoles", controller=consoles.Controller(), + path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("image", "images", controller=images.Controller(), + path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("flavor", "flavors", controller=flavors.Controller(), + path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("shared_ip_group", "shared_ip_groups", + path_prefix="{account_id}/", collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py new file mode 100644 index 000000000..264fdab99 --- /dev/null +++ b/nova/api/openstack/accounts.py @@ -0,0 +1,73 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import common + +from nova import exception +from nova import flags +from nova import log as logging +from nova import wsgi + +from nova.auth import manager + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.api.openstack') + + +def _translate_keys(account): + return dict(id=account.id, + name=account.name, + description=account.description, + manager=account.project_manager_id) + + +class Controller(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "account": ["id", "name", "description", "manager"]}}} + + def __init__(self): + self.manager = manager.AuthManager() + + def _check_admin(self, context): + """ We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here """ + if not context.is_admin: + raise exception.NotAuthorized("Not admin user.") + + def show(self, req, id): + """Return data about the given account id""" + account = self.manager.get_project(id) + return dict(account=_translate_keys(account)) + + def delete(self, req, id): + self._check_admin(req.environ['nova.context']) + self.manager.delete_project(id) + return {} + + def update(self, req, id): + """ This is really create or update. """ + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req) + description = env['account'].get('description') + manager = env['account'].get('manager') + try: + account = self.manager.get_project(id) + self.manager.modify_project(id, manager, description) + except exception.NotFound: + account = self.manager.create_project(id, manager, description) + return dict(account=_translate_keys(account)) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6011e6115..e77910fed 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -28,11 +28,13 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import log as logging from nova import manager from nova import utils from nova import wsgi from nova.api.openstack import faults +LOG = logging.getLogger('nova.api.openstack') FLAGS = flags.FLAGS @@ -50,14 +52,27 @@ class AuthMiddleware(wsgi.Middleware): def __call__(self, req): if not self.has_authentication(req): return self.authenticate(req) - user = self.get_user_by_authentication(req) + account_name = req.path_info_peek() if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - project = self.auth.get_project(FLAGS.default_project) - req.environ['nova.context'] = context.RequestContext(user, project) + if not account_name: + if self.auth.is_admin(user): + account_name = FLAGS.default_project + else: + return faults.Fault(webob.exc.HTTPUnauthorized()) + try: + account = self.auth.get_project(account_name) + except exception.NotFound: + return faults.Fault(webob.exc.HTTPUnauthorized()) + + if not self.auth.is_admin(user) and \ + not self.auth.is_project_member(user, account): + return faults.Fault(webob.exc.HTTPUnauthorized()) + + req.environ['nova.context'] = context.RequestContext(user, account) return self.application def has_authentication(self, req): @@ -70,6 +85,7 @@ class AuthMiddleware(wsgi.Middleware): # Unless the request is explicitly made against // don't # honor it path_info = req.path_info + account_name = None if len(path_info) > 1: return faults.Fault(webob.exc.HTTPUnauthorized()) @@ -79,7 +95,10 @@ class AuthMiddleware(wsgi.Middleware): except KeyError: return faults.Fault(webob.exc.HTTPUnauthorized()) - token, user = self._authorize_user(username, key, req) + if ':' in username: + account_name, username = username.rsplit(':', 1) + + token, user = self._authorize_user(username, account_name, key, req) if user and token: res = webob.Response() res.headers['X-Auth-Token'] = token.token_hash @@ -116,23 +135,44 @@ class AuthMiddleware(wsgi.Middleware): return self.auth.get_user(token.user_id) return None - def _authorize_user(self, username, key, req): + def _authorize_user(self, username, account_name, key, req): """Generates a new token and assigns it to a user. username - string + account_name - string key - string API key req - webob.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) + if account_name: + try: + account = self.auth.get_project(account_name) + except exception.NotFound: + return None, None + else: + # (dragondm) punt and try to determine account. + # this is something of a hack, but a user on 1 account is a + # common case, and is the way the current RS code works. + accounts = self.auth.get_projects(user=user) + if len(accounts) == 1: + account = accounts[0] + else: + #we can't tell what account they are logging in for. + return None, None + if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, time.time())).hexdigest() token_dict = {} token_dict['token_hash'] = token_hash token_dict['cdn_management_url'] = '' - # Same as auth url, e.g. http://foo.org:8774/baz/v1.0 - token_dict['server_management_url'] = req.url + # auth url + project (account) id, e.g. + # http://foo.org:8774/baz/v1.0/myacct/ + os_url = '%s%s%s/' % (req.url, + '' if req.url.endswith('/') else '/', + account.id) + token_dict['server_management_url'] = os_url token_dict['storage_url'] = '' token_dict['user_id'] = user.id token = self.db.auth_token_create(ctxt, token_dict) diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index 7abb5f884..a4d5939df 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -40,15 +40,15 @@ class Controller(wsgi.Controller): def __init__(self): pass - def index(self, req, server_id): + def index(self, req, server_id, **kw): """ Returns the list of backup schedules for a given instance """ return _translate_keys({}) - def create(self, req, server_id): + def create(self, req, server_id, **kw): """ No actual update method required, since the existing API allows both create and update through a POST """ return faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id): + def delete(self, req, server_id, id, **kw): """ Deletes an existing backup schedule """ return faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 9ebdbe710..85b2a4140 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -55,7 +55,7 @@ class Controller(wsgi.Controller): self.console_api = console.API() super(Controller, self).__init__() - def index(self, req, server_id): + def index(self, req, server_id, **kw): """Returns a list of consoles for this instance""" consoles = self.console_api.get_consoles( req.environ['nova.context'], @@ -63,14 +63,14 @@ class Controller(wsgi.Controller): return dict(consoles=[_translate_keys(console) for console in consoles]) - def create(self, req, server_id): + def create(self, req, server_id, **kw): """Creates a new console""" #info = self._deserialize(req.body, req) self.console_api.create_console( req.environ['nova.context'], int(server_id)) - def show(self, req, server_id, id): + def show(self, req, server_id, id, **kw): """Shows in-depth information on a specific console""" try: console = self.console_api.get_console( @@ -81,11 +81,11 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return _translate_detail_keys(console) - def update(self, req, server_id, id): + def update(self, req, server_id, id, **kw): """You can't update a console""" raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id): + def delete(self, req, server_id, id, **kw): """Deletes a console""" try: self.console_api.delete_console(req.environ['nova.context'], diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index f620d4107..79c3e1ab3 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -32,18 +32,18 @@ class Controller(wsgi.Controller): "attributes": { "flavor": ["id", "name", "ram", "disk"]}}} - def index(self, req): + def index(self, req, **kw): """Return all flavors in brief.""" return dict(flavors=[dict(id=flavor['id'], name=flavor['name']) for flavor in self.detail(req)['flavors']]) - def detail(self, req): + def detail(self, req, **kw): """Return all flavors in detail.""" items = [self.show(req, id)['flavor'] for id in self._all_ids()] items = common.limited(items, req) return dict(flavors=items) - def show(self, req, id): + def show(self, req, id, **kw): """Return data about the given flavor id.""" for name, val in instance_types.INSTANCE_TYPES.iteritems(): if val['flavorid'] == int(id): diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cf85a496f..5bc5b9978 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -115,14 +115,14 @@ class Controller(wsgi.Controller): def __init__(self): self._service = utils.import_object(FLAGS.image_service) - def index(self, req): + def index(self, req, **kw): """Return all public images in brief""" items = self._service.index(req.environ['nova.context']) items = common.limited(items, req) items = [_filter_keys(item, ('id', 'name')) for item in items] return dict(images=items) - def detail(self, req): + def detail(self, req, **kw): """Return all public images in detail""" try: items = self._service.detail(req.environ['nova.context']) @@ -136,7 +136,7 @@ class Controller(wsgi.Controller): items = [_translate_status(item) for item in items] return dict(images=items) - def show(self, req, id): + def show(self, req, id, **kw): """Return data about the given image id""" image_id = common.get_image_id_from_image_hash(self._service, req.environ['nova.context'], id) @@ -145,11 +145,11 @@ class Controller(wsgi.Controller): _convert_image_id_to_hash(image) return dict(image=image) - def delete(self, req, id): + def delete(self, req, id, **kw): # Only public images are supported for now. raise faults.Fault(exc.HTTPNotFound()) - def create(self, req): + def create(self, req, **kw): context = req.environ['nova.context'] env = self._deserialize(req.body, req) instance_id = env["image"]["serverId"] @@ -160,7 +160,7 @@ class Controller(wsgi.Controller): return dict(image=image_meta) - def update(self, req, id): + def update(self, req, id, **kw): # Users may not modify public images, and that's all that # we support for now. raise faults.Fault(exc.HTTPNotFound()) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 69273ad7b..426de92be 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -105,11 +105,11 @@ class Controller(wsgi.Controller): self._image_service = utils.import_object(FLAGS.image_service) super(Controller, self).__init__() - def index(self, req): + def index(self, req, **kw): """ Returns a list of server names and ids for a given user """ return self._items(req, entity_maker=_translate_keys) - def detail(self, req): + def detail(self, req, **kw): """ Returns a list of server details for a given user """ return self._items(req, entity_maker=_translate_detail_keys) @@ -123,7 +123,7 @@ class Controller(wsgi.Controller): res = [entity_maker(inst)['server'] for inst in limited_list] return dict(servers=res) - def show(self, req, id): + def show(self, req, id, **kw): """ Returns server details by server id """ try: instance = self.compute_api.get(req.environ['nova.context'], id) @@ -131,7 +131,7 @@ class Controller(wsgi.Controller): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) - def delete(self, req, id): + def delete(self, req, id, **kw): """ Destroys a server """ try: self.compute_api.delete(req.environ['nova.context'], id) @@ -139,7 +139,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def create(self, req): + def create(self, req, **kw): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) if not env: @@ -180,7 +180,7 @@ class Controller(wsgi.Controller): onset_files=env.get('onset_files', [])) return _translate_keys(instances[0]) - def update(self, req, id): + def update(self, req, id, **kw): """ Updates the server name or password """ inst_dict = self._deserialize(req.body, req) if not inst_dict: @@ -202,7 +202,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() - def action(self, req, id): + def action(self, req, id, **kw): """ Multi-purpose method used to reboot, rebuild, and resize a server """ input_dict = self._deserialize(req.body, req) @@ -219,7 +219,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def lock(self, req, id): + def lock(self, req, id, **kw): """ lock the instance with id admin only operation @@ -234,7 +234,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unlock(self, req, id): + def unlock(self, req, id, **kw): """ unlock the instance with id admin only operation @@ -249,7 +249,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_lock(self, req, id): + def get_lock(self, req, id, **kw): """ return the boolean state of (instance with id)'s lock @@ -263,7 +263,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def reset_network(self, req, id): + def reset_network(self, req, id, **kw): """ Reset networking on an instance (admin only). @@ -277,7 +277,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def inject_network_info(self, req, id): + def inject_network_info(self, req, id, **kw): """ Inject network info for an instance (admin only). @@ -291,7 +291,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def pause(self, req, id): + def pause(self, req, id, **kw): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: @@ -302,7 +302,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unpause(self, req, id): + def unpause(self, req, id, **kw): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] try: @@ -313,7 +313,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def suspend(self, req, id): + def suspend(self, req, id, **kw): """permit admins to suspend the server""" context = req.environ['nova.context'] try: @@ -324,7 +324,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def resume(self, req, id): + def resume(self, req, id, **kw): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] try: @@ -335,7 +335,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_ajax_console(self, req, id): + def get_ajax_console(self, req, id, **kw): """ Returns a url to an instance's ajaxterm console. """ try: self.compute_api.get_ajax_console(req.environ['nova.context'], @@ -344,12 +344,12 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def diagnostics(self, req, id): + def diagnostics(self, req, id, **kw): """Permit Admins to retrieve server diagnostics.""" ctxt = req.environ["nova.context"] return self.compute_api.get_diagnostics(ctxt, id) - def actions(self, req, id): + def actions(self, req, id, **kw): """Permit Admins to retrieve server actions.""" ctxt = req.environ["nova.context"] items = self.compute_api.get_actions(ctxt, id) diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index 5d78f9377..e3c917749 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -40,26 +40,26 @@ class Controller(wsgi.Controller): 'attributes': { 'sharedIpGroup': []}}} - def index(self, req): + def index(self, req, **kw): """ Returns a list of Shared IP Groups for the user """ return dict(sharedIpGroups=[]) - def show(self, req, id): + def show(self, req, id, **kw): """ Shows in-depth information on a specific Shared IP Group """ return _translate_keys({}) - def update(self, req, id): + def update(self, req, id, **kw): """ You can't update a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, id): + def delete(self, req, id, **kw): """ Deletes a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def detail(self, req): + def detail(self, req, **kw): """ Returns a complete list of Shared IP Groups """ return _translate_detail_keys({}) - def create(self, req): + def create(self, req, **kw): """ Creates a new Shared IP group """ raise faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py new file mode 100644 index 000000000..c0b7544f9 --- /dev/null +++ b/nova/api/openstack/users.py @@ -0,0 +1,93 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import common + +from nova import exception +from nova import flags +from nova import log as logging +from nova import wsgi + +from nova.auth import manager + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.api.openstack') + + +def _translate_keys(user): + return dict(id=user.id, + name=user.name, + access=user.access, + secret=user.secret, + admin=user.admin) + + +class Controller(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "user": ["id", "name", "access", "secret", "admin"]}}} + + def __init__(self): + self.manager = manager.AuthManager() + + def _check_admin(self, context): + """ We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here """ + if not context.is_admin: + raise exception.NotAuthorized("Not admin user") + + def index(self, req, **kw): + """Return all users in brief""" + users = self.manager.get_users() + users = common.limited(users, req) + users = [_translate_keys(user) for user in users] + return dict(users=users) + + def detail(self, req, **kw): + """Return all users in detail""" + return self.index(req) + + def show(self, req, id, **kw): + """Return data about the given user id""" + user = self.manager.get_user(id) + return dict(user=_translate_keys(user)) + + def delete(self, req, id, **kw): + self._check_admin(req.environ['nova.context']) + self.manager.delete_user(id) + return {} + + def create(self, req, **kw): + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req) + is_admin = env['user'].get('admin') in ('T', 'True', True) + name = env['user'].get('name') + access = env['user'].get('access') + secret = env['user'].get('secret') + user = self.manager.create_user(name, access, secret, is_admin) + return dict(user=_translate_keys(user)) + + def update(self, req, id, **kw): + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req) + is_admin = env['user'].get('admin') + if is_admin is not None: + is_admin = is_admin in ('T', 'True', True) + access = env['user'].get('access') + secret = env['user'].get('secret') + self.manager.modify_user(id, access, secret, is_admin) + return dict(user=_translate_keys(self.manager.get_user(id))) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d5206da20..30bf2b67b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -43,35 +43,35 @@ class Controller(wsgi.Controller): "attributes": { "zone": ["id", "api_url"]}}} - def index(self, req): + def index(self, req, **kw): """Return all zones in brief""" items = db.zone_get_all(req.environ['nova.context']) items = common.limited(items, req) items = [_scrub_zone(item) for item in items] return dict(zones=items) - def detail(self, req): + def detail(self, req, **kw): """Return all zones in detail""" return self.index(req) - def show(self, req, id): + def show(self, req, id, **kw): """Return data about the given zone id""" zone_id = int(id) zone = db.zone_get(req.environ['nova.context'], zone_id) return dict(zone=_scrub_zone(zone)) - def delete(self, req, id): + def delete(self, req, id, **kw): zone_id = int(id) db.zone_delete(req.environ['nova.context'], zone_id) return {} - def create(self, req): + def create(self, req, **kw): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) - def update(self, req, id): + def update(self, req, id, **kw): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone_id = int(id) diff --git a/nova/auth/novarc.template b/nova/auth/novarc.template index cda2ecc28..1c917ad44 100644 --- a/nova/auth/novarc.template +++ b/nova/auth/novarc.template @@ -11,5 +11,5 @@ export EUCALYPTUS_CERT=${NOVA_CERT} # euca-bundle-image seems to require this se alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_PRIVATE_KEY} --user 42 --ec2cert ${NOVA_CERT}" alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}" export NOVA_API_KEY="%(access)s" -export NOVA_USERNAME="%(user)s" +export NOVA_USERNAME="%(project)s:%(user)s" export NOVA_URL="%(os)s" diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 6df2a8843..e311f310a 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1861,8 +1861,11 @@ def project_get_by_user(context, user_id): session = get_session() user = session.query(models.User).\ filter_by(deleted=can_read_deleted(context)).\ + filter_by(id=user_id).\ options(joinedload_all('projects')).\ first() + if not user: + raise exception.NotFound(_('Invalid user_id %s') % user_id) return user.projects diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..03b26e29a 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -26,7 +26,6 @@ from paste import urlmap from glance import client as glance_client -from nova import auth from nova import context from nova import exception as exc from nova import flags @@ -35,6 +34,7 @@ import nova.api.openstack.auth from nova.api import openstack from nova.api.openstack import auth from nova.api.openstack import ratelimiting +from nova.auth.manager import User, Project from nova.image import glance from nova.image import local from nova.image import service @@ -227,19 +227,97 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} + projects = {} + + @classmethod + def clear_fakes(cls): + cls.auth_data = {} + cls.projects = {} + + @classmethod + def reset_fake_data(cls): + cls.auth_data = dict(acc1=User('guy1', 'guy1', 'acc1', + 'fortytwo!', False)) + cls.projects = dict(testacct=Project('testacct', + 'testacct', + 'guy1', + 'test', + [])) def add_user(self, key, user): FakeAuthManager.auth_data[key] = user + def get_users(self): + return FakeAuthManager.auth_data.values() + def get_user(self, uid): for k, v in FakeAuthManager.auth_data.iteritems(): if v.id == uid: return v return None - def get_project(self, pid): + def delete_user(self, uid): + for k, v in FakeAuthManager.auth_data.items(): + if v.id == uid: + del FakeAuthManager.auth_data[k] return None + def create_user(self, name, access=None, secret=None, admin=False): + u = User(name, name, access, secret, admin) + FakeAuthManager.auth_data[access] = u + return u + + def modify_user(self, user_id, access=None, secret=None, admin=None): + user = None + for k, v in FakeAuthManager.auth_data.iteritems(): + if v.id == user_id: + user = v + if user: + user.access = access + user.secret = secret + if admin is not None: + user.admin = admin + + def is_admin(self, user): + return user.admin + + def is_project_member(self, user, project): + return ((user.id in project.member_ids) or + (user.id == project.project_manager_id)) + + def create_project(self, name, manager_user, description=None, + member_users=None): + member_ids = [User.safe_id(m) for m in member_users] \ + if member_users else [] + p = Project(name, name, User.safe_id(manager_user), + description, member_ids) + FakeAuthManager.projects[name] = p + return p + + def delete_project(self, pid): + if pid in FakeAuthManager.projects: + del FakeAuthManager.projects[pid] + + def modify_project(self, project, manager_user=None, description=None): + p = FakeAuthManager.projects.get(project) + p.project_manager_id = User.safe_id(manager_user) + p.description = description + + def get_project(self, pid): + p = FakeAuthManager.projects.get(pid) + if p: + return p + else: + raise exc.NotFound + + def get_projects(self, user=None): + if not user: + return FakeAuthManager.projects.values() + else: + return [p for p in FakeAuthManager.projects.values() + if (user.id in p.member_ids) or + (user.id == p.project_manager_id)] + def get_user_from_access_key(self, key): return FakeAuthManager.auth_data.get(key, None) diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py new file mode 100644 index 000000000..b2e89824a --- /dev/null +++ b/nova/tests/api/openstack/test_accounts.py @@ -0,0 +1,123 @@ +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import stubout +import webob +import json + +import nova.api +import nova.api.openstack.auth +from nova import context +from nova import flags +from nova import test +from nova.auth.manager import User +from nova.tests.api.openstack import fakes + + +FLAGS = flags.FLAGS +FLAGS.verbose = True + + +def fake_init(self): + self.manager = fakes.FakeAuthManager() + + +def fake_admin_check(self, req): + return True + + +class AccountsTest(test.TestCase): + def setUp(self): + super(AccountsTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.openstack.accounts.Controller, '__init__', + fake_init) + self.stubs.Set(nova.api.openstack.accounts.Controller, '_check_admin', + fake_admin_check) + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.projects = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + + self.allow_admin = FLAGS.allow_admin_api + FLAGS.allow_admin_api = True + fakemgr = fakes.FakeAuthManager() + joeuser = User('guy1', 'guy1', 'acc1', 'fortytwo!', False) + superuser = User('guy2', 'guy2', 'acc2', 'swordfish', True) + fakemgr.add_user(joeuser.access, joeuser) + fakemgr.add_user(superuser.access, superuser) + fakemgr.create_project('test1', joeuser) + fakemgr.create_project('test2', superuser) + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(AccountsTest, self).tearDown() + + def test_get_account(self): + req = webob.Request.blank('/v1.0/test1') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['account']['id'], 'test1') + self.assertEqual(res_dict['account']['name'], 'test1') + self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertEqual(res.status_int, 200) + + def test_account_delete(self): + req = webob.Request.blank('/v1.0/test1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('test1' not in fakes.FakeAuthManager.projects) + self.assertEqual(res.status_int, 200) + + def test_account_create(self): + body = dict(account=dict(description='test account', + manager='guy1')) + req = webob.Request.blank('/v1.0/newacct') + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['account']['id'], 'newacct') + self.assertEqual(res_dict['account']['name'], 'newacct') + self.assertEqual(res_dict['account']['description'], 'test account') + self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertTrue('newacct' in + fakes.FakeAuthManager.projects) + self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 3) + + def test_account_update(self): + body = dict(account=dict(description='test account', + manager='guy2')) + req = webob.Request.blank('/v1.0/test1') + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['account']['id'], 'test1') + self.assertEqual(res_dict['account']['name'], 'test1') + self.assertEqual(res_dict['account']['description'], 'test account') + self.assertEqual(res_dict['account']['manager'], 'guy2') + self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 2) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index dfce1b127..7cb9e8450 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -35,7 +35,7 @@ class AdminAPITest(test.TestCase): def setUp(self): super(AdminAPITest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -50,7 +50,7 @@ class AdminAPITest(test.TestCase): def test_admin_enabled(self): FLAGS.allow_admin_api = True # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/flavors') + req = webob.Request.blank('/v1.0/testacct/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are available. @@ -58,7 +58,7 @@ class AdminAPITest(test.TestCase): def test_admin_disabled(self): FLAGS.allow_admin_api = False # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/flavors') + req = webob.Request.blank('/v1.0/testacct/flavors') res = req.get_response(fakes.wsgi_app()) # TODO: Confirm admin operations are unavailable. self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index ff8d42a14..8268a6fb9 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,7 +51,9 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -65,7 +67,9 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -74,7 +78,7 @@ class Test(test.TestCase): self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], - "http://foo/v1.0/") + "http://foo/v1.0/test/") self.assertEqual(result.headers['X-CDN-Management-Url'], "") self.assertEqual(result.headers['X-Storage-Url'], "") @@ -82,7 +86,7 @@ class Test(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/fake') + req = webob.Request.blank('/v1.0/test/fake') req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') @@ -176,6 +180,9 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') @@ -187,7 +194,7 @@ class TestLimiter(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/fake') + req = webob.Request.blank('/v1.0/test/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 761265965..370dc007c 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -28,7 +28,7 @@ class FlavorsTest(test.TestCase): def setUp(self): super(FlavorsTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -39,7 +39,7 @@ class FlavorsTest(test.TestCase): super(FlavorsTest, self).tearDown() def test_get_flavor_list(self): - req = webob.Request.blank('/v1.0/flavors') + req = webob.Request.blank('/v1.0/testacct/flavors') res = req.get_response(fakes.wsgi_app()) def test_get_flavor_by_id(self): diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index e232bc3d5..819ca001e 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -202,7 +202,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): self.orig_image_service = FLAGS.image_service FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -216,7 +216,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): super(ImageControllerWithGlanceServiceTest, self).tearDown() def test_get_image_index(self): - req = webob.Request.blank('/v1.0/images') + req = webob.Request.blank('/v1.0/testacct/images') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -228,7 +228,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): "image %s not in fixture index!" % str(image)) def test_get_image_details(self): - req = webob.Request.blank('/v1.0/images/detail') + req = webob.Request.blank('/v1.0/testacct/images/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..d592e06b0 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -118,7 +118,7 @@ class ServersTest(test.TestCase): def setUp(self): super(ServersTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -150,7 +150,7 @@ class ServersTest(test.TestCase): super(ServersTest, self).tearDown() def test_get_server_by_id(self): - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -161,7 +161,7 @@ class ServersTest(test.TestCase): public = ["1.2.3.4"] new_return_server = return_server_with_addresses(private, public) self.stubs.Set(nova.db.api, 'instance_get', new_return_server) - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -173,7 +173,7 @@ class ServersTest(test.TestCase): self.assertEqual(addresses["private"][0], private) def test_get_server_list(self): - req = webob.Request.blank('/v1.0/servers') + req = webob.Request.blank('/v1.0/testacct/servers') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -224,7 +224,7 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={'hello': 'world', 'open': 'stack'}, personality={})) - req = webob.Request.blank('/v1.0/servers') + req = webob.Request.blank('/v1.0/testacct/servers') req.method = 'POST' req.body = json.dumps(body) @@ -233,7 +233,7 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_update_no_body(self): - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'PUT' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 422) @@ -251,7 +251,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) @@ -267,30 +267,30 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) def test_create_backup_schedules(self): - req = webob.Request.blank('/v1.0/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') req.method = 'POST' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): - req = webob.Request.blank('/v1.0/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): - req = webob.Request.blank('/v1.0/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): - req = webob.Request.blank('/v1.0/servers/detail') + req = webob.Request.blank('/v1.0/testacct/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -321,7 +321,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers_with_host) - req = webob.Request.blank('/v1.0/servers/detail') + req = webob.Request.blank('/v1.0/testacct/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -341,7 +341,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/pause') + req = webob.Request.blank('/v1.0/testacct/servers/1/pause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -353,7 +353,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/unpause') + req = webob.Request.blank('/v1.0/testacct/servers/1/unpause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -365,7 +365,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/suspend') + req = webob.Request.blank('/v1.0/testacct/servers/1/suspend') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -377,7 +377,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/resume') + req = webob.Request.blank('/v1.0/testacct/servers/1/resume') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -389,7 +389,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/reset_network') + req = webob.Request.blank('/v1.0/testacct/servers/1/reset_network') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -401,7 +401,8 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/inject_network_info') + req = webob.Request.blank( + '/v1.0/testacct/servers/1/inject_network_info') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -409,13 +410,13 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 202) def test_server_diagnostics(self): - req = webob.Request.blank("/v1.0/servers/1/diagnostics") + req = webob.Request.blank("/v1.0/testacct/servers/1/diagnostics") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_actions(self): - req = webob.Request.blank("/v1.0/servers/1/actions") + req = webob.Request.blank("/v1.0/testacct/servers/1/actions") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) @@ -424,7 +425,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') + req = webob.Request.blank('/v1.0/testacct/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -434,7 +435,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') + req = webob.Request.blank('/v1.0/testacct/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -444,14 +445,14 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') + req = webob.Request.blank('/v1.0/testacct/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) def test_delete_server_instance(self): - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'DELETE' self.server_delete_called = False diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py new file mode 100644 index 000000000..bd32254cd --- /dev/null +++ b/nova/tests/api/openstack/test_users.py @@ -0,0 +1,139 @@ +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import stubout +import webob +import json + +import nova.api +import nova.api.openstack.auth +from nova import context +from nova import flags +from nova import test +from nova.auth.manager import User, Project +from nova.tests.api.openstack import fakes + + +FLAGS = flags.FLAGS +FLAGS.verbose = True + + +def fake_init(self): + self.manager = fakes.FakeAuthManager() + + +def fake_admin_check(self, req): + return True + + +class UsersTest(test.TestCase): + def setUp(self): + super(UsersTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.openstack.users.Controller, '__init__', + fake_init) + self.stubs.Set(nova.api.openstack.users.Controller, '_check_admin', + fake_admin_check) + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.projects = dict(testacct=Project('testacct', + 'testacct', + 'guy1', + 'test', + [])) + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + + self.allow_admin = FLAGS.allow_admin_api + FLAGS.allow_admin_api = True + fakemgr = fakes.FakeAuthManager() + fakemgr.add_user('acc1', User('guy1', 'guy1', 'acc1', + 'fortytwo!', False)) + fakemgr.add_user('acc2', User('guy2', 'guy2', 'acc2', + 'swordfish', True)) + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(UsersTest, self).tearDown() + + def test_get_user_list(self): + req = webob.Request.blank('/v1.0/testacct/users') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(len(res_dict['users']), 2) + + def test_get_user_by_id(self): + req = webob.Request.blank('/v1.0/testacct/users/guy2') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['name'], 'guy2') + self.assertEqual(res_dict['user']['secret'], 'swordfish') + self.assertEqual(res_dict['user']['admin'], True) + self.assertEqual(res.status_int, 200) + + def test_user_delete(self): + req = webob.Request.blank('/v1.0/testacct/users/guy1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('guy1' not in [u.id for u in + fakes.FakeAuthManager.auth_data.values()]) + self.assertEqual(res.status_int, 200) + + def test_user_create(self): + body = dict(user=dict(name='test_guy', + access='acc3', + secret='invasionIsInNormandy', + admin=True)) + req = webob.Request.blank('/v1.0/testacct/users') + req.method = 'POST' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['user']['id'], 'test_guy') + self.assertEqual(res_dict['user']['name'], 'test_guy') + self.assertEqual(res_dict['user']['access'], 'acc3') + self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['admin'], True) + self.assertTrue('test_guy' in [u.id for u in + fakes.FakeAuthManager.auth_data.values()]) + self.assertEqual(len(fakes.FakeAuthManager.auth_data.values()), 3) + + def test_user_update(self): + body = dict(user=dict(name='guy2', + access='acc2', + secret='invasionIsInNormandy')) + req = webob.Request.blank('/v1.0/testacct/users/guy2') + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['name'], 'guy2') + self.assertEqual(res_dict['user']['access'], 'acc2') + self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['admin'], True) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 555b206b9..51f13af48 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -64,7 +64,7 @@ class ZonesTest(test.TestCase): def setUp(self): super(ZonesTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -85,7 +85,7 @@ class ZonesTest(test.TestCase): super(ZonesTest, self).tearDown() def test_get_zone_list(self): - req = webob.Request.blank('/v1.0/zones') + req = webob.Request.blank('/v1.0/testacct/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -93,7 +93,7 @@ class ZonesTest(test.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): - req = webob.Request.blank('/v1.0/zones/1') + req = webob.Request.blank('/v1.0/testacct/zones/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -103,7 +103,7 @@ class ZonesTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_zone_delete(self): - req = webob.Request.blank('/v1.0/zones/1') + req = webob.Request.blank('/v1.0/testacct/zones/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -111,7 +111,7 @@ class ZonesTest(test.TestCase): def test_zone_create(self): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) - req = webob.Request.blank('/v1.0/zones') + req = webob.Request.blank('/v1.0/testacct/zones') req.method = 'POST' req.body = json.dumps(body) @@ -125,7 +125,7 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) - req = webob.Request.blank('/v1.0/zones/1') + req = webob.Request.blank('/v1.0/testacct/zones/1') req.method = 'PUT' req.body = json.dumps(body) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 7531af4ec..a45d32cb2 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -207,7 +207,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): 'transfer-encoding': 'chunked', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'queued', - 'x-image-meta-type': 'vhd' + 'x-image-meta-type': 'vhd', } for header, value in headers.iteritems(): conn.putheader(header, value) -- cgit From a62e603e8b1cedd89ca0c71f1cdc928d19c68a4d Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 11:04:33 -0500 Subject: adding wsgi.Request class to add custom best_match; adding new class to wsgify decorators; replacing all references to webob.Request in non-test code to wsgi.Request --- nova/api/direct.py | 4 ++-- nova/api/ec2/__init__.py | 14 +++++------ nova/api/ec2/metadatarequesthandler.py | 2 +- nova/api/openstack/__init__.py | 4 ++-- nova/api/openstack/auth.py | 4 ++-- nova/api/openstack/common.py | 2 +- nova/api/openstack/faults.py | 2 +- nova/api/openstack/ratelimiting/__init__.py | 4 ++-- nova/wsgi.py | 37 ++++++++++++++++++++++------- 9 files changed, 47 insertions(+), 26 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index 208b6d086..cd237f649 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -187,7 +187,7 @@ class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): self.service_handle = service_handle - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] action = arg_dict['action'] @@ -218,7 +218,7 @@ class Proxy(object): self.prefix = prefix def __do_request(self, path, context, **kwargs): - req = webob.Request.blank(path) + req = wsgi.Request.blank(path) req.method = 'POST' req.body = urllib.urlencode({'json': utils.dumps(kwargs)}) req.environ['openstack.context'] = context diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5adc2c075..58b1ecf03 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -53,7 +53,7 @@ flags.DEFINE_list('lockout_memcached_servers', None, class RequestLogging(wsgi.Middleware): """Access-Log akin logging for all EC2 API requests.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): start = utils.utcnow() rv = req.get_response(self.application) @@ -112,7 +112,7 @@ class Lockout(wsgi.Middleware): debug=0) super(Lockout, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): access_key = str(req.params['AWSAccessKeyId']) failures_key = "authfailures-%s" % access_key @@ -141,7 +141,7 @@ class Authenticate(wsgi.Middleware): """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): # Read request signature and access id. try: @@ -190,7 +190,7 @@ class Requestify(wsgi.Middleware): super(Requestify, self).__init__(app) self.controller = utils.import_class(controller)() - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Version', 'Timestamp'] @@ -269,7 +269,7 @@ class Authorizer(wsgi.Middleware): }, } - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] controller = req.environ['ec2.request'].controller.__class__.__name__ @@ -303,7 +303,7 @@ class Executor(wsgi.Application): response, or a 400 upon failure. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] api_request = req.environ['ec2.request'] @@ -365,7 +365,7 @@ class Executor(wsgi.Application): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all EC2 versions.""" # available api versions diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 6fb441656..28f99b0ef 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -65,7 +65,7 @@ class MetadataRequestHandler(wsgi.Application): data = data[item] return data - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): cc = cloud.CloudController() remote_address = req.remote_addr diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 274330e3b..b5439788d 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -47,7 +47,7 @@ flags.DEFINE_bool('allow_admin_api', class FaultWrapper(wsgi.Middleware): """Calls down the middleware stack, making exceptions into faults.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): try: return req.get_response(self.application) @@ -115,7 +115,7 @@ class APIRouter(wsgi.Router): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all OpenStack API versions.""" response = { diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6011e6115..de8905f46 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -46,7 +46,7 @@ class AuthMiddleware(wsgi.Middleware): self.auth = auth.manager.AuthManager() super(AuthMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): if not self.has_authentication(req): return self.authenticate(req) @@ -121,7 +121,7 @@ class AuthMiddleware(wsgi.Middleware): username - string key - string API key - req - webob.Request object + req - wsgi.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 9f85c5c8a..49b970d75 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -25,7 +25,7 @@ def limited(items, request, max_limit=1000): Return a slice of items according to requested offset and limit. @param items: A sliceable entity - @param request: `webob.Request` possibly containing 'offset' and 'limit' + @param request: `wsgi.Request` possibly containing 'offset' and 'limit' GET variables. 'offset' is where to start in the list, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 224a7ef0b..c70b00fa3 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -42,7 +42,7 @@ class Fault(webob.exc.HTTPException): """Create a Fault for the given webob.exc.exception.""" self.wrapped_exc = exception - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Generate a WSGI response based on the exception passed to ctor.""" # Replace the body with fault details. diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index cbb4b897e..88ffc3246 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -57,7 +57,7 @@ class RateLimitingMiddleware(wsgi.Middleware): self.limiter = WSGIAppProxy(service_host) super(RateLimitingMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Rate limit the request. @@ -183,7 +183,7 @@ class WSGIApp(object): """Create the WSGI application using the given Limiter instance.""" self.limiter = limiter - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): parts = req.path_info.split('/') # format: /limiter// diff --git a/nova/wsgi.py b/nova/wsgi.py index 1eb66d067..67216d540 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -82,6 +82,27 @@ class Server(object): log=WritableLogger(logger)) +class Request(webob.Request): + + def best_match(self): + """ + Determine the most acceptable content-type based on the + query extension then the Accept header + """ + + parts = self.path.rsplit(".", 1) + + if len(parts) > 1: + format = parts[1] + if format in ["json", "xml"]: + return "application/{0}".format(parts[1]) + + ctypes = ["application/json", "application/xml"] + bm = self.accept.best_match(ctypes) + + return bm or "application/json" + + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -113,7 +134,7 @@ class Application(object): def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: @@ -199,7 +220,7 @@ class Middleware(Application): """Do whatever you'd like to the response.""" return response - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): response = self.process_request(req) if response: @@ -212,7 +233,7 @@ class Debug(Middleware): """Helper class that can be inserted into any WSGI application chain to get information about the request and response.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): print ("*" * 40) + " REQUEST ENVIRON" for key, value in req.environ.items(): @@ -276,7 +297,7 @@ class Router(object): self._router = routes.middleware.RoutesMiddleware(self._dispatch, self.map) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Route the incoming request to a controller based on self.map. @@ -285,7 +306,7 @@ class Router(object): return self._router @staticmethod - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def _dispatch(req): """ Called by self._router after matching the incoming request to a route @@ -304,11 +325,11 @@ class Controller(object): WSGI app that reads routing information supplied by RoutesMiddleware and calls the requested action method upon itself. All action methods must, in addition to their normal parameters, accept a 'req' argument - which is the incoming webob.Request. They raise a webob.exc exception, + which is the incoming wsgi.Request. They raise a webob.exc exception, or return a dict which will be serialized by requested content type. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Call the method specified in req.environ by RoutesMiddleware. @@ -358,7 +379,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = webob.Request.blank('', environ) + req = wsgi.Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json -- cgit From 6d075754bdd4090342bf4f79c726a52923c311a8 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 12:45:34 -0500 Subject: adding wsgi.Controller and wsgi.Request testing; fixing format keyword argument exception --- nova/tests/api/test_wsgi.py | 120 +++++++++++++++++++++++++++++++++++++------- nova/wsgi.py | 4 +- 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 2c7852214..cf2d0e297 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -21,11 +21,13 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ +import json from nova import test import routes import webob +from nova import exception from nova import wsgi @@ -66,30 +68,112 @@ class Test(test.TestCase): result = webob.Request.blank('/bad').get_response(Router()) self.assertNotEqual(result.body, "Router result") - def test_controller(self): - class Controller(wsgi.Controller): - """Test controller to call from router.""" - test = self +class ControllerTest(test.TestCase): + + class TestRouter(wsgi.Router): + + class TestController(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "test": ["id"]}}} def show(self, req, id): # pylint: disable-msg=W0622,C0103 - """Default action called for requests with an ID.""" - self.test.assertEqual(req.path_info, '/tests/123') - self.test.assertEqual(id, '123') - return id + return {"test": {"id": id}} + + def __init__(self): + mapper = routes.Mapper() + mapper.resource("test", "tests", controller=self.TestController()) + wsgi.Router.__init__(self, mapper) + + def test_show(self): + request = wsgi.Request.blank('/tests/123') + result = request.get_response(self.TestRouter()) + self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) + + def test_content_type_from_accept_xml(self): + request = webob.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_content_type_from_accept_json(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_content_type_from_query_extension_xml(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_content_type_from_query_extension_json(self): + request = wsgi.Request.blank('/tests/123.json') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_content_type_default_when_unsupported(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.status_int, 200) + self.assertEqual(result.headers["Content-Type"], "application/json") + + +class RequestTest(test.TestCase): + + def test_content_type_from_accept_xml(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.best_match() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml, application/json" + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = \ + "application/json; q=0.3, application/xml; q=0.9" + result = request.best_match() + self.assertEqual(result, "application/xml") + + def test_content_type_from_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.best_match() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123.json') + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123.invalid') + result = request.best_match() + self.assertEqual(result, "application/json") + + def test_content_type_accept_and_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + request.headers["Accept"] = "application/json" + result = request.best_match() + self.assertEqual(result, "application/xml") + + def test_content_type_accept_default(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.best_match() + self.assertEqual(result, "application/json") - class Router(wsgi.Router): - """Test router.""" - def __init__(self): - mapper = routes.Mapper() - mapper.resource("test", "tests", controller=Controller()) - super(Router, self).__init__(mapper) - result = webob.Request.blank('/tests/123').get_response(Router()) - self.assertEqual(result.body, "123") - result = webob.Request.blank('/test/123').get_response(Router()) - self.assertNotEqual(result.body, "123") class SerializerTest(test.TestCase): diff --git a/nova/wsgi.py b/nova/wsgi.py index 67216d540..4577439cb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -339,6 +339,8 @@ class Controller(object): method = getattr(self, action) del arg_dict['controller'] del arg_dict['action'] + if 'format' in arg_dict: + del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) if type(result) is dict: @@ -379,7 +381,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = wsgi.Request.blank('', environ) + req = Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json -- cgit From 0e1a458166ad1e89ca0755d88b8efec39855ee5c Mon Sep 17 00:00:00 2001 From: Cerberus Date: Thu, 3 Mar 2011 13:18:37 -0600 Subject: Renaming my migration yet again --- .../versions/007_add_instance_migrations.py | 61 ---------------------- .../versions/009_add_instance_migrations.py | 61 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 61 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/009_add_instance_migrations.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py deleted file mode 100644 index 4fda525f1..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_instance_migrations.py +++ /dev/null @@ -1,61 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License.from sqlalchemy import * - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -# Just for the ForeignKey and column creation to succeed, these are not the -# actual definitions of instances or services. -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -# -# New Tables -# - -migrations = Table('migrations', meta, - Column('created_at', DateTime(timezone=False)), - Column('updated_at', DateTime(timezone=False)), - Column('deleted_at', DateTime(timezone=False)), - Column('deleted', Boolean(create_constraint=True, name=None)), - Column('id', Integer(), primary_key=True, nullable=False), - Column('source_compute', String(255)), - Column('dest_compute', String(255)), - Column('dest_host', String(255)), - Column('instance_id', Integer, ForeignKey('instances.id'), - nullable=True), - Column('status', String(255)), - ) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - for table in (migrations, ): - try: - table.create() - except Exception: - logging.info(repr(table)) - logging.exception('Exception while creating table') - raise diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_instance_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_instance_migrations.py new file mode 100644 index 000000000..4fda525f1 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_instance_migrations.py @@ -0,0 +1,61 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.from sqlalchemy import * + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +migrations = Table('migrations', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('source_compute', String(255)), + Column('dest_compute', String(255)), + Column('dest_host', String(255)), + Column('instance_id', Integer, ForeignKey('instances.id'), + nullable=True), + Column('status', String(255)), + ) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (migrations, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise -- cgit From 848aced747a60c47d76efcb2147041339df4a628 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 17:21:21 -0500 Subject: Refactor wsgi.Serializer away from handling Requests directly; now require Content-Type in all requests; fix tests according to new code --- nova/api/direct.py | 2 +- nova/api/openstack/__init__.py | 4 +- nova/api/openstack/consoles.py | 2 +- nova/api/openstack/faults.py | 5 +- nova/api/openstack/images.py | 2 +- nova/api/openstack/servers.py | 9 ++- nova/api/openstack/zones.py | 4 +- nova/exception.py | 4 ++ nova/tests/api/openstack/test_servers.py | 1 + nova/tests/api/openstack/test_zones.py | 15 +++-- nova/tests/api/test_wsgi.py | 95 +++++++++++++++++++------------ nova/tests/test_direct.py | 3 + nova/wsgi.py | 96 ++++++++++++++++++++------------ 13 files changed, 152 insertions(+), 90 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index cd237f649..1d699f947 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req) + return self._serialize(result, req.best_match()) else: return result diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index b5439788d..6e1a2a06c 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -124,4 +124,6 @@ class Versions(wsgi.Application): metadata = { "application/xml": { "attributes": dict(version=["status", "id"])}} - return wsgi.Serializer(req.environ, metadata).to_content_type(response) + + content_type = req.best_match() + return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 9ebdbe710..8c291c2eb 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -65,7 +65,7 @@ class Controller(wsgi.Controller): def create(self, req, server_id): """Creates a new console""" - #info = self._deserialize(req.body, req) + #info = self._deserialize(req.body, req.get_content_type()) self.console_api.create_console( req.environ['nova.context'], int(server_id)) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index c70b00fa3..075fdb997 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -57,6 +57,7 @@ class Fault(webob.exc.HTTPException): fault_data[fault_name]['retryAfter'] = retry # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} - serializer = wsgi.Serializer(req.environ, metadata) - self.wrapped_exc.body = serializer.to_content_type(fault_data) + serializer = wsgi.Serializer(metadata) + content_type = req.best_match() + self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cf85a496f..98f0dd96b 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -151,7 +151,7 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) instance_id = env["image"]["serverId"] name = env["image"]["name"] diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 08b95b46a..24d2826af 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -141,7 +141,7 @@ class Controller(wsgi.Controller): def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -182,7 +182,10 @@ class Controller(wsgi.Controller): def update(self, req, id): """ Updates the server name or password """ - inst_dict = self._deserialize(req.body, req) + if len(req.body) == 0: + raise exc.HTTPUnprocessableEntity() + + inst_dict = self._deserialize(req.body, req.get_content_type()) if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -205,7 +208,7 @@ class Controller(wsgi.Controller): def action(self, req, id): """ Multi-purpose method used to reboot, rebuild, and resize a server """ - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) #TODO(sandy): rebuild/resize not supported. try: reboot_type = input_dict['reboot']['type'] diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d5206da20..cf6cd789f 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -67,13 +67,13 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) zone = db.zone_update(context, zone_id, env["zone"]) return dict(zone=_scrub_zone(zone)) diff --git a/nova/exception.py b/nova/exception.py index 7d65bd6a5..93c5fe3d7 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -88,6 +88,10 @@ class InvalidInputException(Error): pass +class InvalidContentType(Error): + pass + + class TimeoutException(Error): pass diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..fae08d0be 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -227,6 +227,7 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 555b206b9..d0da8eaaf 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -86,24 +86,27 @@ class ZonesTest(test.TestCase): def test_get_zone_list(self): req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('password' in res_dict['zone']) - self.assertEqual(res.status_int, 200) def test_zone_delete(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -112,13 +115,14 @@ class ZonesTest(test.TestCase): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" req.method = 'POST' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://blah.zoo') self.assertFalse('username' in res_dict['zone']) @@ -126,13 +130,14 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" req.method = 'PUT' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('username' in res_dict['zone']) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index cf2d0e297..7c0135656 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -93,29 +93,29 @@ class ControllerTest(test.TestCase): result = request.get_response(self.TestRouter()) self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) - def test_content_type_from_accept_xml(self): + def test_response_content_type_from_accept_xml(self): request = webob.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/xml") - def test_content_type_from_accept_json(self): + def test_response_content_type_from_accept_json(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/json" result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/json") - def test_content_type_from_query_extension_xml(self): + def test_response_content_type_from_query_extension_xml(self): request = wsgi.Request.blank('/tests/123.xml') result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/xml") - def test_content_type_from_query_extension_json(self): + def test_response_content_type_from_query_extension_json(self): request = wsgi.Request.blank('/tests/123.json') result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/json") - def test_content_type_default_when_unsupported(self): + def test_response_content_type_default_when_unsupported(self): request = wsgi.Request.blank('/tests/123.unsupported') request.headers["Accept"] = "application/unsupported1" result = request.get_response(self.TestRouter()) @@ -125,6 +125,17 @@ class ControllerTest(test.TestCase): class RequestTest(test.TestCase): + def test_request_content_type_missing(self): + request = wsgi.Request.blank('/tests/123') + request.body = "" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + + def test_request_content_type_unsupported(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Content-Type"] = "text/html" + request.body = "asdf
" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + def test_content_type_from_accept_xml(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" @@ -173,40 +184,48 @@ class RequestTest(test.TestCase): self.assertEqual(result, "application/json") - - - class SerializerTest(test.TestCase): - def match(self, url, accept, expect): + def test_xml(self): input_dict = dict(servers=dict(a=(2, 3))) expected_xml = '(2,3)' + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/xml") + result = result.replace('\n', '').replace(' ', '') + self.assertEqual(result, expected_xml) + + def test_json(self): + input_dict = dict(servers=dict(a=(2, 3))) expected_json = '{"servers":{"a":[2,3]}}' - req = webob.Request.blank(url, headers=dict(Accept=accept)) - result = wsgi.Serializer(req.environ).to_content_type(input_dict) + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/json") result = result.replace('\n', '').replace(' ', '') - if expect == 'xml': - self.assertEqual(result, expected_xml) - elif expect == 'json': - self.assertEqual(result, expected_json) - else: - raise "Bad expect value" - - def test_basic(self): - self.match('/servers/4.json', None, expect='json') - self.match('/servers/4', 'application/json', expect='json') - self.match('/servers/4', 'application/xml', expect='xml') - self.match('/servers/4.xml', None, expect='xml') - - def test_defaults_to_json(self): - self.match('/servers/4', None, expect='json') - self.match('/servers/4', 'text/html', expect='json') - - def test_suffix_takes_precedence_over_accept_header(self): - self.match('/servers/4.xml', 'application/json', expect='xml') - self.match('/servers/4.xml.', 'application/json', expect='json') - - def test_deserialize(self): + self.assertEqual(result, expected_json) + + def test_unsupported_content_type(self): + serializer = wsgi.Serializer() + self.assertRaises(exception.InvalidContentType, serializer.serialize, + {}, "text/null") + + def test_deserialize_json(self): + data = """{"a": { + "a1": "1", + "a2": "2", + "bs": ["1", "2", "3", {"c": {"c1": "1"}}], + "d": {"e": "1"}, + "f": "1"}}""" + as_dict = dict(a={ + 'a1': '1', + 'a2': '2', + 'bs': ['1', '2', '3', {'c': dict(c1='1')}], + 'd': {'e': '1'}, + 'f': '1'}) + metadata = {} + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(data, "application/json"), + as_dict) + + def test_deserialize_xml(self): xml = """ 123 @@ -221,11 +240,13 @@ class SerializerTest(test.TestCase): 'd': {'e': '1'}, 'f': '1'}) metadata = {'application/xml': dict(plurals={'bs': 'b', 'ts': 't'})} - serializer = wsgi.Serializer({}, metadata) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) def test_deserialize_empty_xml(self): xml = """""" as_dict = {"a": {}} - serializer = wsgi.Serializer({}) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer() + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..85bfcfd85 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -59,6 +59,7 @@ class DirectTestCase(test.TestCase): req.headers['X-OpenStack-User'] = 'user1' req.headers['X-OpenStack-Project'] = 'proj1' resp = req.get_response(self.auth_router) + self.assertEqual(resp.status_int, 200) data = json.loads(resp.body) self.assertEqual(data['user'], 'user1') self.assertEqual(data['project'], 'proj1') @@ -69,6 +70,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'json=%s' % json.dumps({'data': 'foo'}) resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') @@ -78,6 +80,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'data=foo' resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') diff --git a/nova/wsgi.py b/nova/wsgi.py index 4577439cb..c3e08522d 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -36,6 +36,7 @@ import webob.exc from paste import deploy +from nova import exception from nova import flags from nova import log as logging from nova import utils @@ -102,6 +103,14 @@ class Request(webob.Request): return bm or "application/json" + def get_content_type(self): + try: + ct = self.headers["Content-Type"] + assert ct in ("application/xml", "application/json") + return ct + except Exception: + raise webob.exc.HTTPBadRequest("Invalid content type") + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -343,30 +352,41 @@ class Controller(object): del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) + if type(result) is dict: - return self._serialize(result, req) + content_type = req.best_match() + body = self._serialize(result, content_type) + + response = webob.Response() + response.headers["Content-Type"] = content_type + response.body = body + return response + else: return result - def _serialize(self, data, request): + def _serialize(self, data, content_type): """ - Serialize the given dict to the response type requested in request. + Serialize the given dict to the provided content_type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.to_content_type(data) + serializer = Serializer(_metadata) + try: + return serializer.serialize(data, content_type) + except exception.InvalidContentType: + raise webob.exc.HTTPNotAcceptable() - def _deserialize(self, data, request): + def _deserialize(self, data, content_type): """ - Deserialize the request body to the response type requested in request. + Deserialize the request body to the specefied content type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.deserialize(data) + serializer = Serializer(_metadata) + return serializer.deserialize(data, content_type) class Serializer(object): @@ -374,50 +394,52 @@ class Serializer(object): Serializes and deserializes dictionaries to certain MIME types. """ - def __init__(self, environ, metadata=None): + def __init__(self, metadata=None): """ Create a serializer based on the given WSGI environment. 'metadata' is an optional dict mapping MIME types to information needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = Request.blank('', environ) - suffix = req.path_info.split('.')[-1].lower() - if suffix == 'json': - self.handler = self._to_json - elif suffix == 'xml': - self.handler = self._to_xml - elif 'application/json' in req.accept: - self.handler = self._to_json - elif 'application/xml' in req.accept: - self.handler = self._to_xml - else: - # This is the default - self.handler = self._to_json - def to_content_type(self, data): - """ - Serialize a dictionary into a string. + def _get_serialize_handler(self, content_type): + handlers = { + "application/json": self._to_json, + "application/xml": self._to_xml, + } + + try: + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() - The format of the string will be decided based on the Content Type - requested in self.environ: by Accept: header, or by URL suffix. + def serialize(self, data, content_type): + """ + Serialize a dictionary into a string of the specified content type. """ - return self.handler(data) + return self._get_serialize_handler(content_type)(data) - def deserialize(self, datastring): + def deserialize(self, datastring, content_type): """ Deserialize a string to a dictionary. The string must be in the format of a supported MIME type. """ - datastring = datastring.strip() + return self.get_deserialize_handler(content_type)(datastring) + + def get_deserialize_handler(self, content_type): + handlers = { + "application/json": self._from_json, + "application/xml": self._from_xml, + } + try: - is_xml = (datastring[0] == '<') - if not is_xml: - return utils.loads(datastring) - return self._from_xml(datastring) - except: - return None + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() + + def _from_json(self, datastring): + return utils.loads(datastring) def _from_xml(self, datastring): xmldata = self.metadata.get('application/xml', {}) -- cgit From 417f6ca5c54878a6bea4d545126f93ecb6a043b4 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 3 Mar 2011 22:22:00 +0000 Subject: localize a few error messages. --- nova/api/openstack/accounts.py | 2 +- nova/api/openstack/users.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 264fdab99..3b90d2776 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -47,7 +47,7 @@ class Controller(wsgi.Controller): """ We cannot depend on the db layer to check for admin access for the auth manager, so we do it here """ if not context.is_admin: - raise exception.NotAuthorized("Not admin user.") + raise exception.NotAuthorized(_("Not admin user.")) def show(self, req, id): """Return data about the given account id""" diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index c0b7544f9..ae3bf7791 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -48,7 +48,7 @@ class Controller(wsgi.Controller): """ We cannot depend on the db layer to check for admin access for the auth manager, so we do it here """ if not context.is_admin: - raise exception.NotAuthorized("Not admin user") + raise exception.NotAuthorized(_("Not admin user")) def index(self, req, **kw): """Return all users in brief""" -- cgit From c5bfab9a0d213cee549371f05e74747cfcd8f998 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Thu, 3 Mar 2011 23:05:00 +0000 Subject: Changing output of status from showing the user as the owner, to showing the project --- nova/api/ec2/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index b1917e9ea..cadda97db 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -562,7 +562,7 @@ class CloudController(object): if context.is_admin: v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], - volume['user_id'], + volume['project_id'], volume['host'], instance_data, volume['mountpoint']) -- cgit From a433ddeda77aaa4462694661ecdca71eed6db669 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 02:36:55 +0000 Subject: Replace objectstore images with S3 image service backending to glance or local --- bin/nova-manage | 2 +- nova/api/ec2/cloud.py | 127 +++++++++-------- nova/flags.py | 2 +- nova/image/glance.py | 29 ++-- nova/image/s3.py | 280 ++++++++++++++++++++++++++++---------- nova/image/service.py | 4 +- nova/tests/api/openstack/fakes.py | 11 +- nova/tests/fake_flags.py | 1 + nova/tests/test_cloud.py | 22 ++- nova/tests/test_compute.py | 7 +- nova/tests/test_direct.py | 3 +- nova/tests/test_quota.py | 6 +- 12 files changed, 334 insertions(+), 160 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 9bf3a1bb3..0f7604aeb 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -81,7 +81,7 @@ from nova import log as logging from nova import quota from nova import rpc from nova import utils -from nova.api.ec2.cloud import ec2_id_to_id +from nova.api.ec2.ec2utils import ec2_id_to_id from nova.auth import manager from nova.cloudpipe import pipelib from nova.compute import instance_types diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 844ccbe5e..8c2e77d86 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -39,7 +39,9 @@ from nova import log as logging from nova import network from nova import utils from nova import volume +from nova.api.ec2 import ec2utils from nova.compute import instance_types +from nova.image import s3 FLAGS = flags.FLAGS @@ -73,30 +75,19 @@ def _gen_key(context, user_id, key_name): return {'private_key': private_key, 'fingerprint': fingerprint} -def ec2_id_to_id(ec2_id): - """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" - return int(ec2_id.split('-')[-1], 16) - - -def id_to_ec2_id(instance_id, template='i-%08x'): - """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" - return template % instance_id - - class CloudController(object): """ CloudController provides the critical dispatch between inbound API calls through the endpoint and messages sent to the other nodes. """ def __init__(self): - self.image_service = utils.import_object(FLAGS.image_service) + self.image_service = s3.S3ImageService() self.network_api = network.API() self.volume_api = volume.API() self.compute_api = compute.API( network_api=self.network_api, - image_service=self.image_service, volume_api=self.volume_api, - hostname_factory=id_to_ec2_id) + hostname_factory=ec2utils.id_to_ec2_id) self.setup() def __str__(self): @@ -154,7 +145,7 @@ class CloudController(object): availability_zone = self._get_availability_zone_by_host(ctxt, host) floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) - ec2_id = id_to_ec2_id(instance_ref['id']) + ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { @@ -525,7 +516,7 @@ class CloudController(object): ec2_id = instance_id[0] else: ec2_id = instance_id - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) output = self.compute_api.get_console_output( context, instance_id=instance_id) now = datetime.datetime.utcnow() @@ -535,7 +526,7 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) return self.compute_api.get_ajax_console(context, instance_id=instance_id) @@ -543,7 +534,7 @@ class CloudController(object): if volume_id: volumes = [] for ec2_id in volume_id: - internal_id = ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) volume = self.volume_api.get(context, internal_id) volumes.append(volume) else: @@ -556,11 +547,11 @@ class CloudController(object): instance_data = None if volume.get('instance', None): instance_id = volume['instance']['id'] - instance_ec2_id = id_to_ec2_id(instance_id) + instance_ec2_id = ec2utils.id_to_ec2_id(instance_id) instance_data = '%s[%s]' % (instance_ec2_id, volume['instance']['host']) v = {} - v['volumeId'] = id_to_ec2_id(volume['id'], 'vol-%08x') + v['volumeId'] = ec2utils.id_to_ec2_id(volume['id'], 'vol-%08x') v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] @@ -578,8 +569,7 @@ class CloudController(object): 'device': volume['mountpoint'], 'instanceId': instance_ec2_id, 'status': 'attached', - 'volumeId': id_to_ec2_id(volume['id'], - 'vol-%08x')}] + 'volumeId': v['volumeId']}] else: v['attachmentSet'] = [{}] @@ -598,12 +588,12 @@ class CloudController(object): return {'volumeSet': [self._format_volume(context, dict(volume))]} def delete_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) self.volume_api.delete(context, volume_id=volume_id) return True def update_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) updatable_fields = ['display_name', 'display_description'] changes = {} for field in updatable_fields: @@ -614,8 +604,8 @@ class CloudController(object): return True def attach_volume(self, context, volume_id, instance_id, device, **kwargs): - volume_id = ec2_id_to_id(volume_id) - instance_id = ec2_id_to_id(instance_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) msg = _("Attach volume %(volume_id)s to instance %(instance_id)s" " at %(device)s") % locals() LOG.audit(msg, context=context) @@ -626,22 +616,22 @@ class CloudController(object): volume = self.volume_api.get(context, volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], - 'instanceId': id_to_ec2_id(instance_id), + 'instanceId': ec2utils.id_to_ec2_id(instance_id), 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': id_to_ec2_id(volume_id, 'vol-%08x')} + 'volumeId': ec2utils.id_to_ec2_id(volume_id, 'vol-%08x')} def detach_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) LOG.audit(_("Detach volume %s"), volume_id, context=context) volume = self.volume_api.get(context, volume_id) instance = self.compute_api.detach_volume(context, volume_id=volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], - 'instanceId': id_to_ec2_id(instance['id']), + 'instanceId': ec2utils.id_to_ec2_id(instance['id']), 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': id_to_ec2_id(volume_id, 'vol-%08x')} + 'volumeId': ec2utils.id_to_ec2_id(volume_id, 'vol-%08x')} def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -675,7 +665,7 @@ class CloudController(object): if instance_id: instances = [] for ec2_id in instance_id: - internal_id = ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) instance = self.compute_api.get(context, instance_id=internal_id) instances.append(instance) @@ -687,7 +677,7 @@ class CloudController(object): continue i = {} instance_id = instance['id'] - ec2_id = id_to_ec2_id(instance_id) + ec2_id = ec2utils.id_to_ec2_id(instance_id) i['instanceId'] = ec2_id i['imageId'] = instance['image_id'] i['instanceState'] = { @@ -755,7 +745,7 @@ class CloudController(object): if (floating_ip_ref['fixed_ip'] and floating_ip_ref['fixed_ip']['instance']): instance_id = floating_ip_ref['fixed_ip']['instance']['id'] - ec2_id = id_to_ec2_id(instance_id) + ec2_id = ec2utils.id_to_ec2_id(instance_id) address_rv = {'public_ip': address, 'instance_id': ec2_id} if context.is_admin: @@ -778,7 +768,7 @@ class CloudController(object): def associate_address(self, context, instance_id, public_ip, **kwargs): LOG.audit(_("Associate address %(public_ip)s to" " instance %(instance_id)s") % locals(), context=context) - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.associate_floating_ip(context, instance_id=instance_id, address=public_ip) @@ -791,13 +781,17 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) + if kwargs.get('kernel_id'): + kwargs['kernel_id'] = ec2utils.ec2_id_to_id(kwargs['kernel_id']) + if kwargs.get('ramdisk_id'): + kwargs['ramdisk_id'] = ec2utils.ec2_id_to_id(kwargs['ramdisk_id']) instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id=kwargs['image_id'], + image_id=ec2utils.ec2_id_to_id(kwargs['image_id']), min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, - kernel_id=kwargs.get('kernel_id', None), + kernel_id=kwargs.get('kernel_id'), ramdisk_id=kwargs.get('ramdisk_id'), display_name=kwargs.get('display_name'), display_description=kwargs.get('display_description'), @@ -814,7 +808,7 @@ class CloudController(object): instance_id is a kwarg so its name cannot be modified.""" LOG.debug(_("Going to start terminating instances")) for ec2_id in instance_id: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.delete(context, instance_id=instance_id) return True @@ -822,19 +816,19 @@ class CloudController(object): """instance_id is a list of instance ids""" LOG.audit(_("Reboot instance %r"), instance_id, context=context) for ec2_id in instance_id: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.reboot(context, instance_id=instance_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.rescue(context, instance_id=instance_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.unrescue(context, instance_id=instance_id) return True @@ -845,41 +839,50 @@ class CloudController(object): if field in kwargs: changes[field] = kwargs[field] if changes: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.update(context, instance_id=instance_id, **kwargs) return True - def _format_image(self, context, image): + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} i['imageId'] = image.get('id') - i['kernelId'] = image.get('kernel_id') - i['ramdiskId'] = image.get('ramdisk_id') - i['imageOwnerId'] = image.get('owner_id') - i['imageLocation'] = image.get('location') - i['imageState'] = image.get('status') + i['kernelId'] = image['properties'].get('kernel_id') + i['ramdiskId'] = image['properties'].get('ramdisk_id') + i['imageOwnerId'] = image['properties'].get('owner_id') + i['imageLocation'] = image['properties'].get('image_location') + i['imageState'] = image['properties'].get('image_state') i['type'] = image.get('type') - i['isPublic'] = image.get('is_public') - i['architecture'] = image.get('architecture') + i['isPublic'] = image['properties'].get('is_public') == 'True' + i['architecture'] = image['properties'].get('architecture') return i def describe_images(self, context, image_id=None, **kwargs): # NOTE: image_id is a list! - images = self.image_service.index(context) if image_id: - images = filter(lambda x: x['id'] in image_id, images) - images = [self._format_image(context, i) for i in images] + images = [] + for ec2_id in image_id: + try: + image = self.image_service.show(context, ec2_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % + ec2_id) + images.append(image) + else: + images = self.image_service.detail(context) + images = [self._format_image(i) for i in images] return {'imagesSet': images} def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - self.image_service.deregister(context, image_id) + self.image_service.delete(context, image_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - image_id = self.image_service.register(context, image_location) + image = {"image_location": image_location} + image_id = self.image_service.create(context, image) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -890,11 +893,10 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: - image = self._format_image(context, - self.image_service.show(context, + image = self._format_image(self.image_service.show(context, image_id)) - except IndexError: - raise exception.ApiError(_('invalid id: %s') % image_id) + except (IndexError, exception.NotFound): + raise exception.NotFound(_('Image %s not found') % image_id) result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: result['launchPermission'].append({'group': 'all'}) @@ -913,7 +915,14 @@ class CloudController(object): if not operation_type in ['add', 'remove']: raise exception.ApiError(_('operation_type must be add or remove')) LOG.audit(_("Updating image %s publicity"), image_id, context=context) - return self.image_service.modify(context, image_id, operation_type) + + try: + metadata = self.image_service.show(context, image_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % image_id) + del(metadata['id']) + metadata['properties']['is_public'] = (operation_type == 'add') + return self.image_service.update(context, image_id, metadata) def update_image(self, context, image_id, **kwargs): result = self.image_service.update(context, image_id, dict(kwargs)) diff --git a/nova/flags.py b/nova/flags.py index 8cf199b2f..f01a4d096 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -346,7 +346,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.s3.S3ImageService', +DEFINE_string('image_service', 'nova.image.glance.GlanceImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), diff --git a/nova/image/glance.py b/nova/image/glance.py index 593c4bce6..7db94c0d4 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -21,6 +21,8 @@ import httplib import json import urlparse +from glance.common import exception as glance_exception + from nova import exception from nova import flags from nova import log as logging @@ -57,27 +59,32 @@ class GlanceImageService(service.BaseImageService): """ Returns a dict containing image data for the given opaque image id. """ - image = self.client.get_image_meta(id) - if image: - return image - raise exception.NotFound + try: + image = self.client.get_image_meta(id) + except glance_exception.NotFound: + raise exception.NotFound + return image - def create(self, context, data): + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. :raises AlreadyExists if the image already exist. """ - return self.client.add_image(image_meta=data) + return self.client.add_image(metadata, data) - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. :raises NotFound if the image does not exist. """ - return self.client.update_image(image_id, data) + try: + result = self.client.update_image(image_id, metadata, data) + except glance_exception.NotFound: + raise exception.NotFound + return result def delete(self, context, image_id): """ @@ -86,7 +93,11 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - return self.client.delete_image(image_id) + try: + result = self.client.delete_image(image_id) + except glance_exception.NotFound: + raise exception.NotFound + return result def delete_all(self): """ diff --git a/nova/image/s3.py b/nova/image/s3.py index 14135a1ee..a740b010c 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -21,8 +21,12 @@ Proxy AMI-related calls from the cloud controller, to the running objectstore service. """ -import json -import urllib +import binascii +import os +import shutil +import tarfile +import tempfile +from xml.etree import ElementTree import boto.s3.connection @@ -31,84 +35,89 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service +from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS +flags.DEFINE_string('image_decryption_dir', '/tmp', + 'parent dir for tempdir used for image decryption') -def map_s3_to_base(image): - """Convert from S3 format to format defined by BaseImageService.""" - i = {} - i['id'] = image.get('imageId') - i['name'] = image.get('imageId') - i['kernel_id'] = image.get('kernelId') - i['ramdisk_id'] = image.get('ramdiskId') - i['location'] = image.get('imageLocation') - i['owner_id'] = image.get('imageOwnerId') - i['status'] = image.get('imageState') - i['type'] = image.get('type') - i['is_public'] = image.get('isPublic') - i['architecture'] = image.get('architecture') - return i +_type_prefix_map = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + + +def image_ec2_id(image_id, image_type): + prefix = _type_prefix_map[image_type] + template = prefix + '-%08x' + return ec2utils.id_to_ec2_id(int(image_id), template=template) class S3ImageService(service.BaseImageService): + def __init__(self, service=None, *args, **kwargs): + if service == None: + service = utils.import_object(FLAGS.image_service) + self.service = service + self.service.__init__(*args, **kwargs) + + def create(self, context, properties, data=None): + """image should contain image_location""" + image_id, metadata = self._s3_create(context, properties) + return image_ec2_id(image_id, metadata['type']) + + def delete(self, context, image_id): + # FIXME(vish): call to show is to check filter + self.show(context, image_id) + image_id = ec2utils.ec2_id_to_id(image_id) + self.service.delete(context, image_id) - def modify(self, context, image_id, operation): - self._conn(context).make_request( - method='POST', - bucket='_images', - query_args=self._qs({'image_id': image_id, - 'operation': operation})) - return True - - def update(self, context, image_id, attributes): - """update an image's attributes / info.json""" - attributes.update({"image_id": image_id}) - self._conn(context).make_request( - method='POST', - bucket='_images', - query_args=self._qs(attributes)) - return True - - def register(self, context, image_location): - """ rpc call to register a new image based from a manifest """ - image_id = utils.generate_uid('ami') - self._conn(context).make_request( - method='PUT', - bucket='_images', - query_args=self._qs({'image_location': image_location, - 'image_id': image_id})) - return image_id + def update(self, context, image_id, metadata, data=None): + # FIXME(vish): call to show is to check filter + self.show(context, image_id) + image_id = ec2utils.ec2_id_to_id(image_id) + image = self.service.update(context, image_id, metadata, data) + image['id'] = image_ec2_id(image['id'], image['type']) + return image def index(self, context): - """Return a list of all images that a user can see.""" - response = self._conn(context).make_request( - method='GET', - bucket='_images') - images = json.loads(response.read()) - return [map_s3_to_base(i) for i in images] + images = self.service.index(context) + # FIXME(vish): index doesn't filter so we do it manually + return self._filter(context, images) + + def detail(self, context): + images = self.service.detail(context) + # FIXME(vish): detail doesn't filter so we do it manually + return self._filter(context, images) + + @staticmethod + def _is_visible(context, image): + return (context.is_admin + or context.project_id == image['properties']['owner_id'] + or image['properties']['is_public'] == 'True') + + @staticmethod + def _filter(context, images): + filtered = [] + for image in images: + if not S3ImageService._is_visible(context, image): + continue + image['id'] = image_ec2_id(image['id'], image['type']) + filtered.append(image) + return filtered def show(self, context, image_id): - """return a image object if the context has permissions""" - if FLAGS.connection_type == 'fake': - return {'imageId': 'bar'} - result = self.index(context) - result = [i for i in result if i['id'] == image_id] - if not result: - raise exception.NotFound(_('Image %s could not be found') - % image_id) - image = result[0] + image_id = ec2utils.ec2_id_to_id(image_id) + image = self.service.show(context, image_id) + if not self._is_visible(context, image): + raise exception.NotFound + image['id'] = image_ec2_id(image['id'], image['type']) return image - def deregister(self, context, image_id): - """ unregister an image """ - self._conn(context).make_request( - method='DELETE', - bucket='_images', - query_args=self._qs({'image_id': image_id})) - - def _conn(self, context): + @staticmethod + def _conn(context): + # TODO(vish): is there a better way to get creds to sign + # for the user? access = manager.AuthManager().get_access_key(context.user, context.project) secret = str(context.user.secret) @@ -120,8 +129,139 @@ class S3ImageService(service.BaseImageService): port=FLAGS.s3_port, host=FLAGS.s3_host) - def _qs(self, params): - pairs = [] - for key in params.keys(): - pairs.append(key + '=' + urllib.quote(params[key])) - return '&'.join(pairs) + @staticmethod + def _download_file(bucket, filename, local_dir): + key = bucket.get_key(filename) + local_filename = os.path.join(local_dir, filename) + key.get_contents_to_filename(local_filename) + return local_filename + + def _s3_create(self, context, properties): + image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir) + + image_location = properties['image_location'] + bucket_name = image_location.split("/")[0] + manifest_path = image_location[len(bucket_name) + 1:] + bucket = self._conn(context).get_bucket(bucket_name) + key = bucket.get_key(manifest_path) + manifest = key.get_contents_as_string() + + manifest = ElementTree.fromstring(manifest) + image_type = 'machine' + + try: + kernel_id = manifest.find("machine_configuration/kernel_id").text + if kernel_id == 'true': + image_type = 'kernel' + kernel_id = None + except: + kernel_id = None + + try: + ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text + if ramdisk_id == 'true': + image_type = 'ramdisk' + ramdisk_id = None + except: + ramdisk_id = None + + try: + arch = manifest.find("machine_configuration/architecture").text + except: + arch = 'x86_64' + + properties.update({'owner_id': context.project_id, + 'architecture': arch}) + + if kernel_id: + properties['kernel_id'] = kernel_id + + if ramdisk_id: + properties['ramdisk_id'] = ramdisk_id + + properties['is_public'] = False + metadata = {'type': image_type, + 'status': 'queued', + 'is_public': True, + 'properties': properties} + metadata['properties']['image_state'] = 'pending' + image = self.service.create(context, metadata) + image_id = image['id'] + + parts = [] + for fn_element in manifest.find("image").getiterator("filename"): + part = self._download_file(bucket, fn_element.text, image_path) + parts.append(part) + + # NOTE(vish): this may be suboptimal, should we use cat? + encrypted_filename = os.path.join(image_path, 'image.encrypted') + with open(encrypted_filename, 'w') as combined: + for filename in parts: + with open(filename) as part: + shutil.copyfileobj(part, combined) + + metadata['properties']['image_state'] = 'decrypting' + self.service.update(context, image_id, metadata) + + hex_key = manifest.find("image/ec2_encrypted_key").text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find("image/ec2_encrypted_iv").text + encrypted_iv = binascii.a2b_hex(hex_iv) + + # FIXME(vish): grab key from common service so this can run on + # any host. + cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem") + + decrypted_filename = os.path.join(image_path, 'image.tar.gz') + self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename) + + metadata['properties']['image_state'] = 'untarring' + self.service.update(context, image_id, metadata) + + unz_filename = self._untarzip_image(image_path, decrypted_filename) + + metadata['properties']['image_state'] = 'uploading' + with open(unz_filename) as image_file: + self.service.update(context, image_id, metadata, image_file) + metadata['properties']['image_state'] = 'available' + self.service.update(context, image_id, metadata) + + shutil.rmtree(image_path) + return image_id, metadata + + @staticmethod + def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename): + key, err = utils.execute( + 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt private key: %s") + % err) + iv, err = utils.execute( + 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt initialization " + "vector: %s") % err) + + _out, err = utils.execute( + 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' + % (encrypted_filename, key, iv, decrypted_filename), + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt image file " + "%(image_file)s: %(err)s") % + {'image_file': encrypted_filename, + 'err': err}) + + @staticmethod + def _untarzip_image(path, filename): + tar_file = tarfile.open(filename, "r|gz") + tar_file.extractall(path) + image_file = tar_file.getnames()[0] + tar_file.close() + return os.path.join(path, image_file) diff --git a/nova/image/service.py b/nova/image/service.py index ebee2228d..e429955f4 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -76,7 +76,7 @@ class BaseImageService(object): """ raise NotImplementedError - def create(self, context, data): + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. @@ -85,7 +85,7 @@ class BaseImageService(object): """ raise NotImplementedError - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. :raises NotFound if the image does not exist. diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..7b016db08 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -25,6 +25,7 @@ import webob.dec from paste import urlmap from glance import client as glance_client +from glance.common import exception as glance_exc from nova import auth from nova import context @@ -149,25 +150,25 @@ def stub_out_glance(stubs, initial_fixtures=None): for f in self.fixtures: if f['id'] == image_id: return f - return None + raise glance_exc.NotFound - def fake_add_image(self, image_meta): + def fake_add_image(self, image_meta, data=None): id = ''.join(random.choice(string.letters) for _ in range(20)) image_meta['id'] = id self.fixtures.append(image_meta) return id - def fake_update_image(self, image_id, image_meta): + def fake_update_image(self, image_id, image_meta, data=None): f = self.fake_get_image_meta(image_id) if not f: - raise exc.NotFound + raise glance_exc.NotFound f.update(image_meta) def fake_delete_image(self, image_id): f = self.fake_get_image_meta(image_id) if not f: - raise exc.NotFound + raise glance_exc.NotFound self.fixtures.remove(f) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index cbd949477..5d7ca98b5 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -32,6 +32,7 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 8 FLAGS.num_networks = 2 FLAGS.fake_network = True +FLAGS.image_service = 'nova.image.local.LocalImageService' flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 061910013..7d7b91658 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -38,6 +38,8 @@ from nova import test from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils +from nova.image import local from nova.objectstore import image @@ -76,6 +78,11 @@ class CloudTestCase(test.TestCase): project=self.project) host = self.network.get_network_host(self.context.elevated()) + def fake_image_show(meh, context, id): + return dict(kernelId=1, ramdiskId=1) + + self.stubs.Set(local.LocalImageService, 'show', fake_image_show) + def tearDown(self): network_ref = db.project_get_network(self.context, self.project.id) @@ -122,7 +129,7 @@ class CloudTestCase(test.TestCase): self.cloud.allocate_address(self.context) inst = db.instance_create(self.context, {'host': self.compute.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.associate_address(self.context, instance_id=ec2_id, public_ip=address) @@ -158,12 +165,12 @@ class CloudTestCase(test.TestCase): vol2 = db.volume_create(self.context, {}) result = self.cloud.describe_volumes(self.context) self.assertEqual(len(result['volumeSet']), 2) - volume_id = cloud.id_to_ec2_id(vol2['id'], 'vol-%08x') + volume_id = ec2utils.id_to_ec2_id(vol2['id'], 'vol-%08x') result = self.cloud.describe_volumes(self.context, volume_id=[volume_id]) self.assertEqual(len(result['volumeSet']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['volumeSet'][0]['volumeId']), + ec2utils.ec2_id_to_id(result['volumeSet'][0]['volumeId']), vol2['id']) db.volume_destroy(self.context, vol1['id']) db.volume_destroy(self.context, vol2['id']) @@ -200,7 +207,7 @@ class CloudTestCase(test.TestCase): result = self.cloud.describe_instances(self.context) result = result['reservationSet'][0] self.assertEqual(len(result['instancesSet']), 2) - instance_id = cloud.id_to_ec2_id(inst2['id']) + instance_id = ec2utils.id_to_ec2_id(inst2['id']) result = self.cloud.describe_instances(self.context, instance_id=[instance_id]) result = result['reservationSet'][0] @@ -216,6 +223,7 @@ class CloudTestCase(test.TestCase): def test_console_output(self): image_id = FLAGS.default_image + print image_id instance_type = FLAGS.default_instance_type max_count = 1 kwargs = {'image_id': image_id, @@ -347,7 +355,7 @@ class CloudTestCase(test.TestCase): def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.update_instance(self.context, ec2_id, display_name='c00l 1m4g3') inst = db.instance_get(self.context, inst['id']) @@ -365,7 +373,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_display_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), display_name='c00l v0lum3') vol = db.volume_get(self.context, vol['id']) self.assertEqual('c00l v0lum3', vol['display_name']) @@ -374,7 +382,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_wont_update_private_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), mountpoint='/not/here') vol = db.volume_get(self.context, vol['id']) self.assertEqual(None, vol['mountpoint']) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 949b5e6eb..1f49baaf6 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -31,7 +31,7 @@ from nova import test from nova import utils from nova.auth import manager from nova.compute import instance_types - +from nova.image import local LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS @@ -47,6 +47,11 @@ class ComputeTestCase(test.TestCase): network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute.API() + + def fake_image_show(meh, context, id): + return dict(kernelId=1, ramdiskId=1) + + self.stubs.Set(local.LocalImageService, 'show', fake_image_show) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..b130e3f53 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -90,8 +90,7 @@ class DirectTestCase(test.TestCase): class DirectCloudTestCase(test_cloud.CloudTestCase): def setUp(self): super(DirectCloudTestCase, self).setUp() - compute_handle = compute.API(image_service=self.cloud.image_service, - network_api=self.cloud.network_api, + compute_handle = compute.API(network_api=self.cloud.network_api, volume_api=self.cloud.volume_api) direct.register_service('compute', compute_handle) self.router = direct.JsonParamsMiddleware(direct.Router()) diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 4ecb36b54..ca8abdb36 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -57,7 +57,7 @@ class QuotaTestCase(test.TestCase): def _create_instance(self, cores=2): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 'ami-1' inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -123,7 +123,7 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type='m1.small', - image_id='fake') + image_id='ami-1') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -136,7 +136,7 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type='m1.small', - image_id='fake') + image_id='ami-1') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) -- cgit From bc94ec23100de9f07e04b0348823d4f103a9daa5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 02:49:12 +0000 Subject: use LocalImageServiceByDefault --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/flags.py b/nova/flags.py index f01a4d096..cb47ca8d1 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -346,7 +346,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.glance.GlanceImageService', +DEFINE_string('image_service', 'nova.image.local.LocalImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), -- cgit From 13307e02258a5a08bedb1ed933a107668aac6457 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:04:49 +0000 Subject: make local image service work --- nova/api/ec2/cloud.py | 2 +- nova/image/local.py | 64 ++++++++++++++++++++++++++++++++--------------- nova/objectstore/image.py | 3 +-- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8c2e77d86..aa1dcbe33 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -853,7 +853,7 @@ class CloudController(object): i['imageLocation'] = image['properties'].get('image_location') i['imageState'] = image['properties'].get('image_state') i['type'] = image.get('type') - i['isPublic'] = image['properties'].get('is_public') == 'True' + i['isPublic'] = str(image['properties'].get('is_public', '')) == 'True' i['architecture'] = image['properties'].get('architecture') return i diff --git a/nova/image/local.py b/nova/image/local.py index f78b9aa89..b4616729a 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -15,57 +15,81 @@ # License for the specific language governing permissions and limitations # under the License. -import cPickle as pickle +import json import os.path import random -import tempfile +import shutil +from nova import flags from nova import exception from nova.image import service -class LocalImageService(service.BaseImageService): +FLAGS = flags.FLAGS +flags.DEFINE_string('images_path', '$state_path/images', + 'path to decrypted images') +class LocalImageService(service.BaseImageService): """Image service storing images to local disk. + It assumes that image_ids are integers. """ def __init__(self): - self._path = tempfile.mkdtemp() + self._path = FLAGS.images_path - def _path_to(self, image_id): + def _path_to(self, image_id, fname='info.json'): + if fname: + return os.path.join(self._path, str(image_id), fname) return os.path.join(self._path, str(image_id)) def _ids(self): """The list of all image ids.""" - return [int(i) for i in os.listdir(self._path)] + return [int(i) for i in os.listdir(self._path) + if unicode(i).isnumeric()] def index(self, context): - return [dict(id=i['id'], name=i['name']) for i in self.detail(context)] + return [dict(image_id=i['id'], name=i.get('name')) + for i in self.detail(context)] def detail(self, context): - return [self.show(context, id) for id in self._ids()] - - def show(self, context, id): + images = [] + for image_id in self._ids(): + try: + image = self.show(context, image_id) + images.append(image) + except exception.NotFound: + continue + return images + + def show(self, context, image_id): try: - return pickle.load(open(self._path_to(id))) + with open(self._path_to(image_id)) as metadata_file: + return json.load(metadata_file) except IOError: raise exception.NotFound - def create(self, context, data): + def create(self, context, metadata, data=None): """Store the image data and return the new image id.""" - id = random.randint(0, 2 ** 31 - 1) - data['id'] = id - self.update(context, id, data) - return id + image_id = random.randint(0, 2 ** 31 - 1) + image_path = self._path_to(image_id, None) + if not os.path.exists(image_path): + os.mkdir(image_path) + return self.update(context, image_id, metadata, data) - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data.""" + metadata['id'] = image_id try: - pickle.dump(data, open(self._path_to(image_id), 'w')) + with open(self._path_to(image_id), 'w') as metadata_file: + json.dump(metadata, metadata_file) + if data: + with open(self._path_to(image_id, 'image'), 'w') as image_file: + shutil.copyfileobj(data, image_file) except IOError: raise exception.NotFound + return metadata def delete(self, context, image_id): """Delete the given image. @@ -79,8 +103,8 @@ class LocalImageService(service.BaseImageService): def delete_all(self): """Clears out all images in local directory.""" - for id in self._ids(): - os.unlink(self._path_to(id)) + for image_id in self._ids(): + os.unlink(self._path_to(image_id)) def delete_imagedir(self): """Deletes the local directory. diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 27227e2ca..8013cbd9c 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -37,8 +37,7 @@ from nova.objectstore import bucket FLAGS = flags.FLAGS -flags.DEFINE_string('images_path', '$state_path/images', - 'path to decrypted images') +flags.DECLARE('images_path', 'nova.image.local') class Image(object): -- cgit From cf9bc248f0fc318c4a9fb5087f257216312e39d1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:21:28 +0000 Subject: fix a couple issues with local, update the glance fake to actually return the same types as the real client, fix the image tests --- nova/image/local.py | 12 +++--------- nova/tests/api/openstack/fakes.py | 3 ++- nova/tests/api/openstack/test_images.py | 15 +++++++++------ 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/nova/image/local.py b/nova/image/local.py index b4616729a..6fa648b6b 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -29,6 +29,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('images_path', '$state_path/images', 'path to decrypted images') + class LocalImageService(service.BaseImageService): """Image service storing images to local disk. @@ -97,18 +98,11 @@ class LocalImageService(service.BaseImageService): """ try: - os.unlink(self._path_to(image_id)) + shutil.rmtree(self._path_to(image_id, None)) except IOError: raise exception.NotFound def delete_all(self): """Clears out all images in local directory.""" for image_id in self._ids(): - os.unlink(self._path_to(image_id)) - - def delete_imagedir(self): - """Deletes the local directory. - Raises OSError if directory is not empty. - - """ - os.rmdir(self._path) + shutil.rmtree(self._path_to(image_id, None)) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 7b016db08..2c4e57246 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -156,7 +156,7 @@ def stub_out_glance(stubs, initial_fixtures=None): id = ''.join(random.choice(string.letters) for _ in range(20)) image_meta['id'] = id self.fixtures.append(image_meta) - return id + return image_meta def fake_update_image(self, image_id, image_meta, data=None): f = self.fake_get_image_meta(image_id) @@ -164,6 +164,7 @@ def stub_out_glance(stubs, initial_fixtures=None): raise glance_exc.NotFound f.update(image_meta) + return f def fake_delete_image(self, image_id): f = self.fake_get_image_meta(image_id) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index e232bc3d5..eb5039bdb 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,6 +22,8 @@ and as a WSGI layer import json import datetime +import shutil +import tempfile import stubout import webob @@ -54,7 +56,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index(self.context)) - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] self.assertNotEquals(None, id) self.assertEquals(num_images + 1, @@ -71,7 +73,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index(self.context)) - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] self.assertNotEquals(None, id) @@ -89,7 +91,7 @@ class BaseImageServiceTests(object): 'instance_id': None, 'progress': None} - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] fixture['status'] = 'in progress' @@ -118,7 +120,7 @@ class BaseImageServiceTests(object): ids = [] for fixture in fixtures: - new_id = self.service.create(self.context, fixture) + new_id = self.service.create(self.context, fixture)['id'] ids.append(new_id) num_images = len(self.service.index(self.context)) @@ -137,14 +139,15 @@ class LocalImageServiceTest(test.TestCase, def setUp(self): super(LocalImageServiceTest, self).setUp() + self.tempdir = tempfile.mkdtemp() + self.flags(images_path=self.tempdir) self.stubs = stubout.StubOutForTesting() service_class = 'nova.image.local.LocalImageService' self.service = utils.import_object(service_class) self.context = context.RequestContext(None, None) def tearDown(self): - self.service.delete_all() - self.service.delete_imagedir() + shutil.rmtree(self.tempdir) self.stubs.UnsetAll() super(LocalImageServiceTest, self).tearDown() -- cgit From e2c95e198f1982bc50bc95bc61ef3211b17937a2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:55:41 +0000 Subject: spawn a greenthread for image registration because it is slow --- nova/image/s3.py | 67 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/nova/image/s3.py b/nova/image/s3.py index a740b010c..e9542c7bd 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -22,6 +22,7 @@ objectstore service. """ import binascii +import eventlet import os import shutil import tarfile @@ -188,46 +189,50 @@ class S3ImageService(service.BaseImageService): image = self.service.create(context, metadata) image_id = image['id'] - parts = [] - for fn_element in manifest.find("image").getiterator("filename"): - part = self._download_file(bucket, fn_element.text, image_path) - parts.append(part) + def delayed_create(): + parts = [] + for fn_element in manifest.find("image").getiterator("filename"): + part = self._download_file(bucket, fn_element.text, image_path) + parts.append(part) - # NOTE(vish): this may be suboptimal, should we use cat? - encrypted_filename = os.path.join(image_path, 'image.encrypted') - with open(encrypted_filename, 'w') as combined: - for filename in parts: - with open(filename) as part: - shutil.copyfileobj(part, combined) + # NOTE(vish): this may be suboptimal, should we use cat? + encrypted_filename = os.path.join(image_path, 'image.encrypted') + with open(encrypted_filename, 'w') as combined: + for filename in parts: + with open(filename) as part: + shutil.copyfileobj(part, combined) - metadata['properties']['image_state'] = 'decrypting' - self.service.update(context, image_id, metadata) + metadata['properties']['image_state'] = 'decrypting' + self.service.update(context, image_id, metadata) - hex_key = manifest.find("image/ec2_encrypted_key").text - encrypted_key = binascii.a2b_hex(hex_key) - hex_iv = manifest.find("image/ec2_encrypted_iv").text - encrypted_iv = binascii.a2b_hex(hex_iv) + hex_key = manifest.find("image/ec2_encrypted_key").text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find("image/ec2_encrypted_iv").text + encrypted_iv = binascii.a2b_hex(hex_iv) - # FIXME(vish): grab key from common service so this can run on - # any host. - cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem") + # FIXME(vish): grab key from common service so this can run on + # any host. + cloud_pk = os.path.join(FLAGS.ca_path, "private/cakey.pem") - decrypted_filename = os.path.join(image_path, 'image.tar.gz') - self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, - cloud_private_key, decrypted_filename) + decrypted_filename = os.path.join(image_path, 'image.tar.gz') + self._decrypt_image(encrypted_filename, encrypted_key, + encrypted_iv, cloud_pk, decrypted_filename) - metadata['properties']['image_state'] = 'untarring' - self.service.update(context, image_id, metadata) + metadata['properties']['image_state'] = 'untarring' + self.service.update(context, image_id, metadata) - unz_filename = self._untarzip_image(image_path, decrypted_filename) + unz_filename = self._untarzip_image(image_path, decrypted_filename) - metadata['properties']['image_state'] = 'uploading' - with open(unz_filename) as image_file: - self.service.update(context, image_id, metadata, image_file) - metadata['properties']['image_state'] = 'available' - self.service.update(context, image_id, metadata) + metadata['properties']['image_state'] = 'uploading' + with open(unz_filename) as image_file: + self.service.update(context, image_id, metadata, image_file) + metadata['properties']['image_state'] = 'available' + self.service.update(context, image_id, metadata) + + shutil.rmtree(image_path) + + eventlet.spawn_n(delayed_create) - shutil.rmtree(image_path) return image_id, metadata @staticmethod -- cgit From 517a571f8905c32efd45f7b3410fb263ad705545 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:58:49 +0000 Subject: add the ec2utils file i forgot --- nova/api/ec2/ec2utils.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 nova/api/ec2/ec2utils.py diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py new file mode 100644 index 000000000..0ea22c0e6 --- /dev/null +++ b/nova/api/ec2/ec2utils.py @@ -0,0 +1,27 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def ec2_id_to_id(ec2_id): + """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" + return int(ec2_id.split('-')[-1], 16) + + +def id_to_ec2_id(instance_id, template='i-%08x'): + """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" + return template % instance_id -- cgit From 1d8914fc752f7182f942cdd40f2ba18baedeed0c Mon Sep 17 00:00:00 2001 From: Cerberus Date: Fri, 4 Mar 2011 11:19:35 -0600 Subject: More fixes --- nova/api/openstack/servers.py | 4 ++-- nova/compute/api.py | 8 ++++---- nova/compute/manager.py | 2 +- nova/tests/xenapi/stubs.py | 2 +- nova/virt/xenapi/vm_utils.py | 16 ++++++++-------- nova/virt/xenapi/vmops.py | 11 ++++++----- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ceb17c9e4..c2bf42b72 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -203,8 +203,8 @@ class Controller(wsgi.Controller): return exc.HTTPNoContent() def action(self, req, id): - """ Multi-purpose method used to reboot, rebuild, or - resize a server """ + """Multi-purpose method used to reboot, rebuild, or + resize a server""" actions = { 'reboot': self._action_reboot, diff --git a/nova/compute/api.py b/nova/compute/api.py index bfa5c7dba..ce9da727d 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -430,8 +430,8 @@ class API(base.Base): migration_ref = self.db.migration_get_by_instance_and_status(context, instance_id, 'finished') if not migration_ref: - raise exception.NotFound(_("No finished migrations found for \ - instance")) + raise exception.NotFound(_("No finished migrations found for " + "instance")) params = {'migration_id': migration_ref['id']} self._cast_compute_message('revert_resize', context, instance_id, @@ -444,8 +444,8 @@ class API(base.Base): migration_ref = self.db.migration_get_by_instance_and_status(context, instance_id, 'finished') if not migration_ref: - raise exception.NotFound(_("No finished migrations found for \ - instance")) + raise exception.NotFound(_("No finished migrations found for " + "instance")) instance_ref = self.db.instance_get(context, instance_id) params = {'migration_id': migration_ref['id']} self._cast_compute_message('confirm_resize', context, instance_id, diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 1c42b383c..b3e864154 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -414,7 +414,7 @@ class ComputeManager(manager.Manager): @exception.wrap_exception @checks_instance_lock def confirm_resize(self, context, instance_id, migration_id): - """ Destroys the source instance """ + """Destroys the source instance""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) migration_ref = self.db.migration_get(context, migration_id) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index d17951b81..caefcff34 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -221,7 +221,7 @@ class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): class FakeSessionForMigrationTests(fake.SessionBase): - """ Stubs out a XenAPISession for Migration tests """ + """Stubs out a XenAPISession for Migration tests""" def __init__(self, uri): super(FakeSessionForMigrationTests, self).__init__(uri) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index ca2d634f1..eff207a51 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -270,8 +270,8 @@ class VMHelper(HelperBase): @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): - """ Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI, - Snapshot VHD """ + """Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI, + Snapshot VHD""" #TODO(sirp): Add quiesce and VSS locking support when Windows support # is added LOG.debug(_("Snapshotting VM %(vm_ref)s with label '%(label)s'...") @@ -284,7 +284,7 @@ class VMHelper(HelperBase): original_parent_uuid = get_vhd_parent_uuid(session, vm_vdi_ref) task = session.call_xenapi('Async.VM.snapshot', vm_ref, label) - template_vm_ref = session.wait_for_task(instance_id, task) + template_vm_ref = session.wait_for_task(task, instance_id) template_vdi_rec = cls.get_vdi_for_vm_safely(session, template_vm_ref)[1] template_vdi_uuid = template_vdi_rec["uuid"] @@ -302,14 +302,14 @@ class VMHelper(HelperBase): @classmethod def get_sr(cls, session, sr_label='slices'): - """ Finds the SR named by the given name label and returns - the UUID """ + """Finds the SR named by the given name label and returns + the UUID""" return session.call_xenapi('SR.get_by_name_label', sr_label)[0] @classmethod def get_sr_path(cls, session, sr_label='slices'): - """ Finds the SR and then coerces it into a path on the dom0 file - system """ + """Finds the SR and then coerces it into a path on the dom0 file + system""" return FLAGS.xenapi_sr_base_path + cls.get_sr(session, sr_label) @classmethod @@ -643,7 +643,7 @@ class VMHelper(HelperBase): if sr_ref: LOG.debug(_("Re-scanning SR %s"), sr_ref) task = session.call_xenapi('Async.SR.scan', sr_ref) - session.wait_for_task(instance_id, task) + session.wait_for_task(task, instance_id) @classmethod def scan_default_sr(cls, session): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 60ce51f4a..01bfa2dc5 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -233,7 +233,7 @@ class VMOps(object): "start") def snapshot(self, instance, image_id): - """ Create snapshot from a running VM instance + """Create snapshot from a running VM instance :param instance: instance to be snapshotted :param image_id: id of image to upload to @@ -285,7 +285,7 @@ class VMOps(object): return def migrate_disk_and_power_off(self, instance, dest): - """ Copies a VHD from one host machine to another + """Copies a VHD from one host machine to another :param instance: the instance that owns the VHD in question :param dest: the destination host machine @@ -314,7 +314,7 @@ class VMOps(object): task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) - self._session.wait_for_task(instance.id, task) + self._session.wait_for_task(task, instance.id) # Now power down the instance and transfer the COW VHD self._shutdown(instance, vm_ref, method='clean') @@ -326,7 +326,7 @@ class VMOps(object): task = self._session.async_call_plugin('migration', 'transfer_vhd', {'params': pickle.dumps(params)}) - self._session.wait_for_task(instance.id, task) + self._session.wait_for_task(task, instance.id) finally: if template_vm_ref: @@ -338,6 +338,7 @@ class VMOps(object): return {'base_copy': base_copy_uuid, 'cow': cow_uuid} def attach_disk(self, instance, disk_info): + """Links the base copy VHD to the COW via the XAPI plugin""" vm_ref = VMHelper.lookup(self._session, instance.name) new_base_copy_uuid = str(uuid.uuid4()) new_cow_uuid = str(uuid.uuid4()) @@ -350,7 +351,7 @@ class VMOps(object): task = self._session.async_call_plugin('migration', 'move_vhds_into_sr', {'params': pickle.dumps(params)}) - self._session.wait_for_task(instance.id, task) + self._session.wait_for_task(task, instance.id) # Now we rescan the SR so we find the VHDs VMHelper.scan_default_sr(self._session) -- cgit From 68d894be2ec3b4eaa14dc5c90143f45f7db1e4b8 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Fri, 4 Mar 2011 17:48:28 +0000 Subject: * Tests to verify correct vm-params for Windows and Linux instances --- nova/compute/api.py | 5 +- .../versions/007_add_os_type_to_instances.py | 4 +- nova/tests/db/fakes.py | 1 + nova/tests/test_xenapi.py | 99 ++++++++++++++++++---- nova/virt/xenapi/vm_utils.py | 14 +-- nova/virt/xenapi/vmops.py | 8 +- 6 files changed, 101 insertions(+), 30 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 8bdf712a0..d79371e94 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -125,7 +125,10 @@ class API(base.Base): raise quota.QuotaError(msg, "MetadataLimitExceeded") image = self.image_service.show(context, image_id) - os_type = image['properties'].get('os_type', 'linux') + + os_type = None + if 'properties' in image and 'os_type' in image['properties']: + os_type = image['properties']['os_type'] if kernel_id is None: kernel_id = image.get('kernel_id', None) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py index 21f21b040..d6d964b95 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py @@ -34,7 +34,7 @@ instances_os_type = Column('os_type', String(length=255, convert_unicode=False, assert_unicode=None, unicode_error=None, _warn_on_bytestring=False), - nullable=False) + nullable=True) def upgrade(migrate_engine): @@ -43,5 +43,3 @@ def upgrade(migrate_engine): meta.bind = migrate_engine instances.create_column(instances_os_type) - - diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py index 05bdd172e..facd6efae 100644 --- a/nova/tests/db/fakes.py +++ b/nova/tests/db/fakes.py @@ -62,6 +62,7 @@ def stub_out_db_instance_api(stubs): 'mac_address': values['mac_address'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], + 'os_type': values['os_type'] } return FakeModel(base_options) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index b9bb6d5b4..24a5698e5 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -18,6 +18,7 @@ Test suite for XenAPI """ +import functools import stubout from nova import db @@ -41,6 +42,21 @@ from nova.tests.glance import stubs as glance_stubs FLAGS = flags.FLAGS +def stub_vm_utils_with_vdi_attached_here(function, should_return=True): + """ + vm_utils.with_vdi_attached_here needs to be stubbed out because it + calls down to the filesystem to attach a vdi. This provides a + decorator to handle that. + """ + @functools.wraps(function) + def decorated_function(self, *args, **kwargs): + orig_with_vdi_attached_here = vm_utils.with_vdi_attached_here + vm_utils.with_vdi_attached_here = lambda *x: should_return + function(self, *args, **kwargs) + vm_utils.with_vdi_attached_here = orig_with_vdi_attached_here + return decorated_function + + class XenAPIVolumeTestCase(test.TestCase): """ Unit tests for Volume operations @@ -62,6 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux' } def _create_volume(self, size='0'): @@ -219,7 +236,7 @@ class XenAPIVMTestCase(test.TestCase): check() - def check_vm_record(self, conn): + def create_vm_record(self, conn, os_type): instances = conn.list_instances() self.assertEquals(instances, [1]) @@ -231,28 +248,63 @@ class XenAPIVMTestCase(test.TestCase): in xenapi_fake.get_all_records('VM').iteritems() if not rec['is_control_domain']] vm = vms[0] + self.vm_info = vm_info + self.vm = vm + def check_vm_record(self): # Check that m1.large above turned into the right thing. instance_type = instance_types.INSTANCE_TYPES['m1.large'] mem_kib = long(instance_type['memory_mb']) << 10 mem_bytes = str(mem_kib << 10) vcpus = instance_type['vcpus'] - self.assertEquals(vm_info['max_mem'], mem_kib) - self.assertEquals(vm_info['mem'], mem_kib) - self.assertEquals(vm['memory_static_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_min'], mem_bytes) - self.assertEquals(vm['VCPUs_max'], str(vcpus)) - self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + self.assertEquals(self.vm_info['max_mem'], mem_kib) + self.assertEquals(self.vm_info['mem'], mem_kib) + self.assertEquals(self.vm['memory_static_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(self.vm['VCPUs_max'], str(vcpus)) + self.assertEquals(self.vm['VCPUs_at_startup'], str(vcpus)) # Check that the VM is running according to Nova - self.assertEquals(vm_info['state'], power_state.RUNNING) + self.assertEquals(self.vm_info['state'], power_state.RUNNING) # Check that the VM is running according to XenAPI. - self.assertEquals(vm['power_state'], 'Running') + self.assertEquals(self.vm['power_state'], 'Running') + + def check_vm_params_for_windows(self): + self.assertEquals(self.vm['platform']['nx'], 'true') + self.assertEquals(self.vm['HVM_boot_params'], {'order': 'dc'}) + self.assertEquals(self.vm['HVM_boot_policy'], 'BIOS order') + + # check that these are not set + self.assertEquals(self.vm['PV_args'], '') + self.assertEquals(self.vm['PV_bootloader'], '') + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + + def check_vm_params_for_linux(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies') + self.assertEquals(self.vm['PV_bootloader'], 'pygrub') + + # check that these are not set + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') + + def check_vm_params_for_linux_with_external_kernel(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'root=/dev/xvda1') + self.assertNotEquals(self.vm['PV_kernel'], '') + self.assertNotEquals(self.vm['PV_ramdisk'], '') + + # check that these are not set + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') def _test_spawn(self, image_id, kernel_id, ramdisk_id, - instance_type="m1.large"): + instance_type="m1.large", os_type="linux"): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) values = {'name': 1, 'id': 1, @@ -263,11 +315,13 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type': instance_type, 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': os_type } conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) conn.spawn(instance) - self.check_vm_record(conn) + self.create_vm_record(conn, os_type) + self.check_vm_record() def test_spawn_not_enough_memory(self): FLAGS.xenapi_image_service = 'glance' @@ -283,24 +337,37 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_image_service = 'objectstore' self._test_spawn(1, 2, 3) + @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) + self.check_vm_params_for_linux() + + def test_spawn_vhd_glance_linux(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="linux") + self.check_vm_params_for_linux() - def test_spawn_vhd_glance(self): + def test_spawn_vhd_glance_windows(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="windows") + self.check_vm_params_for_windows() def test_spawn_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK) + self.check_vm_params_for_linux_with_external_kernel() def tearDown(self): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + self.vm_info = None + self.vm = None self.stubs.UnsetAll() def _create_instance(self): @@ -314,7 +381,8 @@ class XenAPIVMTestCase(test.TestCase): 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff'} + 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux'} instance = db.instance_create(values) self.conn.spawn(instance) return instance @@ -360,6 +428,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance = FakeInstance() self.fake_instance.id = 42 + self.fake_instance.os_type = 'linux' def assert_disk_type(self, disk_type): dt = vm_utils.VMHelper.determine_disk_image_type( diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 9c0bb5579..a26e391df 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -80,7 +80,8 @@ class VMHelper(HelperBase): """ @classmethod - def create_vm(cls, session, instance, kernel, ramdisk, use_pv_kernel=False): + def create_vm(cls, session, instance, kernel, ramdisk, + use_pv_kernel=False): """Create a VM record. Returns a Deferred that gives the new VM reference. the use_pv_kernel flag indicates whether the guest is HVM or PV @@ -319,7 +320,7 @@ class VMHelper(HelperBase): 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, 'sr_path': get_sr_path(session), - 'os_type': instance.get('os_type', 'linux')} + 'os_type': instance.os_type} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) @@ -524,7 +525,7 @@ class VMHelper(HelperBase): Determine whether the VM will use a paravirtualized kernel or if it will use hardware virtualization. - 1. Objectstore (any image type): + 1. Objectstore (any image type): We use plugin to figure out whether the VDI uses PV 2. Glance (VHD): then we use `os_type`, raise if not set @@ -540,7 +541,8 @@ class VMHelper(HelperBase): session, vdi_ref, disk_image_type, os_type) else: # 1. Objecstore - return cls._determine_is_pv_objectstore(session, instance_id, vdi_ref) + return cls._determine_is_pv_objectstore(session, instance_id, + vdi_ref) @classmethod def _determine_is_pv_objectstore(cls, session, instance_id, vdi_ref): @@ -564,7 +566,7 @@ class VMHelper(HelperBase): """ For a Glance image, determine if we need paravirtualization. - The relevant scenarios are: + The relevant scenarios are: 2. Glance (VHD): then we use `os_type`, raise if not set 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is @@ -582,7 +584,7 @@ class VMHelper(HelperBase): is_pv = True elif disk_image_type == ImageType.DISK_RAW: # 3. RAW - is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) elif disk_image_type == ImageType.DISK: # 4. Disk is_pv = True diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 1edf39c5b..eedb07a50 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -87,8 +87,6 @@ class VMOps(object): vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - os_type = instance.get('os_type', 'linux') - kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, @@ -99,8 +97,8 @@ class VMOps(object): ramdisk = VMHelper.fetch_image(self._session, instance.id, instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) - use_pv_kernel = VMHelper.determine_is_pv( - self._session, instance.id, vdi_ref, disk_image_type, os_type) + use_pv_kernel = VMHelper.determine_is_pv(self._session, instance.id, + vdi_ref, disk_image_type, instance.os_type) vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, use_pv_kernel) @@ -242,7 +240,7 @@ class VMOps(object): finally: self._destroy(instance, template_vm_ref, shutdown=False, destroy_kernel_ramdisk=False) - + logging.debug(_("Finished snapshot and upload for VM %s"), instance) def reboot(self, instance): -- cgit From f3c1c99ca0f6f3164430b33f46772ef8bdc87b70 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 19:54:19 +0000 Subject: move the id wrapping into cloud layer instead of image_service --- nova/api/ec2/cloud.py | 33 ++++++++++++++++++++++++--------- nova/image/s3.py | 28 +++++----------------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index aa1dcbe33..3ea3fa07e 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -843,10 +843,19 @@ class CloudController(object): self.compute_api.update(context, instance_id=instance_id, **kwargs) return True + _type_prefix_map = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + + def _image_ec2_id(self, image_id, image_type): + prefix = self._type_prefix_map[image_type] + template = prefix + '-%08x' + return ec2utils.id_to_ec2_id(int(image_id), template=template) + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - i['imageId'] = image.get('id') + i['imageId'] = self._image_ec2_id(image.get('id'), image.get('type')) i['kernelId'] = image['properties'].get('kernel_id') i['ramdiskId'] = image['properties'].get('ramdisk_id') i['imageOwnerId'] = image['properties'].get('owner_id') @@ -863,7 +872,8 @@ class CloudController(object): images = [] for ec2_id in image_id: try: - image = self.image_service.show(context, ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) + image = self.image_service.show(context, internal_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % ec2_id) @@ -875,14 +885,16 @@ class CloudController(object): def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - self.image_service.delete(context, image_id) + internal_id = ec2utils.ec2_id_to_id(image_id) + self.image_service.delete(context, internal_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - image = {"image_location": image_location} - image_id = self.image_service.create(context, image) + metadata = {"image_location": image_location} + image = self.image_service.create(context, metadata) + image_id = self._image_ec2_id(image['id'], image['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -893,8 +905,9 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: + internal_id = ec2utils.ec2_id_to_id(image_id) image = self._format_image(self.image_service.show(context, - image_id)) + internal_id)) except (IndexError, exception.NotFound): raise exception.NotFound(_('Image %s not found') % image_id) result = {'image_id': image_id, 'launchPermission': []} @@ -917,13 +930,15 @@ class CloudController(object): LOG.audit(_("Updating image %s publicity"), image_id, context=context) try: - metadata = self.image_service.show(context, image_id) + internal_id = ec2utils.ec2_id_to_id(image_id) + metadata = self.image_service.show(context, internal_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % image_id) del(metadata['id']) metadata['properties']['is_public'] = (operation_type == 'add') - return self.image_service.update(context, image_id, metadata) + return self.image_service.update(context, internal_id, metadata) def update_image(self, context, image_id, **kwargs): - result = self.image_service.update(context, image_id, dict(kwargs)) + internal_id = ec2utils.ec2_id_to_id(image_id) + result = self.image_service.update(context, internal_id, dict(kwargs)) return result diff --git a/nova/image/s3.py b/nova/image/s3.py index e9542c7bd..c7446f4b0 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -36,7 +36,6 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service -from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS @@ -44,17 +43,6 @@ flags.DEFINE_string('image_decryption_dir', '/tmp', 'parent dir for tempdir used for image decryption') -_type_prefix_map = {'machine': 'ami', - 'kernel': 'aki', - 'ramdisk': 'ari'} - - -def image_ec2_id(image_id, image_type): - prefix = _type_prefix_map[image_type] - template = prefix + '-%08x' - return ec2utils.id_to_ec2_id(int(image_id), template=template) - - class S3ImageService(service.BaseImageService): def __init__(self, service=None, *args, **kwargs): if service == None: @@ -62,23 +50,20 @@ class S3ImageService(service.BaseImageService): self.service = service self.service.__init__(*args, **kwargs) - def create(self, context, properties, data=None): - """image should contain image_location""" - image_id, metadata = self._s3_create(context, properties) - return image_ec2_id(image_id, metadata['type']) + def create(self, context, metadata, data=None): + """metadata should contain image_location""" + image = self._s3_create(context, metadata) + return image def delete(self, context, image_id): # FIXME(vish): call to show is to check filter self.show(context, image_id) - image_id = ec2utils.ec2_id_to_id(image_id) self.service.delete(context, image_id) def update(self, context, image_id, metadata, data=None): # FIXME(vish): call to show is to check filter self.show(context, image_id) - image_id = ec2utils.ec2_id_to_id(image_id) image = self.service.update(context, image_id, metadata, data) - image['id'] = image_ec2_id(image['id'], image['type']) return image def index(self, context): @@ -103,16 +88,13 @@ class S3ImageService(service.BaseImageService): for image in images: if not S3ImageService._is_visible(context, image): continue - image['id'] = image_ec2_id(image['id'], image['type']) filtered.append(image) return filtered def show(self, context, image_id): - image_id = ec2utils.ec2_id_to_id(image_id) image = self.service.show(context, image_id) if not self._is_visible(context, image): raise exception.NotFound - image['id'] = image_ec2_id(image['id'], image['type']) return image @staticmethod @@ -233,7 +215,7 @@ class S3ImageService(service.BaseImageService): eventlet.spawn_n(delayed_create) - return image_id, metadata + return image @staticmethod def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, -- cgit From 1eed366b7508c0f225b2c9691e1f62a6f88ee3f8 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Fri, 4 Mar 2011 21:07:03 +0100 Subject: Added initial support to delete networks nova-manage --- bin/nova-manage | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index 9bf3a1bb3..9557f2423 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -546,6 +546,16 @@ class NetworkCommands(object): network.dns) + def delete(self, fixed_range): + """Deletes a network""" + try: + network = [n for n in db.network_get_all(context.get_admin_context()) + if n.cidr == fixed_range][0] + + print network.id, network.cidr, network.project_id + except IndexError: + raise ValueError(_("Network does not exist")) + class ServiceCommands(object): """Enable and disable running services""" -- cgit From e63cd9d5dc856f81477cf6c0e6c77ed7d1f4d70c Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Fri, 4 Mar 2011 22:17:53 +0000 Subject: * os_type is no longer `not null` --- .../versions/007_add_os_type_to_instances.py | 45 ----------------- .../versions/009_add_os_type_to_instances.py | 56 ++++++++++++++++++++++ nova/virt/xenapi/vm_utils.py | 8 ++-- 3 files changed, 60 insertions(+), 49 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py deleted file mode 100644 index d6d964b95..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py +++ /dev/null @@ -1,45 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -# FIXME(dubs) should this be not null? Maybe create as nullable, then -# populate all existing rows with 'linux', then adding not null constraint. -instances_os_type = Column('os_type', - String(length=255, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False), - nullable=True) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - - instances.create_column(instances_os_type) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py new file mode 100644 index 000000000..a50f31e16 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py @@ -0,0 +1,56 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from sqlalchemy.sql import text +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# FIXME(dubs) should this be not null? Maybe create as nullable, then +# populate all existing rows with 'linux', then adding not null constraint. +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) + migrate_engine.execute(instances.update()\ + .where(instances.c.os_type==None)\ + .values(os_type='linux')) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances.drop_column('os_type') + diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 7bff81b66..150824400 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -117,7 +117,7 @@ class VMHelper(HelperBase): 'memory_target': mem, 'name_description': '', 'name_label': instance.name, -# 'other_config': {'allowvssprovider': False}, + 'other_config': {'allowvssprovider': False}, 'other_config': {}, 'PCI_bus': '', 'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true', @@ -313,14 +313,14 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) - # TODO(dubs): os_type is currently defaulting to linux, we actually - # want to make this a NOT NULL column and require it to be specified. + os_type = instance.os_type and instance.os_type or 'linux' + params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, 'sr_path': get_sr_path(session), - 'os_type': instance.os_type} + 'os_type': os_type} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) -- cgit From a5bee00af4d6ec3eed6ed0abd866948f4510f041 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 01:25:01 +0000 Subject: make compute get the new images properly, fix a bunch of tests, and provide conversion commands --- bin/nova-manage | 131 ++++++++++++++++++++++++++++++++++++++++++- nova/api/ec2/cloud.py | 74 +++++++++++++++--------- nova/api/ec2/ec2utils.py | 6 +- nova/compute/api.py | 4 +- nova/image/glance.py | 35 ++++++++++-- nova/image/local.py | 52 +++++++++++++---- nova/image/s3.py | 33 +++++++---- nova/image/service.py | 18 ++++-- nova/tests/test_cloud.py | 16 +++--- nova/tests/test_compute.py | 12 ++-- nova/tests/test_console.py | 2 +- nova/tests/test_quota.py | 32 ++++++----- nova/tests/test_scheduler.py | 4 +- nova/tests/test_volume.py | 2 +- nova/virt/images.py | 25 +++++---- nova/virt/libvirt_conn.py | 8 ++- 16 files changed, 345 insertions(+), 109 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 0f7604aeb..ebeda05a0 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -55,6 +55,8 @@ import datetime import gettext +import glob +import json import os import re import sys @@ -81,7 +83,7 @@ from nova import log as logging from nova import quota from nova import rpc from nova import utils -from nova.api.ec2.ec2utils import ec2_id_to_id +from nova.api.ec2 import ec2utils from nova.auth import manager from nova.cloudpipe import pipelib from nova.compute import instance_types @@ -104,7 +106,7 @@ def param2id(object_id): args: [object_id], e.g. 'vol-0000000a' or 'volume-0000000a' or '10' """ if '-' in object_id: - return ec2_id_to_id(object_id) + return ec2utils.ec2_id_to_id(object_id) else: return int(object_id) @@ -735,6 +737,130 @@ class InstanceTypeCommands(object): self._print_instance_types(name, inst_types) +class ImageCommands(object): + """Methods for dealing with a cloud in an odd state""" + + def __init__(self, *args, **kwargs): + self.image_service = utils.import_object(FLAGS.image_service) + + def _register(self, image_type, path, owner, name=None, + is_public='T', architecture='x86_64', + kernel_id=None, ramdisk_id=None): + meta = {'type': image_type, + 'is_public': True, + 'name': name, + 'properties': {'image_state': 'available', + 'owner': owner, + 'architecture': architecture, + 'image_location': 'local', + 'is_public': (is_public == 'T')}} + if kernel_id: + meta['properties']['kernel_id'] = int(kernel_id) + if ramdisk_id: + meta['properties']['ramdisk_id'] = int(ramdisk_id) + elevated = context.get_admin_context() + try: + with open(path) as ifile: + image = self.image_service.create(elevated, meta, ifile) + new = image['id'] + print _("Image registered to %(new)s (%(new)08x).") % locals() + return new + except Exception as exc: + print _("Failed to register %(path)s: %(exc)s") % locals() + + def register_all(self, image, kernel, ramdisk, owner, name=None, + is_public='T', architecture='x86_64'): + """Uploads an image, kernel, and ramdisk into the image_service + arguments: image kernel ramdisk owner [name] [is_public='T'] + [architecture='x86_64']""" + kernel_id = self._register('kernel', kernel, owner, None, + is_public, architecture) + ramdisk_id = self._register('ramdisk', ramdisk, owner, None, + is_public, architecture) + self._register('machine', image, owner, name, is_public, + architecture, kernel_id, ramdisk_id) + + def image_register(self, path, owner, name=None, is_public='T', + architecture='x86_64', kernel_id=None, ramdisk_id=None): + """Uploads an image into the image_service + arguments: path owner [name] [is_public='T'] [architecture='x86_64'] + [kernel_id] [ramdisk_id]""" + self._register('machine', path, owner, name, is_public, + architecture, kernel_id, ramdisk_id) + + def kernel_register(self, path, owner, name=None, is_public='T', + architecture='x86_64'): + """Uploads a kernel into the image_service + arguments: path owner [name] [is_public='T'] [architecture='x86_64'] + """ + self._register('kernel', path, owner, name, is_public, architecture) + + def ramdisk_register(self, path, owner, name=None, is_public='T', + architecture='x86_64'): + """Uploads a ramdisk into the image_service + arguments: path owner [name] [is_public='T'] [architecture='x86_64'] + """ + self._register('ramdisk', path, owner, name, is_public, architecture) + + def _lookup(self, old_image_id): + try: + internal_id = ec2utils.ec2_id_to_id(old_image_id) + image = self.image_service.show(context, internal_id) + except exception.NotFound: + image = self.image_service.show_by_name(context, old_image_id) + return image['id'] + + def _old_to_new(self, old): + new = {'type': old['type'], + 'is_public': True, + 'name': old['imageId'], + 'properties': {'image_state': old['imageState'], + 'owner': old['imageOwnerId'], + 'architecture': old['architecture'], + 'image_location': old['imageLocation'], + 'is_public': old['isPublic']}} + if old.get('kernelId'): + new['properties']['kernel_id'] = self._lookup(old['kernelId']) + if old.get('ramdiskId'): + new['properties']['ramdisk_id'] = self._lookup(old['ramdiskId']) + return new + + def _convert_images(self, images): + elevated = context.get_admin_context() + for image_path, image_metadata in images.iteritems(): + meta = self._old_to_new(image_metadata) + old = meta['name'] + try: + with open(image_path) as ifile: + image = self.image_service.create(elevated, meta, ifile) + new = image['id'] + print _("Image %(old)s converted to " \ + "%(new)s (%(new)08x).") % locals() + except Exception as exc: + print _("Failed to convert %(old)s: %(exc)s") % locals() + + + def convert(self, directory): + """Uploads old objectstore images in directory to new service + arguments: directory""" + machine_images = {} + other_images = {} + for fn in glob.glob("%s/*/info.json" % directory): + try: + image_path = os.path.join(fn.rpartition('/')[0], 'image') + with open(fn) as metadata_file: + image_metadata = json.load(metadata_file) + if image_metadata['type'] == 'machine': + machine_images[image_path] = image_metadata + else: + other_images[image_path] = image_metadata + except Exception as exc: + print _("Failed to load %(fn)s.") % locals() + # NOTE(vish): do kernels and ramdisks first so images + self._convert_images(other_images) + self._convert_images(machine_images) + + CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), @@ -749,6 +875,7 @@ CATEGORIES = [ ('db', DbCommands), ('volume', VolumeCommands), ('instance_type', InstanceTypeCommands), + ('image', ImageCommands), ('flavor', InstanceTypeCommands)] diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 3ea3fa07e..496e944fe 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -146,10 +146,13 @@ class CloudController(object): floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) + image_ec2_id = self._image_ec2_id(instance_ref['image_id'], 'machine') + k_ec2_id = self._image_ec2_id(instance_ref['kernel_id'], 'kernel') + r_ec2_id = self._image_ec2_id(instance_ref['ramdisk_id'], 'ramdisk') data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { - 'ami-id': instance_ref['image_id'], + 'ami-id': image_ec2_id, 'ami-launch-index': instance_ref['launch_index'], 'ami-manifest-path': 'FIXME', 'block-device-mapping': { @@ -164,12 +167,12 @@ class CloudController(object): 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, 'local-ipv4': address, - 'kernel-id': instance_ref['kernel_id'], + 'kernel-id': k_ec2_id, + 'ramdisk-id': r_ec2_id, 'placement': {'availability-zone': availability_zone}, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', 'public-keys': keys, - 'ramdisk-id': instance_ref['ramdisk_id'], 'reservation-id': instance_ref['reservation_id'], 'security-groups': '', 'mpi': mpi}} @@ -679,7 +682,7 @@ class CloudController(object): instance_id = instance['id'] ec2_id = ec2utils.id_to_ec2_id(instance_id) i['instanceId'] = ec2_id - i['imageId'] = instance['image_id'] + i['imageId'] = self._image_ec2_id(instance['image_id']) i['instanceState'] = { 'code': instance['state'], 'name': instance['state_description']} @@ -782,13 +785,15 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) if kwargs.get('kernel_id'): - kwargs['kernel_id'] = ec2utils.ec2_id_to_id(kwargs['kernel_id']) + kernel = self._get_image(context, kwargs['kernel_id']) + kwargs['kernel_id'] = kernel['id'] if kwargs.get('ramdisk_id'): - kwargs['ramdisk_id'] = ec2utils.ec2_id_to_id(kwargs['ramdisk_id']) + ramdisk = self._get_image(context, kwargs['ramdisk_id']) + kwargs['ramdisk_id'] = ramdisk['id'] instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id=ec2utils.ec2_id_to_id(kwargs['image_id']), + image_id=self._get_image(context, kwargs['image_id'])['id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), @@ -847,17 +852,34 @@ class CloudController(object): 'kernel': 'aki', 'ramdisk': 'ari'} - def _image_ec2_id(self, image_id, image_type): + def _image_ec2_id(self, image_id, image_type='machine'): prefix = self._type_prefix_map[image_type] template = prefix + '-%08x' return ec2utils.id_to_ec2_id(int(image_id), template=template) + def _get_image(self, context, ec2_id): + try: + internal_id = ec2utils.ec2_id_to_id(ec2_id) + return self.image_service.show(context, internal_id) + except exception.NotFound: + return self.image_service.show_by_name(context, ec2_id) + + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - i['imageId'] = self._image_ec2_id(image.get('id'), image.get('type')) - i['kernelId'] = image['properties'].get('kernel_id') - i['ramdiskId'] = image['properties'].get('ramdisk_id') + ec2_id = self._image_ec2_id(image.get('id'), image.get('type')) + name = image.get('name') + if name: + i['imageId'] = "%s (%s)" % (ec2_id, name) + else: + i['imageId'] = ec2_id + kernel_id = image['properties'].get('kernel_id') + if kernel_id: + i['kernelId'] = self._image_ec2_id(kernel_id, 'kernel') + ramdisk_id = image['properties'].get('ramdisk_id') + if ramdisk_id: + i['ramdiskId'] = self._image_ec2_id(ramdisk_id, 'ramdisk') i['imageOwnerId'] = image['properties'].get('owner_id') i['imageLocation'] = image['properties'].get('image_location') i['imageState'] = image['properties'].get('image_state') @@ -872,8 +894,7 @@ class CloudController(object): images = [] for ec2_id in image_id: try: - internal_id = ec2utils.ec2_id_to_id(ec2_id) - image = self.image_service.show(context, internal_id) + image = self._get_image(context, ec2_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % ec2_id) @@ -885,16 +906,17 @@ class CloudController(object): def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - internal_id = ec2utils.ec2_id_to_id(image_id) + image = self._get_image(context, image_id) + internal_id = image['id'] self.image_service.delete(context, internal_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - metadata = {"image_location": image_location} + metadata = {'properties': {'image_location': image_location}} image = self.image_service.create(context, metadata) - image_id = self._image_ec2_id(image['id'], image['type']) + image_id = self._image_ec2_id(image['id'], image['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -905,13 +927,11 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: - internal_id = ec2utils.ec2_id_to_id(image_id) - image = self._format_image(self.image_service.show(context, - internal_id)) - except (IndexError, exception.NotFound): + image = self._get_image(context, image_id) + except exception.NotFound: raise exception.NotFound(_('Image %s not found') % image_id) - result = {'image_id': image_id, 'launchPermission': []} - if image['isPublic']: + result = {'imageId': image_id, 'launchPermission': []} + if image['properties']['is_public']: result['launchPermission'].append({'group': 'all'}) return result @@ -930,13 +950,13 @@ class CloudController(object): LOG.audit(_("Updating image %s publicity"), image_id, context=context) try: - internal_id = ec2utils.ec2_id_to_id(image_id) - metadata = self.image_service.show(context, internal_id) + image = self._get_image(context, image_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % image_id) - del(metadata['id']) - metadata['properties']['is_public'] = (operation_type == 'add') - return self.image_service.update(context, internal_id, metadata) + internal_id = image['id'] + del(image['id']) + image['properties']['is_public'] = (operation_type == 'add') + return self.image_service.update(context, internal_id, image) def update_image(self, context, image_id, **kwargs): internal_id = ec2utils.ec2_id_to_id(image_id) diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index 0ea22c0e6..e4df80cf8 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -16,10 +16,14 @@ # License for the specific language governing permissions and limitations # under the License. +from nova import exception def ec2_id_to_id(ec2_id): """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" - return int(ec2_id.split('-')[-1], 16) + try: + return int(ec2_id.split('-')[-1], 16) + except ValueError: + raise exception.NotFound(_("Id %s Not Found") % ec2_id) def id_to_ec2_id(instance_id, template='i-%08x'): diff --git a/nova/compute/api.py b/nova/compute/api.py index 35a7d7bc0..58118121a 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -126,9 +126,9 @@ class API(base.Base): image = self.image_service.show(context, image_id) if kernel_id is None: - kernel_id = image.get('kernel_id', None) + kernel_id = image['properties'].get('kernel_id', None) if ramdisk_id is None: - ramdisk_id = image.get('ramdisk_id', None) + ramdisk_id = image['properties'].get('ramdisk_id', None) # FIXME(sirp): is there a way we can remove null_kernel? # No kernel and ramdisk for raw images if kernel_id == str(FLAGS.null_kernel): diff --git a/nova/image/glance.py b/nova/image/glance.py index 7db94c0d4..fb383f5e6 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -17,9 +17,6 @@ """Implementation of an image service that uses Glance as the backend""" from __future__ import absolute_import -import httplib -import json -import urlparse from glance.common import exception as glance_exception @@ -55,16 +52,44 @@ class GlanceImageService(service.BaseImageService): """ return self.client.get_images_detailed() - def show(self, context, id): + def show(self, context, image_id): """ Returns a dict containing image data for the given opaque image id. """ try: - image = self.client.get_image_meta(id) + image = self.client.get_image_meta(image_id) except glance_exception.NotFound: raise exception.NotFound return image + def show_by_name(self, context, name): + """ + Returns a dict containing image data for the given name. + """ + # TODO(vish): replace this with more efficient call when glance + # supports it. + images = self.detail(context) + image = None + for cantidate in images: + if name == cantidate.get('name'): + image = cantidate + break + if image == None: + raise exception.NotFound + return image + + def get(self, context, image_id, data): + """ + Calls out to Glance for metadata and data and writes data. + """ + try: + metadata, image_chunks = self.client.get_image(image_id) + except glance_exception.NotFound: + raise exception.NotFound + for chunk in image_chunks: + data.write(chunk) + return metadata + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. diff --git a/nova/image/local.py b/nova/image/local.py index 6fa648b6b..c4ac3baaa 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -42,13 +42,12 @@ class LocalImageService(service.BaseImageService): def _path_to(self, image_id, fname='info.json'): if fname: - return os.path.join(self._path, str(image_id), fname) - return os.path.join(self._path, str(image_id)) + return os.path.join(self._path, '%08x' % int(image_id), fname) + return os.path.join(self._path, '%08x' % int(image_id)) def _ids(self): """The list of all image ids.""" - return [int(i) for i in os.listdir(self._path) - if unicode(i).isnumeric()] + return [int(i, 16) for i in os.listdir(self._path)] def index(self, context): return [dict(image_id=i['id'], name=i.get('name')) @@ -68,27 +67,56 @@ class LocalImageService(service.BaseImageService): try: with open(self._path_to(image_id)) as metadata_file: return json.load(metadata_file) - except IOError: + except (IOError, ValueError): raise exception.NotFound + def show_by_name(self, context, name): + """Returns a dict containing image data for the given name.""" + # NOTE(vish): Not very efficient, but the local image service + # is for testing so it should be fine. + images = self.detail(context) + image = None + for cantidate in images: + if name == cantidate.get('name'): + image = cantidate + break + if image == None: + raise exception.NotFound + return image + + def get(self, context, image_id, data): + """Get image and metadata.""" + try: + with open(self._path_to(image_id)) as metadata_file: + metadata = json.load(metadata_file) + with open(self._path_to(image_id, 'image')) as image_file: + shutil.copyfileobj(image_file, data) + except (IOError, ValueError): + raise exception.NotFound + return metadata + def create(self, context, metadata, data=None): - """Store the image data and return the new image id.""" + """Store the image data and return the new image.""" image_id = random.randint(0, 2 ** 31 - 1) image_path = self._path_to(image_id, None) if not os.path.exists(image_path): os.mkdir(image_path) - return self.update(context, image_id, metadata, data) + return self.update(context, image_id, metadata, data) def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data.""" metadata['id'] = image_id try: - with open(self._path_to(image_id), 'w') as metadata_file: - json.dump(metadata, metadata_file) if data: - with open(self._path_to(image_id, 'image'), 'w') as image_file: + location = self._path_to(image_id, 'image') + with open(location, 'w') as image_file: shutil.copyfileobj(data, image_file) - except IOError: + # NOTE(vish): update metadata similarly to glance + metadata['status'] = 'active' + metadata['location'] = location + with open(self._path_to(image_id), 'w') as metadata_file: + json.dump(metadata, metadata_file) + except (IOError, ValueError): raise exception.NotFound return metadata @@ -99,7 +127,7 @@ class LocalImageService(service.BaseImageService): """ try: shutil.rmtree(self._path_to(image_id, None)) - except IOError: + except (IOError, ValueError): raise exception.NotFound def delete_all(self): diff --git a/nova/image/s3.py b/nova/image/s3.py index c7446f4b0..ab6eea8cf 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -36,6 +36,7 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service +from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS @@ -51,7 +52,7 @@ class S3ImageService(service.BaseImageService): self.service.__init__(*args, **kwargs) def create(self, context, metadata, data=None): - """metadata should contain image_location""" + """metadata['properties'] should contain image_location""" image = self._s3_create(context, metadata) return image @@ -97,6 +98,12 @@ class S3ImageService(service.BaseImageService): raise exception.NotFound return image + def show_by_name(self, context, name): + image = self.service.show_by_name(context, name) + if not self._is_visible(context, image): + raise exception.NotFound + return image + @staticmethod def _conn(context): # TODO(vish): is there a better way to get creds to sign @@ -119,10 +126,12 @@ class S3ImageService(service.BaseImageService): key.get_contents_to_filename(local_filename) return local_filename - def _s3_create(self, context, properties): + def _s3_create(self, context, metadata): + """Gets a manifext from s3 and makes an image""" + image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir) - image_location = properties['image_location'] + image_location = metadata['properties']['image_location'] bucket_name = image_location.split("/")[0] manifest_path = image_location[len(bucket_name) + 1:] bucket = self._conn(context).get_bucket(bucket_name) @@ -153,25 +162,27 @@ class S3ImageService(service.BaseImageService): except: arch = 'x86_64' - properties.update({'owner_id': context.project_id, - 'architecture': arch}) + properties = metadata['properties'] + properties['owner_id'] = context.project_id + properties['architecture'] = arch if kernel_id: - properties['kernel_id'] = kernel_id + properties['kernel_id'] = ec2utils.ec2_id_to_id(kernel_id) if ramdisk_id: - properties['ramdisk_id'] = ramdisk_id + properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id) properties['is_public'] = False - metadata = {'type': image_type, - 'status': 'queued', - 'is_public': True, - 'properties': properties} + metadata.update({'type': image_type, + 'status': 'queued', + 'is_public': True, + 'properties': properties}) metadata['properties']['image_state'] = 'pending' image = self.service.create(context, metadata) image_id = image['id'] def delayed_create(): + """This handles the fetching and decrypting of the part files.""" parts = [] for fn_element in manifest.find("image").getiterator("filename"): part = self._download_file(bucket, fn_element.text, image_path) diff --git a/nova/image/service.py b/nova/image/service.py index e429955f4..c09052cab 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -56,9 +56,9 @@ class BaseImageService(object): """ raise NotImplementedError - def show(self, context, id): + def show(self, context, image_id): """ - Returns a dict containing image data for the given opaque image id. + Returns a dict containing image metadata for the given opaque image id. :retval a mapping with the following signature: @@ -76,9 +76,19 @@ class BaseImageService(object): """ raise NotImplementedError + def get(self, context, data): + """ + Returns a dict containing image metadata and writes image data to data. + + :param data: a file-like object to hold binary image data + + :raises NotFound if the image does not exist + """ + raise NotImplementedError + def create(self, context, metadata, data=None): """ - Store the image data and return the new image id. + Store the image metadata and data and return the new image id. :raises AlreadyExists if the image already exist. @@ -86,7 +96,7 @@ class BaseImageService(object): raise NotImplementedError def update(self, context, image_id, metadata, data=None): - """Replace the contents of the given image with the new data. + """Update the given image with the new metadata and data. :raises NotFound if the image does not exist. diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 7d7b91658..8d2cd9573 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -78,10 +78,11 @@ class CloudTestCase(test.TestCase): project=self.project) host = self.network.get_network_host(self.context.elevated()) - def fake_image_show(meh, context, id): - return dict(kernelId=1, ramdiskId=1) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} - self.stubs.Set(local.LocalImageService, 'show', fake_image_show) + self.stubs.Set(local.LocalImageService, 'show', fake_show) + self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) def tearDown(self): network_ref = db.project_get_network(self.context, @@ -195,8 +196,10 @@ class CloudTestCase(test.TestCase): def test_describe_instances(self): """Makes sure describe_instances works and filters results.""" inst1 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host1'}) inst2 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host2'}) comp1 = db.service_create(self.context, {'host': 'host1', 'availability_zone': 'zone1', @@ -222,11 +225,9 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, comp2['id']) def test_console_output(self): - image_id = FLAGS.default_image - print image_id instance_type = FLAGS.default_instance_type max_count = 1 - kwargs = {'image_id': image_id, + kwargs = {'image_id': 'ami-1', 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) @@ -242,8 +243,7 @@ class CloudTestCase(test.TestCase): greenthread.sleep(0.3) def test_ajax_console(self): - image_id = FLAGS.default_image - kwargs = {'image_id': image_id} + kwargs = {'image_id': 'ami-1'} rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] greenthread.sleep(0.3) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 1f49baaf6..8c18fafc6 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -47,16 +47,16 @@ class ComputeTestCase(test.TestCase): network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute.API() - - def fake_image_show(meh, context, id): - return dict(kernelId=1, ramdiskId=1) - - self.stubs.Set(local.LocalImageService, 'show', fake_image_show) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext('fake', 'fake', False) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} + + self.stubs.Set(local.LocalImageService, 'show', fake_show) + def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) @@ -65,7 +65,7 @@ class ComputeTestCase(test.TestCase): def _create_instance(self): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 49ff24413..d47c70d88 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -57,7 +57,7 @@ class ConsoleTestCase(test.TestCase): inst = {} #inst['host'] = self.host #inst['name'] = 'instance-1234' - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index ca8abdb36..45b544753 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -20,11 +20,12 @@ from nova import compute from nova import context from nova import db from nova import flags +from nova import network from nova import quota from nova import test from nova import utils +from nova import volume from nova.auth import manager -from nova.api.ec2 import cloud from nova.compute import instance_types @@ -41,7 +42,6 @@ class QuotaTestCase(test.TestCase): quota_gigabytes=20, quota_floating_ips=1) - self.cloud = cloud.CloudController() self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('admin', 'admin', 'admin') @@ -57,7 +57,7 @@ class QuotaTestCase(test.TestCase): def _create_instance(self, cores=2): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-1' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -118,12 +118,12 @@ class QuotaTestCase(test.TestCase): for i in range(FLAGS.quota_instances): instance_id = self._create_instance() instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, instance_type='m1.small', - image_id='ami-1') + image_id=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -131,12 +131,12 @@ class QuotaTestCase(test.TestCase): instance_ids = [] instance_id = self._create_instance(cores=4) instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, instance_type='m1.small', - image_id='ami-1') + image_id=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -145,9 +145,12 @@ class QuotaTestCase(test.TestCase): for i in range(FLAGS.quota_volumes): volume_id = self._create_volume() volume_ids.append(volume_id) - self.assertRaises(quota.QuotaError, self.cloud.create_volume, - self.context, - size=10) + self.assertRaises(quota.QuotaError, + volume.API().create, + self.context, + size=10, + name='', + description='') for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -156,9 +159,11 @@ class QuotaTestCase(test.TestCase): volume_id = self._create_volume(size=20) volume_ids.append(volume_id) self.assertRaises(quota.QuotaError, - self.cloud.create_volume, + volume.API().create, self.context, - size=10) + size=10, + name='', + description='') for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -172,7 +177,8 @@ class QuotaTestCase(test.TestCase): # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertRaises(quota.QuotaError, self.cloud.allocate_address, + self.assertRaises(quota.QuotaError, + network.API().allocate_floating_ip, self.context) db.floating_ip_destroy(context.get_admin_context(), address) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index b6888c4d2..bb279ac4b 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -155,7 +155,7 @@ class SimpleDriverTestCase(test.TestCase): def _create_instance(self, **kwargs): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -169,8 +169,6 @@ class SimpleDriverTestCase(test.TestCase): def _create_volume(self): """Create a test volume""" vol = {} - vol['image_id'] = 'ami-test' - vol['reservation_id'] = 'r-fakeres' vol['size'] = 1 vol['availability_zone'] = 'test' return db.volume_create(self.context, vol)['id'] diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index b40ca004b..f698c85b5 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -99,7 +99,7 @@ class VolumeTestCase(test.TestCase): def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance.""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = 'fake' diff --git a/nova/virt/images.py b/nova/virt/images.py index 7a6fef330..06b3a8ecd 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -28,29 +28,32 @@ import time import urllib2 import urlparse +from nova import context from nova import flags from nova import log as logging from nova import utils from nova.auth import manager from nova.auth import signer -from nova.objectstore import image FLAGS = flags.FLAGS -flags.DEFINE_bool('use_s3', True, - 'whether to get images from s3 or use local copy') - LOG = logging.getLogger('nova.virt.images') -def fetch(image, path, user, project): - if FLAGS.use_s3: - f = _fetch_s3_image - else: - f = _fetch_local_image - return f(image, path, user, project) +def fetch(image_id, path, _user, _project): + # TODO(vish): Improve context handling and add owner and auth data + # when it is added to glance. Right now there is no + # auth checking in glance, so we assume that access was + # checked before we got here. + image_service = utils.import_object(FLAGS.image_service) + with open(path, "wb") as image_file: + elevated = context.get_admin_context() + metadata = image_service.get(elevated, image_id, image_file) + return metadata +# NOTE(vish): The methods below should be unnecessary, but I'm leaving +# them in case the glance client does not work on windows. def _fetch_image_no_curl(url, path, headers): request = urllib2.Request(url) for (k, v) in headers.iteritems(): @@ -110,6 +113,8 @@ def _image_path(path): return os.path.join(FLAGS.images_path, path) +# TODO(vish): xenapi should use the glance client code directly instead +# of retrieving the image using this method. def image_url(image): if FLAGS.image_service == "nova.image.glance.GlanceImageService": return "http://%s:%s/images/%s" % (FLAGS.glance_host, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9f7315c17..02a8208d5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -579,21 +579,23 @@ class LibvirtConnection(object): 'ramdisk_id': inst['ramdisk_id']} if disk_images['kernel_id']: + fname = '%08x' % int(disk_images['kernel_id']) self._cache_image(fn=self._fetch_image, target=basepath('kernel'), - fname=disk_images['kernel_id'], + fname=fname, image_id=disk_images['kernel_id'], user=user, project=project) if disk_images['ramdisk_id']: + fname = '%08x' % int(disk_images['ramdisk_id']) self._cache_image(fn=self._fetch_image, target=basepath('ramdisk'), - fname=disk_images['ramdisk_id'], + fname=fname, image_id=disk_images['ramdisk_id'], user=user, project=project) - root_fname = disk_images['image_id'] + root_fname = '%08x' % int(disk_images['image_id']) size = FLAGS.minimum_root_size if inst['instance_type'] == 'm1.tiny' or suffix == '.rescue': size = None -- cgit From ac681cdddac29b973b107d6ee06f0fc2039a1d7e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 6 Mar 2011 19:21:14 -0800 Subject: zipfile needs to be extracted after nova is running --- contrib/nova.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 1187f2728..028184605 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -166,9 +166,6 @@ NOVA_CONF_EOF $NOVA_DIR/bin/nova-manage user admin admin admin admin # create a project called 'admin' with project manager of 'admin' $NOVA_DIR/bin/nova-manage project create admin admin - # export environment variables for project 'admin' and user 'admin' - $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip - unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ # create a small network $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 1 32 @@ -184,6 +181,11 @@ NOVA_CONF_EOF screen_it scheduler "$NOVA_DIR/bin/nova-scheduler" screen_it volume "$NOVA_DIR/bin/nova-volume" screen_it ajax_console_proxy "$NOVA_DIR/bin/nova-ajax-console-proxy" + + # export environment variables for project 'admin' and user 'admin' + $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip + unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ + screen_it test ". $NOVA_DIR/novarc" screen -S nova -x fi -- cgit From b3d3366b8fd4eaf81bb9e03ad808c1a139e5b5b0 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 7 Mar 2011 12:07:23 -0500 Subject: Generate 'adminPass' and call set_password when creating servers. --- nova/api/openstack/servers.py | 10 +++++++--- nova/tests/api/openstack/test_servers.py | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 08b95b46a..6cd8bb451 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -84,13 +84,11 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) - def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ return dict(server=dict(id=inst['id'], name=inst['display_name'])) - class Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ @@ -178,7 +176,13 @@ class Controller(wsgi.Controller): key_data=key_pair['public_key'], metadata=metadata, onset_files=env.get('onset_files', [])) - return _translate_keys(instances[0]) + + server = _translate_keys(instances[0]) + password = "%s%s" % (server['server']['name'][:4], + utils.generate_password(12)) + server['server']['adminPass'] = password + self.compute_api.set_admin_password(context, server['server']['id']) + return server def update(self, req, id): """ Updates the server name or password """ diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..16b48a13b 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -186,7 +186,7 @@ class ServersTest(test.TestCase): def test_create_instance(self): def instance_create(context, inst): - return {'id': '1', 'display_name': ''} + return {'id': '1', 'display_name': 'server_test'} def server_update(context, id, params): return instance_create(context, id) @@ -230,6 +230,12 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) + server = json.loads(res.body)['server'] + self.assertEqual('serv', server['adminPass'][:4]) + self.assertEqual(16, len(server['adminPass'])) + self.assertEqual('server_test', server['name']) + self.assertEqual('1', server['id']) + self.assertEqual(res.status_int, 200) def test_update_no_body(self): -- cgit From a775c4eee279e11268a6cc447aee24c452e4665a Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Mon, 7 Mar 2011 17:17:41 +0000 Subject: Merge prop changes and test fixes --- nova/tests/xenapi/stubs.py | 26 +++++++++++++------------- nova/virt/xenapi/vm_utils.py | 30 ++++++++++++------------------ nova/virt/xenapi/vmops.py | 4 ++-- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index caefcff34..11e89c9b4 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -227,18 +227,8 @@ class FakeSessionForMigrationTests(fake.SessionBase): def stub_out_migration_methods(stubs): - class FakeSnapshot(object): - def __getattr__(self, key): - return str(key) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - pass - def fake_get_snapshot(self, instance): - return FakeSnapshot() + return 'foo', 'bar' @classmethod def fake_get_vdi(cls, session, vm_ref): @@ -251,11 +241,21 @@ def stub_out_migration_methods(stubs): pass @classmethod - def fake_scan_sr(cls, session): + def fake_sr(cls, session, *args): + pass + + @classmethod + def fake_get_sr_path(cls, *args): + return "fake" + + def fake_destroy(*args, **kwargs): pass - stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_scan_sr) + stubs.Set(vmops.VMOps, '_destroy', fake_destroy) + stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr) + stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr) stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot) stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi) stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None) + stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path) stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index eff207a51..80b7540d4 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -307,10 +307,16 @@ class VMHelper(HelperBase): return session.call_xenapi('SR.get_by_name_label', sr_label)[0] @classmethod - def get_sr_path(cls, session, sr_label='slices'): - """Finds the SR and then coerces it into a path on the dom0 file - system""" - return FLAGS.xenapi_sr_base_path + cls.get_sr(session, sr_label) + def get_sr_path(cls, session): + """Return the path to our storage repository + + This is used when we're dealing with VHDs directly, either by taking + snapshots or by restoring an image in the DISK_VHD format. + """ + sr_ref = safe_find_sr(session) + sr_rec = session.get_xenapi().SR.get_record(sr_ref) + sr_uuid = sr_rec["uuid"] + return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid) @classmethod def upload_image(cls, session, instance_id, vdi_uuids, image_id): @@ -326,7 +332,7 @@ class VMHelper(HelperBase): 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, - 'sr_path': get_sr_path(session)} + 'sr_path': cls.get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) @@ -369,7 +375,7 @@ class VMHelper(HelperBase): 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, 'uuid_stack': uuid_stack, - 'sr_path': get_sr_path(session)} + 'sr_path': cls.get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'download_vhd', kwargs) @@ -775,18 +781,6 @@ def find_sr(session): return None -def get_sr_path(session): - """Return the path to our storage repository - - This is used when we're dealing with VHDs directly, either by taking - snapshots or by restoring an image in the DISK_VHD format. - """ - sr_ref = safe_find_sr(session) - sr_rec = session.get_xenapi().SR.get_record(sr_ref) - sr_uuid = sr_rec["uuid"] - return os.path.join(FLAGS.xenapi_sr_base_path, sr_uuid) - - def remap_vbd_dev(dev): """Return the appropriate location for a plugged-in VBD device diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 01bfa2dc5..b862c9de9 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -262,7 +262,7 @@ class VMOps(object): self._session, instance.id, template_vdi_uuids, image_id) finally: if template_vm_ref: - self.virt._destroy(self.instance, template_vm_ref, + self._destroy(instance, template_vm_ref, shutdown=False, destroy_kernel_ramdisk=False) logging.debug(_("Finished snapshot and upload for VM %s"), instance) @@ -330,7 +330,7 @@ class VMOps(object): finally: if template_vm_ref: - self.virt._destroy(self.instance, template_vm_ref, + self._destroy(instance, template_vm_ref, shutdown=False, destroy_kernel_ramdisk=False) # TODO(mdietz): we could also consider renaming these to something -- cgit From f72366f007239656d3d5e3fc80cd277758eedf9b Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Mon, 7 Mar 2011 19:33:24 +0000 Subject: Create --paste_config flag defaulting to api-paste.ini and mv etc/nova-api.conf to match --- bin/nova-api | 7 +++-- etc/api-paste.ini | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ etc/nova-api.conf | 91 ------------------------------------------------------- 3 files changed, 96 insertions(+), 93 deletions(-) create mode 100644 etc/api-paste.ini delete mode 100644 etc/nova-api.conf diff --git a/bin/nova-api b/bin/nova-api index 14be4b841..0b2a44c88 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -43,6 +43,8 @@ from nova import wsgi LOG = logging.getLogger('nova.api') FLAGS = flags.FLAGS +flags.DEFINE_string('paste_config', "api-paste.ini", + 'File name for the paste.deploy config for nova-api') flags.DEFINE_string('ec2_listen', "0.0.0.0", 'IP address for EC2 API to listen') flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') @@ -90,8 +92,9 @@ if __name__ == '__main__': for flag in FLAGS: flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - conf = wsgi.paste_config_file('nova-api.conf') + conf = wsgi.paste_config_file(FLAGS.paste_config) if conf: run_app(conf) else: - LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') + LOG.error(_("No paste configuration found for: %s"), + FLAGS.paste_config) diff --git a/etc/api-paste.ini b/etc/api-paste.ini new file mode 100644 index 000000000..9f7e93d4c --- /dev/null +++ b/etc/api-paste.ini @@ -0,0 +1,91 @@ +####### +# EC2 # +####### + +[composite:ec2] +use = egg:Paste#urlmap +/: ec2versions +/services/Cloud: ec2cloud +/services/Admin: ec2admin +/latest: ec2metadata +/2007-01-19: ec2metadata +/2007-03-01: ec2metadata +/2007-08-29: ec2metadata +/2007-10-10: ec2metadata +/2007-12-15: ec2metadata +/2008-02-01: ec2metadata +/2008-09-01: ec2metadata +/2009-04-04: ec2metadata +/1.0: ec2metadata + +[pipeline:ec2cloud] +pipeline = logrequest authenticate cloudrequest authorizer ec2executor +#pipeline = logrequest ec2lockout authenticate cloudrequest authorizer ec2executor + +[pipeline:ec2admin] +pipeline = logrequest authenticate adminrequest authorizer ec2executor + +[pipeline:ec2metadata] +pipeline = logrequest ec2md + +[pipeline:ec2versions] +pipeline = logrequest ec2ver + +[filter:logrequest] +paste.filter_factory = nova.api.ec2:RequestLogging.factory + +[filter:ec2lockout] +paste.filter_factory = nova.api.ec2:Lockout.factory + +[filter:authenticate] +paste.filter_factory = nova.api.ec2:Authenticate.factory + +[filter:cloudrequest] +controller = nova.api.ec2.cloud.CloudController +paste.filter_factory = nova.api.ec2:Requestify.factory + +[filter:adminrequest] +controller = nova.api.ec2.admin.AdminController +paste.filter_factory = nova.api.ec2:Requestify.factory + +[filter:authorizer] +paste.filter_factory = nova.api.ec2:Authorizer.factory + +[app:ec2executor] +paste.app_factory = nova.api.ec2:Executor.factory + +[app:ec2ver] +paste.app_factory = nova.api.ec2:Versions.factory + +[app:ec2md] +paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.factory + +############# +# Openstack # +############# + +[composite:osapi] +use = egg:Paste#urlmap +/: osversions +/v1.0: openstackapi + +[pipeline:openstackapi] +pipeline = faultwrap auth ratelimit osapiapp + +[filter:faultwrap] +paste.filter_factory = nova.api.openstack:FaultWrapper.factory + +[filter:auth] +paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory + +[filter:ratelimit] +paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory + +[app:osapiapp] +paste.app_factory = nova.api.openstack:APIRouter.factory + +[pipeline:osversions] +pipeline = faultwrap osversionapp + +[app:osversionapp] +paste.app_factory = nova.api.openstack:Versions.factory diff --git a/etc/nova-api.conf b/etc/nova-api.conf deleted file mode 100644 index 9f7e93d4c..000000000 --- a/etc/nova-api.conf +++ /dev/null @@ -1,91 +0,0 @@ -####### -# EC2 # -####### - -[composite:ec2] -use = egg:Paste#urlmap -/: ec2versions -/services/Cloud: ec2cloud -/services/Admin: ec2admin -/latest: ec2metadata -/2007-01-19: ec2metadata -/2007-03-01: ec2metadata -/2007-08-29: ec2metadata -/2007-10-10: ec2metadata -/2007-12-15: ec2metadata -/2008-02-01: ec2metadata -/2008-09-01: ec2metadata -/2009-04-04: ec2metadata -/1.0: ec2metadata - -[pipeline:ec2cloud] -pipeline = logrequest authenticate cloudrequest authorizer ec2executor -#pipeline = logrequest ec2lockout authenticate cloudrequest authorizer ec2executor - -[pipeline:ec2admin] -pipeline = logrequest authenticate adminrequest authorizer ec2executor - -[pipeline:ec2metadata] -pipeline = logrequest ec2md - -[pipeline:ec2versions] -pipeline = logrequest ec2ver - -[filter:logrequest] -paste.filter_factory = nova.api.ec2:RequestLogging.factory - -[filter:ec2lockout] -paste.filter_factory = nova.api.ec2:Lockout.factory - -[filter:authenticate] -paste.filter_factory = nova.api.ec2:Authenticate.factory - -[filter:cloudrequest] -controller = nova.api.ec2.cloud.CloudController -paste.filter_factory = nova.api.ec2:Requestify.factory - -[filter:adminrequest] -controller = nova.api.ec2.admin.AdminController -paste.filter_factory = nova.api.ec2:Requestify.factory - -[filter:authorizer] -paste.filter_factory = nova.api.ec2:Authorizer.factory - -[app:ec2executor] -paste.app_factory = nova.api.ec2:Executor.factory - -[app:ec2ver] -paste.app_factory = nova.api.ec2:Versions.factory - -[app:ec2md] -paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.factory - -############# -# Openstack # -############# - -[composite:osapi] -use = egg:Paste#urlmap -/: osversions -/v1.0: openstackapi - -[pipeline:openstackapi] -pipeline = faultwrap auth ratelimit osapiapp - -[filter:faultwrap] -paste.filter_factory = nova.api.openstack:FaultWrapper.factory - -[filter:auth] -paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory - -[filter:ratelimit] -paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory - -[app:osapiapp] -paste.app_factory = nova.api.openstack:APIRouter.factory - -[pipeline:osversions] -pipeline = faultwrap osversionapp - -[app:osversionapp] -paste.app_factory = nova.api.openstack:Versions.factory -- cgit From bcb18ee3d0d095b616c0909c92a151a599d4e17f Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Mon, 7 Mar 2011 15:05:07 -0500 Subject: Invalid values for offset and limit params in http requests now return a 400 response with a useful message in the body. Also added and updated tests. --- nova/api/openstack/common.py | 11 +++++++---- nova/tests/api/openstack/test_common.py | 20 ++++---------------- nova/tests/api/openstack/test_servers.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 9f85c5c8a..f7a9cc3f0 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -36,15 +36,18 @@ def limited(items, request, max_limit=1000): try: offset = int(request.GET.get('offset', 0)) except ValueError: - offset = 0 + raise webob.exc.HTTPBadRequest(_('offset param must be an integer')) try: limit = int(request.GET.get('limit', max_limit)) except ValueError: - limit = max_limit + raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) - if offset < 0 or limit < 0: - raise webob.exc.HTTPBadRequest() + if limit < 0: + raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + + if offset < 0: + raise webob.exc.HTTPBadRequest(_('offset param must be positive')) limit = min(max_limit, limit or max_limit) range_end = offset + limit diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 92023362c..8f57c5b67 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -79,20 +79,14 @@ class LimiterTest(test.TestCase): Test offset key works with a blank offset. """ req = Request.blank('/?offset=') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_offset_bad(self): """ Test offset key works with a BAD offset. """ req = Request.blank(u'/?offset=\u0020aa') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_nothing(self): """ @@ -166,18 +160,12 @@ class LimiterTest(test.TestCase): """ Test a negative limit. """ - def _limit_large(): - limited(self.large, req, max_limit=2000) - req = Request.blank('/?limit=-3000') - self.assertRaises(webob.exc.HTTPBadRequest, _limit_large) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_negative_offset(self): """ Test a negative offset. """ - def _limit_large(): - limited(self.large, req, max_limit=2000) - req = Request.blank('/?offset=-30') - self.assertRaises(webob.exc.HTTPBadRequest, _limit_large) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..10fb2bafb 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -184,6 +184,34 @@ class ServersTest(test.TestCase): self.assertEqual(s.get('imageId', None), None) i += 1 + def test_get_servers_with_limit(self): + req = webob.Request.blank('/v1.0/servers?limit=3') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [0, 1, 2]) + + req = webob.Request.blank('/v1.0/servers?limit=aaa') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + self.assertTrue('limit' in res.body) + + def test_get_servers_with_offset(self): + req = webob.Request.blank('/v1.0/servers?offset=2') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [2, 3, 4]) + + req = webob.Request.blank('/v1.0/servers?offset=aaa') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + self.assertTrue('offset' in res.body) + + def test_get_servers_with_limit_and_offset(self): + req = webob.Request.blank('/v1.0/servers?limit=2&offset=1') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [1, 2]) + def test_create_instance(self): def instance_create(context, inst): return {'id': '1', 'display_name': ''} -- cgit From 8c3bc15c96c6a6f1c99d829337921f2645608410 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:43:31 +0100 Subject: Make iptables rules class __ne__ just be inverted __eq__. --- nova/network/linux_net.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 8832f7c68..57e77f8d5 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -85,10 +85,7 @@ class IptablesRule(object): (self.wrap == other.wrap)) def __ne__(self, other): - return ((self.chain != other.chain) or - (self.rule != other.rule) or - (self.top != other.top) or - (self.wrap != other.wrap)) + return not self == other def __str__(self): if self.wrap: -- cgit From 7b7abe7e7a25c0cd07c64c34f69ce050c669cfc3 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:54:25 +0100 Subject: Log failed command execution if there are more retry attempts left. --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/utils.py b/nova/utils.py index 829adfb9e..80e5b6bbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -166,6 +166,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, if not attempts: raise else: + LOG.debug(_("%r failed. Retrying."), cmd) greenthread.sleep(random.randint(20, 200) / 100.0) -- cgit From 4e9c570fbf8b3987d556da085b61f159f32c16f1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:59:05 +0100 Subject: Use IptablesManager.semapahore from securitygroups driver to ensure we don't apply half a rule set. --- nova/virt/libvirt_conn.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b900cb8eb..825bcb0d7 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1372,8 +1372,12 @@ class IptablesFirewallDriver(FirewallDriver): def refresh_security_group_rules(self, security_group): for instance in self.instances.values(): - self.remove_filters_for_instance(instance) - self.add_filters_for_instance(instance) + # We use the semaphore to make sure noone applies the rule set + # after we've yanked the existing rules but before we've put in + # the new ones. + with self.iptables.semaphore: + self.remove_filters_for_instance(instance) + self.add_filters_for_instance(instance) self.iptables.apply() def _security_group_chain_name(self, security_group_id): -- cgit From 0abd5bfecd279272e5fe1b0de04478909cd77010 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 7 Mar 2011 22:18:15 +0100 Subject: added network_get_by_cidr method to nova.db api --- bin/nova-manage | 9 +-------- nova/db/api.py | 7 +++++++ nova/db/sqlalchemy/api.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 9557f2423..b274c5bd1 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -545,16 +545,9 @@ class NetworkCommands(object): network.dhcp_start, network.dns) - def delete(self, fixed_range): """Deletes a network""" - try: - network = [n for n in db.network_get_all(context.get_admin_context()) - if n.cidr == fixed_range][0] - - print network.id, network.cidr, network.project_id - except IndexError: - raise ValueError(_("Network does not exist")) + print db.network_get_by_cidr(context.get_admin_context(), fixed_range) class ServiceCommands(object): """Enable and disable running services""" diff --git a/nova/db/api.py b/nova/db/api.py index d23f14a3c..c73796487 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -459,6 +459,10 @@ def network_associate(context, project_id): """Associate a free network to a project.""" return IMPL.network_associate(context, project_id) +def network_is_associated(context, project_id): + """Returns true the the network is associated to a project""" + return IMPL.network_is_associated(context, project_id) + def network_count(context): """Return the number of networks.""" @@ -525,6 +529,9 @@ def network_get_by_bridge(context, bridge): """Get a network by bridge or raise if it does not exist.""" return IMPL.network_get_by_bridge(context, bridge) +def network_get_by_cidr(context, cidr): + """Get a network by cidr or raise if it does not exist""" + return IMPL.network_get_by_cidr(context, cidr) def network_get_by_instance(context, instance_id): """Get a network by instance id or raise if it does not exist.""" diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 919dda118..bd2de70c7 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -993,6 +993,13 @@ def network_associate(context, project_id): return network_ref +@require_admin_context +def network_is_associated(context, project_id): + session = get_session() + network = session.query(models.Network.project_id).filter(project_id=1).first() + print network + + @require_admin_context def network_count(context): session = get_session() @@ -1116,6 +1123,17 @@ def network_get_by_bridge(context, bridge): return result +@require_admin_context +def network_get_by_cidr(context, cidr): + session = get_session() + result = session.query(models.Network).\ + filter_by(cidr=cidr).first() + + if not result: + raise exception.NotFound(_('Network with cidr %s does not exist') % + cidr) + return result.id + @require_admin_context def network_get_by_instance(_context, instance_id): session = get_session() -- cgit From c944e902aa68d170c0d97a1d50e28fe5e59c572b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 21:20:32 +0000 Subject: rework register commands based on review --- bin/nova-manage | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index ebeda05a0..b61a5d412 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -768,16 +768,16 @@ class ImageCommands(object): except Exception as exc: print _("Failed to register %(path)s: %(exc)s") % locals() - def register_all(self, image, kernel, ramdisk, owner, name=None, + def all_register(self, image, kernel, ramdisk, owner, name=None, is_public='T', architecture='x86_64'): """Uploads an image, kernel, and ramdisk into the image_service arguments: image kernel ramdisk owner [name] [is_public='T'] [architecture='x86_64']""" - kernel_id = self._register('kernel', kernel, owner, None, + kernel_id = self.kernel_register(kernel, owner, None, is_public, architecture) - ramdisk_id = self._register('ramdisk', ramdisk, owner, None, + ramdisk_id = self.ramdisk_register(ramdisk, owner, None, is_public, architecture) - self._register('machine', image, owner, name, is_public, + self.image_register(image, owner, name, is_public, architecture, kernel_id, ramdisk_id) def image_register(self, path, owner, name=None, is_public='T', @@ -785,7 +785,7 @@ class ImageCommands(object): """Uploads an image into the image_service arguments: path owner [name] [is_public='T'] [architecture='x86_64'] [kernel_id] [ramdisk_id]""" - self._register('machine', path, owner, name, is_public, + return self._register('machine', path, owner, name, is_public, architecture, kernel_id, ramdisk_id) def kernel_register(self, path, owner, name=None, is_public='T', @@ -793,14 +793,16 @@ class ImageCommands(object): """Uploads a kernel into the image_service arguments: path owner [name] [is_public='T'] [architecture='x86_64'] """ - self._register('kernel', path, owner, name, is_public, architecture) + return self._register('kernel', path, owner, name, is_public, + architecture) def ramdisk_register(self, path, owner, name=None, is_public='T', architecture='x86_64'): """Uploads a ramdisk into the image_service arguments: path owner [name] [is_public='T'] [architecture='x86_64'] """ - self._register('ramdisk', path, owner, name, is_public, architecture) + return self._register('ramdisk', path, owner, name, is_public, + architecture) def _lookup(self, old_image_id): try: -- cgit From fd95523689b80f53972c59c3738e6b786a7160ff Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 21:22:06 +0000 Subject: pep8 --- bin/nova-manage | 1 - nova/api/ec2/cloud.py | 1 - nova/api/ec2/ec2utils.py | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b61a5d412..f8cc6e68e 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -841,7 +841,6 @@ class ImageCommands(object): except Exception as exc: print _("Failed to convert %(old)s: %(exc)s") % locals() - def convert(self, directory): """Uploads old objectstore images in directory to new service arguments: directory""" diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 496e944fe..6479c9445 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -864,7 +864,6 @@ class CloudController(object): except exception.NotFound: return self.image_service.show_by_name(context, ec2_id) - def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index e4df80cf8..3b34f6ea5 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -18,6 +18,7 @@ from nova import exception + def ec2_id_to_id(ec2_id): """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" try: -- cgit From 02e6a17bec06beee5dbffe085073c97281abb586 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 21:30:20 +0000 Subject: move the images_dir out of the way when converting --- bin/nova-manage | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bin/nova-manage b/bin/nova-manage index f8cc6e68e..b97d8b81d 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -846,6 +846,16 @@ class ImageCommands(object): arguments: directory""" machine_images = {} other_images = {} + directory = os.path.abspath(directory) + # NOTE(vish): If we're importing from the images path dir, attempt + # to move the files out of the way before importing + # so we aren't writing to the same directory. This + # may fail if the dir was a mointpoint. + if directory == os.path.abspath(FLAGS.images_path): + new_dir = "%s_bak" % directory + os.move(directory, new_dir) + os.mkdir(directory) + directory = new_dir for fn in glob.glob("%s/*/info.json" % directory): try: image_path = os.path.join(fn.rpartition('/')[0], 'image') -- cgit From 56ee811efd52d0971d7fea4c232a904b3ee78ac6 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 7 Mar 2011 22:37:26 +0100 Subject: deleted network_is_associated from nova.db api --- bin/nova-manage | 4 ++-- nova/db/api.py | 5 ----- nova/db/sqlalchemy/api.py | 9 +-------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b274c5bd1..94b0d5946 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -547,8 +547,8 @@ class NetworkCommands(object): def delete(self, fixed_range): """Deletes a network""" - print db.network_get_by_cidr(context.get_admin_context(), fixed_range) - + network = db.network_get_by_cidr(context.get_admin_context(), fixed_range) + class ServiceCommands(object): """Enable and disable running services""" diff --git a/nova/db/api.py b/nova/db/api.py index c73796487..04f5fd72f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -459,11 +459,6 @@ def network_associate(context, project_id): """Associate a free network to a project.""" return IMPL.network_associate(context, project_id) -def network_is_associated(context, project_id): - """Returns true the the network is associated to a project""" - return IMPL.network_is_associated(context, project_id) - - def network_count(context): """Return the number of networks.""" return IMPL.network_count(context) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index bd2de70c7..c8f42425d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -993,13 +993,6 @@ def network_associate(context, project_id): return network_ref -@require_admin_context -def network_is_associated(context, project_id): - session = get_session() - network = session.query(models.Network.project_id).filter(project_id=1).first() - print network - - @require_admin_context def network_count(context): session = get_session() @@ -1132,7 +1125,7 @@ def network_get_by_cidr(context, cidr): if not result: raise exception.NotFound(_('Network with cidr %s does not exist') % cidr) - return result.id + return result @require_admin_context def network_get_by_instance(_context, instance_id): -- cgit From f79220a1f6a12621463b410d26e31e29a9e6ea3e Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Mon, 7 Mar 2011 15:41:37 -0600 Subject: cleaned up virt.xenapi.vmops._get_vm_opaque_ref. more reliable approach to checking if param is an opaque ref. code is cleaner --- nova/virt/xenapi/vmops.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b862c9de9..b1671fde4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -188,30 +188,32 @@ class VMOps(object): """Refactored out the common code of many methods that receive either a vm name or a vm instance, and want a vm instance in return. """ - vm = None - try: - if instance_or_vm.startswith("OpaqueRef:"): - # Got passed an opaque ref; return it + # if instance_or_vm is a string it must be opaque ref or instance name + if isinstance(instance_or_vm, str): + vm_rec = self._session.get_xenapi().VM.get_record(instance_or_vm) + if vm_rec != None: + # an opaque ref was passed in, return it return instance_or_vm else: - # Must be the instance name + # it must be an instance name instance_name = instance_or_vm - except (AttributeError, KeyError): - # Note the the KeyError will only happen with fakes.py - # Not a string; must be an ID or a vm instance - if isinstance(instance_or_vm, (int, long)): - ctx = context.get_admin_context() - try: - instance_obj = db.instance_get(ctx, instance_or_vm) - instance_name = instance_obj.name - except exception.NotFound: - # The unit tests screw this up, as they use an integer for - # the vm name. I'd fix that up, but that's a matter for - # another bug report. So for now, just try with the passed - # value - instance_name = instance_or_vm - else: - instance_name = instance_or_vm.name + + # if instance_or_vm is an int/long it must be instance id + elif isinstance(instance_or_vm, (int, long)): + ctx = context.get_admin_context() + try: + instance_obj = db.instance_get(ctx, instance_or_vm) + instance_name = instance_obj.name + except exception.NotFound: + # The unit tests screw this up, as they use an integer for + # the vm name. I'd fix that up, but that's a matter for + # another bug report. So for now, just try with the passed + # value + instance_name = instance_or_vm + + # otherwise instance_or_vm is an instance object + else: + instance_name = instance_or_vm.name vm = VMHelper.lookup(self._session, instance_name) if vm is None: raise exception.NotFound( -- cgit From 59f73e3180731cec644b590d448e0da74711ae03 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Mon, 7 Mar 2011 16:11:10 -0600 Subject: virt.xenapi.vmops._get_vm_opaque_ref exception caught properly --- nova/virt/xenapi/vmops.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b1671fde4..ae4609418 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -190,13 +190,16 @@ class VMOps(object): """ # if instance_or_vm is a string it must be opaque ref or instance name if isinstance(instance_or_vm, str): - vm_rec = self._session.get_xenapi().VM.get_record(instance_or_vm) - if vm_rec != None: - # an opaque ref was passed in, return it - return instance_or_vm - else: - # it must be an instance name - instance_name = instance_or_vm + ref = None + try: + ref = self._session.get_xenapi().VM.get_record(instance_or_vm) + if ref != None: + # an opaque ref was passed in, return it + return instance_or_vm + except: + pass + # wasn't an opaque ref, must be an instance name + instance_name = instance_or_vm # if instance_or_vm is an int/long it must be instance id elif isinstance(instance_or_vm, (int, long)): -- cgit From 3fc6b8cbbd1be5baffc300112a0e39a807209c36 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Mon, 7 Mar 2011 16:34:59 -0600 Subject: virt.xenapi.vmops._get_vm_opaque_ref checks for basestring instance instead of str --- nova/virt/xenapi/vmops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index ae4609418..30fa5bdd7 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -189,7 +189,7 @@ class VMOps(object): a vm name or a vm instance, and want a vm instance in return. """ # if instance_or_vm is a string it must be opaque ref or instance name - if isinstance(instance_or_vm, str): + if isinstance(instance_or_vm, basestring): ref = None try: ref = self._session.get_xenapi().VM.get_record(instance_or_vm) -- cgit From 88c5555e867c730065c18541a35b161eb861b502 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Mon, 7 Mar 2011 16:40:19 -0600 Subject: First part of the bug fix --- nova/compute/manager.py | 4 +--- nova/virt/xenapi/vmops.py | 8 +++++++- nova/virt/xenapi_conn.py | 9 +++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index b3e864154..b35216dd3 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -510,9 +510,7 @@ class ComputeManager(manager.Manager): instance_ref = self.db.instance_get(context, migration_ref['instance_id']) - # this may get passed into the following spawn instead - new_disk_info = self.driver.attach_disk(instance_ref, disk_info) - self.driver.spawn(instance_ref, disk=new_disk_info) + self.driver.finish_resize(instance_ref, disk_info) self.db.migration_update(context, migration_id, {'status': 'finished', }) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index b862c9de9..37f513599 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -72,7 +72,13 @@ class VMOps(object): LOG.debug(_("Starting instance %s"), instance.name) self._session.call_xenapi('VM.start', vm_ref, False, False) - def spawn(self, instance, disk): + def spawn(self, instance): + self._spawn(instance, disk=None) + + def spawn_with_disk(self, instance, disk): + self._spawn(instance, disk=disk) + + def _spawn(self, instance, disk): """Create VM instance""" instance_name = instance.name vm = VMHelper.lookup(self._session, instance_name) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 62e17e851..7e8f825e9 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -154,9 +154,14 @@ class XenAPIConnection(object): """List VM instances""" return self._vmops.list_instances() - def spawn(self, instance, disk=None): + def spawn(self, instance): """Create VM instance""" - self._vmops.spawn(instance, disk) + self._vmops.spawn(instance) + + def finish_resize(self, instance, disk_info) + """Completes a resize, turning on the migrated instance""" + new_disk_info = self.attach_disk(instance, disk_info) + self._vmops.spawn_with_disk(instance, new_disk_info) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ -- cgit From ede88283729663f11d913cc54bcf8ee08028d98f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 7 Mar 2011 14:42:36 -0800 Subject: A few formatting niceties --- nova/service.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nova/service.py b/nova/service.py index 8fdaca0a5..389a6b2df 100644 --- a/nova/service.py +++ b/nova/service.py @@ -42,6 +42,7 @@ from nova import utils from nova import version from nova import wsgi + FLAGS = flags.FLAGS flags.DEFINE_integer('report_interval', 10, 'seconds between nodes reporting state to datastore', @@ -271,6 +272,11 @@ def serve(*services): x.start() +def wait(): + while True: + greenthread.sleep(5) + + def serve_wsgi(cls, conf): try: service = cls.create(conf) @@ -290,11 +296,6 @@ def serve_wsgi(cls, conf): return service -def wait(): - while True: - greenthread.sleep(5) - - def _run_wsgi(paste_config_file, apis): logging.debug(_("Using paste.deploy config at: %s"), paste_config_file) apps = [] -- cgit From 5c7ee13b058fb954fd9bbc4a3550716b8faa0b97 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Mon, 7 Mar 2011 22:50:35 +0000 Subject: And unit tests --- nova/tests/test_xenapi.py | 5 +++++ nova/tests/xenapi/stubs.py | 4 ++++ nova/virt/xenapi_conn.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 7f437c2b8..6e458558d 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -383,6 +383,11 @@ class XenAPIMigrateInstance(test.TestCase): conn = xenapi_conn.get_connection(False) conn.attach_disk(instance, {'base_copy': 'hurr', 'cow': 'durr'}) + def test_finish_resize(self): + instance = db.instance_create(self.values) + stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) + conn = xenapi_conn.get_connection(False) + conn.finish_resize(instance, dict(base_copy='hurr', cow='durr')) class XenAPIDetermineDiskImageTestCase(test.TestCase): """ diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 11e89c9b4..28037c2ba 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -251,6 +251,9 @@ def stub_out_migration_methods(stubs): def fake_destroy(*args, **kwargs): pass + def fake_spawn_with_disk(*args, **kwargs): + pass + stubs.Set(vmops.VMOps, '_destroy', fake_destroy) stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr) stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr) @@ -259,3 +262,4 @@ def stub_out_migration_methods(stubs): stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None) stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path) stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown) + stubs.Set(vmops.VMOps, 'spawn_with_disk', fake_spawn_with_disk) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 7e8f825e9..3991496b2 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -158,7 +158,7 @@ class XenAPIConnection(object): """Create VM instance""" self._vmops.spawn(instance) - def finish_resize(self, instance, disk_info) + def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" new_disk_info = self.attach_disk(instance, disk_info) self._vmops.spawn_with_disk(instance, new_disk_info) -- cgit From 2f0845b7b80081d18ee268b94fe38326f3c5401e Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Mon, 7 Mar 2011 23:07:05 +0000 Subject: A few more changes --- nova/tests/test_xenapi.py | 6 ------ nova/virt/xenapi/vmops.py | 10 +++++----- nova/virt/xenapi_conn.py | 9 +++------ 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 6e458558d..f5b154a51 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -377,12 +377,6 @@ class XenAPIMigrateInstance(test.TestCase): conn = xenapi_conn.get_connection(False) conn.migrate_disk_and_power_off(instance, '127.0.0.1') - def test_attach_disk(self): - instance = db.instance_create(self.values) - stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) - conn = xenapi_conn.get_connection(False) - conn.attach_disk(instance, {'base_copy': 'hurr', 'cow': 'durr'}) - def test_finish_resize(self): instance = db.instance_create(self.values) stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 37f513599..e658de7f3 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -75,8 +75,8 @@ class VMOps(object): def spawn(self, instance): self._spawn(instance, disk=None) - def spawn_with_disk(self, instance, disk): - self._spawn(instance, disk=disk) + def spawn_with_disk(self, instance, vdi_uuid): + self._spawn(instance, disk=vdi_uuid) def _spawn(self, instance, disk): """Create VM instance""" @@ -343,14 +343,14 @@ class VMOps(object): # sensible so we don't need to blindly pass around dictionaries return {'base_copy': base_copy_uuid, 'cow': cow_uuid} - def attach_disk(self, instance, disk_info): + def attach_disk(self, instance, base_copy_uuid, cow_uuid): """Links the base copy VHD to the COW via the XAPI plugin""" vm_ref = VMHelper.lookup(self._session, instance.name) new_base_copy_uuid = str(uuid.uuid4()) new_cow_uuid = str(uuid.uuid4()) params = {'instance_id': instance.id, - 'old_base_copy_uuid': disk_info['base_copy'], - 'old_cow_uuid': disk_info['cow'], + 'old_base_copy_uuid': base_copy_uuid, + 'old_cow_uuid': cow_uuid, 'new_base_copy_uuid': new_base_copy_uuid, 'new_cow_uuid': new_cow_uuid, 'sr_path': VMHelper.get_sr_path(self._session), } diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 3991496b2..9965accad 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -160,8 +160,9 @@ class XenAPIConnection(object): def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" - new_disk_info = self.attach_disk(instance, disk_info) - self._vmops.spawn_with_disk(instance, new_disk_info) + cow_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], + disk_info['cow']) + self._vmops.spawn_with_disk(instance, cow_uuid) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ @@ -202,10 +203,6 @@ class XenAPIConnection(object): off the instance copies over the COW disk""" return self._vmops.migrate_disk_and_power_off(instance, dest) - def attach_disk(self, instance, disk_info): - """Moves the copied VDIs into the SR""" - return self._vmops.attach_disk(instance, disk_info) - def suspend(self, instance, callback): """suspend the specified instance""" self._vmops.suspend(instance, callback) -- cgit From 8e0fd37ddfbe88df296cf45583f0b3e4fa4d7a75 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 7 Mar 2011 15:22:59 -0800 Subject: Converted tabs to spaces in bin/nova-api --- bin/nova-api | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 2d2ef6d0c..c921ec45c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -62,5 +62,5 @@ if __name__ == '__main__': LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') sys.exit(1) else: - service = service.serve_wsgi(service.ApiService, conf) + service = service.serve_wsgi(service.ApiService, conf) service.wait() -- cgit From e69c802aaf40f3b90789aeef8bf3ef5dcbbcb2f3 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 7 Mar 2011 15:36:04 -0800 Subject: Moved FLAGS.paste_config to its re-usable location --- bin/nova-api | 14 +++----------- nova/service.py | 10 +++++++--- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index f48dbe5a5..85ca4eefd 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -45,9 +45,6 @@ from nova import wsgi LOG = logging.getLogger('nova.api') FLAGS = flags.FLAGS -flags.DEFINE_string('paste_config', "api-paste.ini", - 'File name for the paste.deploy config for nova-api') - if __name__ == '__main__': utils.default_flagfile() @@ -59,11 +56,6 @@ if __name__ == '__main__': for flag in FLAGS: flag_get = FLAGS.get(flag, None) LOG.debug("%(flag)s : %(flag_get)s" % locals()) - conf = wsgi.paste_config_file(FLAGS.paste_config) - if not conf: - LOG.error(_("No paste configuration found for: %s"), - FLAGS.paste_config) - sys.exit(1) - else: - service = service.serve_wsgi(service.ApiService, conf) - service.wait() + + service = service.serve_wsgi(service.ApiService) + service.wait() diff --git a/nova/service.py b/nova/service.py index 389a6b2df..5a8d58695 100644 --- a/nova/service.py +++ b/nova/service.py @@ -56,6 +56,8 @@ flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') flags.DEFINE_string('osapi_listen', "0.0.0.0", 'IP address for OpenStack API to listen') flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') +flags.DEFINE_string('paste_config', "api-paste.ini", + 'File name for the paste.deploy config for nova-api') class Service(object): @@ -238,9 +240,11 @@ class ApiService(WsgiService): @classmethod def create(cls, conf=None): if not conf: - conf = wsgi.paste_config_file('nova-api.conf') + conf = wsgi.paste_config_file(FLAGS.paste_config) if not conf: - raise exception.Error(_("Cannot load nova-api.conf")) + message = (_("No paste configuration found for: %s"), + FLAGS.paste_config) + raise exception.Error(message) api_endpoints = ['ec2', 'osapi'] service = cls(conf, api_endpoints) return service @@ -277,7 +281,7 @@ def wait(): greenthread.sleep(5) -def serve_wsgi(cls, conf): +def serve_wsgi(cls, conf=None): try: service = cls.create(conf) except Exception: -- cgit From e39995def6a2a11cdd430b0e6f603b493be5542b Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Mon, 7 Mar 2011 23:51:20 +0000 Subject: Some more refactoring and a tighter unit test --- nova/tests/test_xenapi.py | 14 ++++++++++---- nova/tests/xenapi/stubs.py | 15 +++++++++++++-- nova/virt/xenapi/vmops.py | 30 ++++++++++++++---------------- nova/virt/xenapi_conn.py | 4 ++-- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index f5b154a51..919a38c06 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -360,16 +360,22 @@ class XenAPIMigrateInstance(test.TestCase): db_fakes.stub_out_db_instance_api(self.stubs) stubs.stub_out_get_target(self.stubs) xenapi_fake.reset() + self.manager = manager.AuthManager() + self.user = self.manager.create_user('fake', 'fake', 'fake', + admin=True) + self.project = self.manager.create_project('fake', 'fake', 'fake') self.values = {'name': 1, 'id': 1, - 'project_id': 'fake', - 'user_id': 'fake', + 'project_id': self.project.id, + 'user_id': self.user.id, 'image_id': 1, - 'kernel_id': 2, - 'ramdisk_id': 3, + 'kernel_id': None, + 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', } stubs.stub_out_migration_methods(self.stubs) + glance_stubs.stubout_glance_client(self.stubs, + glance_stubs.FakeGlance) def test_migrate_disk_and_power_off(self): instance = db.instance_create(self.values) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 28037c2ba..d8e358611 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -225,6 +225,17 @@ class FakeSessionForMigrationTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForMigrationTests, self).__init__(uri) + def VDI_get_by_uuid(*args): + return 'hurr' + + def VM_start(self, _1, ref, _2, _3): + vm = fake.get_record('VM', ref) + if vm['power_state'] != 'Halted': + raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted', + vm['power_state']]) + vm['power_state'] = 'Running' + vm['is_a_template'] = False + vm['is_control_domain'] = False def stub_out_migration_methods(stubs): def fake_get_snapshot(self, instance): @@ -251,7 +262,7 @@ def stub_out_migration_methods(stubs): def fake_destroy(*args, **kwargs): pass - def fake_spawn_with_disk(*args, **kwargs): + def fake_reset_network(*args, **kwargs): pass stubs.Set(vmops.VMOps, '_destroy', fake_destroy) @@ -261,5 +272,5 @@ def stub_out_migration_methods(stubs): stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi) stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None) stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path) + stubs.Set(vmops.VMOps, 'reset_network', fake_reset_network) stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown) - stubs.Set(vmops.VMOps, 'spawn_with_disk', fake_spawn_with_disk) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index e658de7f3..7fe1f6ff0 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -72,13 +72,19 @@ class VMOps(object): LOG.debug(_("Starting instance %s"), instance.name) self._session.call_xenapi('VM.start', vm_ref, False, False) - def spawn(self, instance): - self._spawn(instance, disk=None) - - def spawn_with_disk(self, instance, vdi_uuid): - self._spawn(instance, disk=vdi_uuid) + def create_disk(self, instance): + user = AuthManager().get_user(instance.user_id) + project = AuthManager().get_project(instance.project_id) + disk_image_type = VMHelper.determine_disk_image_type(instance) + vdi_uuid = VMHelper.fetch_image(self._session, instance.id, + instance.image_id, user, project, disk_image_type) + return vdi_uuid - def _spawn(self, instance, disk): + def spawn(self, instance): + vdi_uuid = self.create_disk(instance) + self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) + + def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" instance_name = instance.name vm = VMHelper.lookup(self._session, instance_name) @@ -101,17 +107,9 @@ class VMOps(object): vdi_ref = kernel = ramdisk = pv_kernel = None # Are we building from a pre-existing disk? - if not disk: - #if kernel is not present we must download a raw disk - - disk_image_type = VMHelper.determine_disk_image_type(instance) - vdi_uuid = VMHelper.fetch_image(self._session, instance.id, - instance.image_id, user, project, disk_image_type) - vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - - else: - vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', disk) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) + disk_image_type = VMHelper.determine_disk_image_type(instance) if disk_image_type == ImageType.DISK_RAW: # Have a look at the VDI and see if it has a PV kernel pv_kernel = VMHelper.lookup_image(self._session, instance.id, diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 9965accad..b63a5f8c3 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -160,9 +160,9 @@ class XenAPIConnection(object): def finish_resize(self, instance, disk_info): """Completes a resize, turning on the migrated instance""" - cow_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], + vdi_uuid = self._vmops.attach_disk(instance, disk_info['base_copy'], disk_info['cow']) - self._vmops.spawn_with_disk(instance, cow_uuid) + self._vmops._spawn_with_disk(instance, vdi_uuid) def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ -- cgit From ecc6bce311ce85b05802cf04dd2b03a3b91d178d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 16:01:43 -0800 Subject: add a delay before grabbing zipfile --- contrib/nova.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 028184605..d6c9b1081 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -181,7 +181,7 @@ NOVA_CONF_EOF screen_it scheduler "$NOVA_DIR/bin/nova-scheduler" screen_it volume "$NOVA_DIR/bin/nova-volume" screen_it ajax_console_proxy "$NOVA_DIR/bin/nova-ajax-console-proxy" - + sleep 2 # export environment variables for project 'admin' and user 'admin' $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ -- cgit From 5ec9cbcdee3de3868a47ca5ec351a9a2594ceea2 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Mon, 7 Mar 2011 18:05:27 -0600 Subject: virt.xenapi.vmops._get_vm_opaque_ref assumes VM.get_record raises --- nova/virt/xenapi/vmops.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 30fa5bdd7..c0fbf96fc 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -192,14 +192,12 @@ class VMOps(object): if isinstance(instance_or_vm, basestring): ref = None try: + # check for opaque ref ref = self._session.get_xenapi().VM.get_record(instance_or_vm) - if ref != None: - # an opaque ref was passed in, return it - return instance_or_vm - except: - pass - # wasn't an opaque ref, must be an instance name - instance_name = instance_or_vm + return instance_or_vm + except self.XenAPI.Failure: + # wasn't an opaque ref, must be an instance name + instance_name = instance_or_vm # if instance_or_vm is an int/long it must be instance id elif isinstance(instance_or_vm, (int, long)): -- cgit From 4e8b6a14324ef2d1f550233cbcfc94c6363533d8 Mon Sep 17 00:00:00 2001 From: Trey Morris Date: Mon, 7 Mar 2011 18:46:44 -0600 Subject: virt.xenapi.vmops._get_vm_opaque_ref changed vm to vm_ref and ref to obj --- nova/virt/xenapi/vmops.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c0fbf96fc..0adabe7f5 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -190,10 +190,10 @@ class VMOps(object): """ # if instance_or_vm is a string it must be opaque ref or instance name if isinstance(instance_or_vm, basestring): - ref = None + obj = None try: # check for opaque ref - ref = self._session.get_xenapi().VM.get_record(instance_or_vm) + obj = self._session.get_xenapi().VM.get_record(instance_or_vm) return instance_or_vm except self.XenAPI.Failure: # wasn't an opaque ref, must be an instance name @@ -215,11 +215,11 @@ class VMOps(object): # otherwise instance_or_vm is an instance object else: instance_name = instance_or_vm.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is None: + vm_ref = VMHelper.lookup(self._session, instance_name) + if vm_ref is None: raise exception.NotFound( _('Instance not present %s') % instance_name) - return vm + return vm_ref def _acquire_bootlock(self, vm): """Prevent an instance from booting""" -- cgit From cac5881eaa35f94e004c18dd34ca78014f067976 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Tue, 8 Mar 2011 01:01:41 -0500 Subject: execvp --- nova/crypto.py | 32 +-- nova/network/linux_net.py | 222 +++++++++++---------- nova/tests/test_network.py | 16 +- nova/utils.py | 16 +- nova/virt/disk.py | 44 ++-- nova/virt/images.py | 5 +- nova/virt/libvirt_conn.py | 36 ++-- nova/virt/xenapi/vm_utils.py | 11 +- nova/volume/driver.py | 71 +++---- .../networking/etc/xensource/scripts/vif_rules.py | 91 ++++++--- 10 files changed, 296 insertions(+), 248 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index b240a3958..dd24723b8 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,10 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen','-q','-b',"%d" % bits,'-N','""','-f',keyfile) - (out, err) = utils.execute('ssh-keygen','-q','-l','-f',"%s.pub" % (keyfile)) + utils.execute('ssh-keygen', '-q', '-b', '%d' % bits, '-N', '', + '-f', keyfile) + (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', + '%s.pub' % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +120,7 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen','-y','-f','/dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', '/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,9 +145,10 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute('openssl','ca','-config','./openssl.cnf','-revoke',"'%s'" % file_name) - utils.execute('openssl','ca','-gencrl','-config','./openssl.cnf','-out',"'%s'" % - FLAGS.crl_file) + utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', + '%s' % file_name) + utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', + '-out', '%s' % FLAGS.crl_file) os.chdir(start) @@ -193,8 +196,9 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute('openssl','genrsa','-out',keyfile,bits) - utils.execute('openssl','req','-new','-key',keyfile,'-out',csrfile,'-batch','-subj',subject) + utils.execute('openssl', 'genrsa', '-out', keyfile, bits) + utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, + '-batch', '-subj', subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -211,7 +215,8 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute('sh','geninter.sh',project_id, _project_cert_subject(project_id)) + utils.execute('sh', 'geninter.sh', project_id, + _project_cert_subject(project_id)) os.chdir(start) @@ -226,7 +231,7 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute('sh','genvpn.sh', + utils.execute('sh', 'genvpn.sh', project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() @@ -257,9 +262,10 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute('openssl','ca','-batch','-out',outbound,'-config' - './openssl.cnf','-infiles',inbound) - out, _err = utils.execute('openssl','x509','-in',outbound','-serial','-noout') + utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config', + './openssl.cnf', '-infiles', inbound) + out, _err = utils.execute('openssl', 'x509', '-in', outbound, + '-serial', '-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 535ce87bc..ad019a8c0 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -65,113 +65,117 @@ flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', def metadata_forward(): """Create forwarding rule for metadata""" - _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " - "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + _confirm_rule("PREROUTING", '-t', 'nat', '-s', '0.0.0.0/0', + '-d', '169.254.169.254/32', '-p', 'tcp', '-m', 'tcp', + '--dport', '80', '-j', 'DNAT', + '--to-destination', '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): """Basic networking setup goes here""" if FLAGS.use_nova_chains: - _execute("sudo iptables -N nova_input", check_exit_code=False) - _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, + _execute('sudo', 'iptables', '-N', 'nova_input', check_exit_code=False) + _execute('sudo', 'iptables', '-D', FLAGS.input_chain, + '-j', 'nova_input', check_exit_code=False) - _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) - - _execute("sudo iptables -N nova_forward", check_exit_code=False) - _execute("sudo iptables -D FORWARD -j nova_forward", + _execute('sudo', 'iptables', '-A', FLAGS.input_chain, + '-j', 'nova_input') + _execute('sudo', 'iptables', '-N', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _execute("sudo iptables -N nova_output", check_exit_code=False) - _execute("sudo iptables -D OUTPUT -j nova_output", + _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A OUTPUT -j nova_output") - - _execute("sudo iptables -t nat -N nova_prerouting", + _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') + _execute('sudo', 'iptables', '-N', 'nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-D', 'OUTPUT', '-j', 'nova_output', check_exit_code=False) - _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", + _execute('sudo', 'iptables', '-A', 'OUTPUT', '-j', 'nova_output') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_prerouting', check_exit_code=False) - _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") - - _execute("sudo iptables -t nat -N nova_postrouting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'PREROUTING', + '-j', 'nova_prerouting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'PREROUTING', + '-j', 'nova_prerouting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_postrouting', check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j', 'nova_postrouting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'nova_postrouting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_snatting', check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") - - _execute("sudo iptables -t nat -N nova_snatting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j nova_snatting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'nova_snatting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_output', check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") - - _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) - _execute("sudo iptables -t nat -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -t nat -A OUTPUT -j nova_output") + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'OUTPUT', + '-j nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'OUTPUT', + '-j', 'nova_output') else: # NOTE(vish): This makes it easy to ensure snatting rules always # come after the accept rules in the postrouting chain - _execute("sudo iptables -t nat -N SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'SNATTING', check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j', 'SNATTING', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'SNATTING') # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("SNATTING", "-t nat -s %s " - "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) + _confirm_rule("SNATTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-j', 'SNAT', '--to-source', FLAGS.routing_source_ip, + append=True) - _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % - (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % - {'range': FLAGS.fixed_range}) + _confirm_rule("POSTROUTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-d', FLAGS.dmz_cidr, '-j', 'ACCEPT') + _confirm_rule("POSTROUTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-d', FLAGS.fixed_range, '-j', 'ACCEPT') def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" - _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface), + _execute('sudo', 'ip', 'addr', 'add', floating_ip, + 'dev', FLAGS.public_interface), check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" - _execute("sudo ip addr del %s dev %s" % (floating_ip, - FLAGS.public_interface)) + _execute('sudo', 'ip', 'addr', 'del', floating_ip, + 'dev', FLAGS.public_interface)) def ensure_vlan_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan""" - _confirm_rule("FORWARD", "-d %s -p udp --dport 1194 -j ACCEPT" % - private_ip) - _confirm_rule("PREROUTING", - "-t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (public_ip, port, private_ip)) + _confirm_rule("FORWARD", '-d', private_ip, '-p', 'udp', + '--dport', '1194', '-j', 'ACCEPT') + _confirm_rule("PREROUTING", '-t', 'nat', '-d', public_ip, '-p', 'udp', + '--dport', port, '-j', 'DNAT', '--to', '%s:1194' + % private_ip) def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" - _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + _confirm_rule("PREROUTING", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _confirm_rule("OUTPUT", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _confirm_rule("SNATTING", '-t', 'nat', '-s', fixed_ip, '-j', 'SNAT', + '--to', floating_ip) def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" - _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + _remove_rule("PREROUTING", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _remove_rule("OUTPUT", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _remove_rule("SNATTING", '-t', 'nat', '-s', fixed_ip, '-j', 'SNAT', + '--to', floating_ip) def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): @@ -185,9 +189,9 @@ def ensure_vlan(vlan_num): interface = "vlan%s" % vlan_num if not _device_exists(interface): LOG.debug(_("Starting VLAN inteface %s"), interface) - _execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") - _execute("sudo vconfig add %s %s" % (FLAGS.vlan_interface, vlan_num)) - _execute("sudo ip link set %s up" % interface) + _execute('sudo', 'vconfig', 'set_name_type', 'VLAN_PLUS_VID_NO_PAD') + _execute('sudo', 'vconfig', 'add', FLAGS.vlan_interface, vlan_num) + _execute('sudo', 'ip', 'link', 'set', interface, 'up') return interface @@ -206,52 +210,54 @@ def ensure_bridge(bridge, interface, net_attrs=None): """ if not _device_exists(bridge): LOG.debug(_("Starting Bridge interface for %s"), interface) - _execute("sudo brctl addbr %s" % bridge) - _execute("sudo brctl setfd %s 0" % bridge) + _execute('sudo', 'brctl', 'addbr', bridge) + _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute("sudo brctl stp %s off" % bridge) - _execute("sudo ip link set %s up" % bridge) + _execute('sudo', 'brctl', 'stp', bridge', 'off') + _execute('sudo', 'ip', 'link', 'set', bridge, up) if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] - out, err = _execute("sudo ip addr add %s/%s brd %s dev %s" % - (net_attrs['gateway'], - suffix, - net_attrs['broadcast'], - bridge), + out, err = _execute('sudo', 'ip', 'addr', 'add', + "%s/%s" % + (net_attrs['gateway'], suffix), + 'brd', + net-attrs['broadcast'], + 'dev', + bridge, check_exit_code=False) if err and err != "RTNETLINK answers: File exists\n": raise exception.Error("Failed to add ip: %s" % err) if(FLAGS.use_ipv6): - _execute("sudo ip -f inet6 addr change %s dev %s" % - (net_attrs['cidr_v6'], bridge)) + _execute('sudo', 'ip', '-f', 'inet6', 'addr', + 'change', net_attrs['cidr_v6'], + 'dev', bridge) # NOTE(vish): If the public interface is the same as the # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute("sudo ip link set dev %s promisc on" % bridge) + _execute('sudo', 'ip', 'link', 'set', 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge gateway = None - out, err = _execute("sudo route -n") + out, err = _execute('sudo', 'route', '-n') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "0.0.0.0" and fields[-1] == interface: gateway = fields[1] - out, err = _execute("sudo ip addr show dev %s scope global" % - interface) + out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, + 'scope', 'global') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "inet": params = ' '.join(fields[1:-1]) - _execute("sudo ip addr del %s dev %s" % (params, fields[-1])) - _execute("sudo ip addr add %s dev %s" % (params, bridge)) + _execute('sudo', 'ip', 'addr', 'del', params, 'dev', fields[-1]) + _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) if gateway: - _execute("sudo route add 0.0.0.0 gw %s" % gateway) - out, err = _execute("sudo brctl addif %s %s" % - (bridge, interface), + _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) + out, err = _execute('sudo', 'brctl', 'addif, bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " @@ -259,18 +265,18 @@ def ensure_bridge(bridge, interface, net_attrs=None): raise exception.Error("Failed to add interface: %s" % err) if FLAGS.use_nova_chains: - (out, err) = _execute("sudo iptables -N nova_forward", + (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward, check_exit_code=False) if err != 'iptables: Chain already exists.\n': # NOTE(vish): chain didn't exist link chain - _execute("sudo iptables -D FORWARD -j nova_forward", + _execute('sudo', 'iptables, '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") + _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') - _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) - _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) - _execute("sudo iptables -N nova-local", check_exit_code=False) - _confirm_rule("FORWARD", "-j nova-local") + _confirm_rule("FORWARD", '--in-interface', bridge, '-j', 'ACCEPT') + _confirm_rule("FORWARD", '--out-interface', bridge, '-j', 'ACCEPT') + _execute('sudo', 'iptables', '-N', 'nova-local', check_exit_code=False) + _confirm_rule("FORWARD", '-j', 'nova-local') def get_dhcp_hosts(context, network_id): @@ -304,11 +310,11 @@ def update_dhcp(context, network_id): # if dnsmasq is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' % pid, + out, _err = _execute('cat', "/proc/%d/cmdline" % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill -HUP %d' % pid) + _execute('sudo', 'kill', '-HUP', pid) return except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Hupping dnsmasq threw %s"), exc) @@ -349,11 +355,11 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' + out, _err = _execute('cat', "/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill %d' % pid) + _execute('sudo', 'kill', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("killing radvd threw %s"), exc) else: @@ -374,23 +380,23 @@ def _host_dhcp(fixed_ip_ref): fixed_ip_ref['address']) -def _execute(cmd, *args, **kwargs): +def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", cmd) + LOG.debug("FAKE NET: %s", ' '.join(cmd)) return "fake", 0 else: - return utils.execute(cmd, *args, **kwargs) + return utils.execute(*cmd, **kwargs) def _device_exists(device): """Check if ethernet device exists""" - (_out, err) = _execute("ip link show dev %s" % device, + (_out, err) = _execute('ip', 'link', 'show', 'dev', device, check_exit_code=False) return not err -def _confirm_rule(chain, cmd, append=False): +def _confirm_rule(chain, *cmd, append=False): """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() @@ -398,16 +404,16 @@ def _confirm_rule(chain, cmd, append=False): loc = "-A" else: loc = "-I" - _execute("sudo iptables --delete %s %s" % (chain, cmd), + _execute('sudo', 'iptables', '--delete', chain, *cmd, check_exit_code=False) - _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) + _execute('sudo', 'iptables', loc, chain, *cmd) -def _remove_rule(chain, cmd): +def _remove_rule(chain, *cmd): """Remove iptables rule""" if FLAGS.use_nova_chains: chain = "%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd)) + _execute('sudo', 'iptables', '--delete', chain, *cmd) def _dnsmasq_cmd(net): @@ -444,7 +450,7 @@ def _stop_dnsmasq(network): if pid: try: - _execute('sudo kill -TERM %d' % pid) + _execute('sudo', 'kill', '-TERM', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Killing dnsmasq threw %s"), exc) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index ce1c77210..6d2d8b771 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,13 +343,13 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'add' + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("ISSUE_IP: %s, %s ", out, err) @@ -359,11 +359,11 @@ def release_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'del', + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("RELEASE_IP: %s, %s ", out, err) diff --git a/nova/utils.py b/nova/utils.py index 40a8d8d8c..c96b85294 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,15 +125,15 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl","--fail",url,"-o",target) + execute("curl", "--fail", url, "-o", target) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) +def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, + obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -148,7 +148,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) # NOTE(termie): this appears to be necessary to let the subprocess call # clean something up in between calls, without it two # execute calls in a row hangs the second one @@ -158,7 +158,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (SSH): %s"), cmd) + LOG.debug(_("Running cmd (SSH): %s"), ' '.join(cmd)) if addl_env: raise exception.Error("Environment not supported over SSH") @@ -187,7 +187,7 @@ def ssh_execute(ssh, cmd, process_input=None, raise exception.ProcessExecutionError(exit_code=exit_status, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) return (stdout, stderr) @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip","-f","inet6","-o","addr","show", interface) + if_str = execute("ip", "-f", "inet6", "-o", "addr", "show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 2bded07a4..203517275 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -49,10 +49,10 @@ def extend(image, size): file_size = os.path.getsize(image) if file_size >= size: return - utils.execute('truncate -s %s %s' % (size, image)) + utils.execute('truncate', '-s', size, image) # NOTE(vish): attempts to resize filesystem - utils.execute('e2fsck -fp %s' % image, check_exit_code=False) - utils.execute('resize2fs %s' % image, check_exit_code=False) + utils.execute('e2fsck', '-fp', mage, check_exit_code=False) + utils.execute('resize2fs', image, check_exit_code=False) def inject_data(image, key=None, net=None, partition=None, nbd=False): @@ -68,7 +68,7 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): try: if not partition is None: # create partition - out, err = utils.execute('sudo kpartx -a %s' % device) + out, err = utils.execute('sudo', 'kpartx', '-a', device) if err: raise exception.Error(_('Failed to load partition: %s') % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], @@ -84,13 +84,14 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): mapped_device) # Configure ext2fs so that it doesn't auto-check every N boots - out, err = utils.execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + out, err = utils.execute('sudo', 'tune2fs', + '-c', 0, '-i', 0, mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir out, err = utils.execute( - 'sudo mount %s %s' % (mapped_device, tmpdir)) + 'sudo', 'mount', mapped_device, tmpdir) if err: raise exception.Error(_('Failed to mount filesystem: %s') % err) @@ -103,13 +104,13 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): _inject_net_into_fs(net, tmpdir) finally: # unmount device - utils.execute('sudo umount %s' % mapped_device) + utils.execute('sudo', 'umount', mapped_device) finally: # remove temporary directory - utils.execute('rmdir %s' % tmpdir) + utils.execute('rmdir', tmpdir) if not partition is None: # remove partitions - utils.execute('sudo kpartx -d %s' % device) + utils.execute('sudo', 'kpartx', '-d', device) finally: _unlink_device(device, nbd) @@ -118,7 +119,7 @@ def _link_device(image, nbd): """Link image to device using loopback or nbd""" if nbd: device = _allocate_device() - utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) + utils.execute('sudo', 'qemu-nbd', '-c', device, image) # NOTE(vish): this forks into another process, so give it a chance # to set up before continuuing for i in xrange(FLAGS.timeout_nbd): @@ -127,7 +128,7 @@ def _link_device(image, nbd): time.sleep(1) raise exception.Error(_('nbd device %s did not show up') % device) else: - out, err = utils.execute('sudo losetup --find --show %s' % image) + out, err = utils.execute('sudo', 'losetup', '--find', '--show', image) if err: raise exception.Error(_('Could not attach image to loopback: %s') % err) @@ -137,10 +138,10 @@ def _link_device(image, nbd): def _unlink_device(device, nbd): """Unlink image from device using loopback or nbd""" if nbd: - utils.execute('sudo qemu-nbd -d %s' % device) + utils.execute('sudo', 'qemu-nbd', '-d', device) _free_device(device) else: - utils.execute('sudo losetup --detach %s' % device) + utils.execute('sudo', 'losetup', '--detach', device) _DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.max_nbd_devices)] @@ -170,11 +171,12 @@ def _inject_key_into_fs(key, fs): fs is the path to the base of the filesystem into which to inject the key. """ sshdir = os.path.join(fs, 'root', '.ssh') - utils.execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - utils.execute('sudo chown root %s' % sshdir) - utils.execute('sudo chmod 700 %s' % sshdir) + utils.execute('sudo', 'mkdir', '-p', sshdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root', sshdir) + utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - utils.execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + # TODO:EWINDISCH: not sure about the following /w execv patch + utils.execute('sudo', 'tee', '-a', keyfile, '\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): @@ -183,8 +185,8 @@ def _inject_net_into_fs(net, fs): net is the contents of /etc/network/interfaces. """ netdir = os.path.join(os.path.join(fs, 'etc'), 'network') - utils.execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter - utils.execute('sudo chown root:root %s' % netdir) - utils.execute('sudo chmod 755 %s' % netdir) + utils.execute('sudo', 'mkdir', '-p', netdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root:root', netdir) + utils.execute('sudo', 'chmod', 755, netdir) netfile = os.path.join(netdir, 'interfaces') - utils.execute('sudo tee %s' % netfile, net) + utils.execute('sudo', 'tee', netfile, net) diff --git a/nova/virt/images.py b/nova/virt/images.py index 7a6fef330..4b11d1667 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -94,8 +94,7 @@ def _fetch_s3_image(image, path, user, project): cmd += ['-H', '\'%s: %s\'' % (k, v)] cmd += ['-o', path] - cmd_out = ' '.join(cmd) - return utils.execute(cmd_out) + return utils.execute(*cmd) def _fetch_local_image(image, path, user, project): @@ -103,7 +102,7 @@ def _fetch_local_image(image, path, user, project): if sys.platform.startswith('win'): return shutil.copy(source, path) else: - return utils.execute('cp %s %s' % (source, path)) + return utils.execute('cp', source, path) def _image_path(path): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4e0fd106f..464ec475c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -438,8 +438,10 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): LOG.info(_("cool, it's a device")) - out, err = utils.execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) + out, err = utils.execute('sudo', 'dd', + "if=%s" % virsh_output, + 'iflag=nonblock', + check_exit_code=False) return out else: return '' @@ -461,11 +463,11 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) + utils.execute('sudo', 'chown', s.getuid(), console_log) if FLAGS.libvirt_type == 'xen': # Xen is special - virsh_output = utils.execute("virsh ttyconsole %s" % + virsh_output = utils.execute('virsh', 'ttyconsole', instance['name']) data = self._flush_xen_console(virsh_output) fpath = self._append_to_file(data, console_log) @@ -482,7 +484,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - cmd = 'netcat 0.0.0.0 %s -w 1 Date: Tue, 8 Mar 2011 01:08:13 -0500 Subject: Fix todo comment --- nova/virt/libvirt_conn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e1cd75306..a5256bbc2 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -486,8 +486,9 @@ class LibvirtConnection(object): # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - # TODO:ewindisch: subprocess lets us do this... - # but utils.execute abstracts it away from us + # TODO(ewindisch): broken /w execvp patch. + # subprocess lets us do this, but utils.execute + # abstracts it away from us cmd = 'netcat', '0.0.0.0', port, '-w', '1', ' Date: Tue, 8 Mar 2011 14:35:53 +0000 Subject: * pep8 cleanups in migrations * a few bugfixes --- .../sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py | 5 +---- nova/tests/test_xenapi.py | 4 ++-- nova/virt/xenapi/vm_utils.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py index a50f31e16..514b92b81 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py @@ -29,8 +29,6 @@ instances = Table('instances', meta, Column('id', Integer(), primary_key=True, nullable=False), ) -# FIXME(dubs) should this be not null? Maybe create as nullable, then -# populate all existing rows with 'linux', then adding not null constraint. instances_os_type = Column('os_type', String(length=255, convert_unicode=False, assert_unicode=None, unicode_error=None, @@ -45,7 +43,7 @@ def upgrade(migrate_engine): instances.create_column(instances_os_type) migrate_engine.execute(instances.update()\ - .where(instances.c.os_type==None)\ + .where(instances.c.os_type == None)\ .values(os_type='linux')) @@ -53,4 +51,3 @@ def downgrade(migrate_engine): meta.bind = migrate_engine instances.drop_column('os_type') - diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 27f0e5dd7..25070e108 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -251,7 +251,7 @@ class XenAPIVMTestCase(test.TestCase): self.vm_info = vm_info self.vm = vm - def check_vm_record(self): + def check_vm_record(self, conn): # Check that m1.large above turned into the right thing. instance_type = db.instance_type_get_by_name(conn, 'm1.large') mem_kib = long(instance_type['memory_mb']) << 10 @@ -321,7 +321,7 @@ class XenAPIVMTestCase(test.TestCase): instance = db.instance_create(values) conn.spawn(instance) self.create_vm_record(conn, os_type) - self.check_vm_record() + self.check_vm_record(conn) def test_spawn_not_enough_memory(self): FLAGS.xenapi_image_service = 'glance' diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 150824400..604e8a4e0 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -324,7 +324,7 @@ class VMHelper(HelperBase): kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) - session.wait_for_task(task, instance_id) + session.wait_for_task(task, instance.id) @classmethod def fetch_image(cls, session, instance_id, image, user, project, -- cgit From cbc2956a4e863c1bc952c7cef6045c39d293818d Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 8 Mar 2011 17:18:13 +0000 Subject: Remove addition of account to service url. --- nova/api/openstack/__init__.py | 24 +++------------- nova/api/openstack/auth.py | 46 ++++++----------------------- nova/auth/novarc.template | 2 +- nova/tests/api/openstack/test_accounts.py | 8 +++--- nova/tests/api/openstack/test_adminapi.py | 4 +-- nova/tests/api/openstack/test_auth.py | 2 +- nova/tests/api/openstack/test_flavors.py | 4 +-- nova/tests/api/openstack/test_images.py | 4 +-- nova/tests/api/openstack/test_servers.py | 48 +++++++++++++++---------------- nova/tests/api/openstack/test_users.py | 10 +++---- nova/tests/api/openstack/test_zones.py | 10 +++---- 11 files changed, 59 insertions(+), 103 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 005d330a6..a655b1c85 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -73,18 +73,6 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - accounts_controller = accounts.Controller() - mapper.connect("account", "/{id}", - controller=accounts_controller, action="show", - conditions=dict(method=["GET"])) - if FLAGS.allow_admin_api: - mapper.connect("/{id}", - controller=accounts_controller, action="update", - conditions=dict(method=["PUT"])) - mapper.connect("/{id}", - controller=accounts_controller, action="delete", - conditions=dict(method=["DELETE"])) - server_members = {'action': 'POST'} if FLAGS.allow_admin_api: LOG.debug(_("Including admin operations in API.")) @@ -101,38 +89,34 @@ class APIRouter(wsgi.Router): server_members['inject_network_info'] = 'POST' mapper.resource("zone", "zones", controller=zones.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("user", "users", controller=users.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) + mapper.resource("account", "accounts", + controller=accounts.Controller(), + collection={'detail': 'GET'}) + mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, - path_prefix="{account_id}/", member=server_members) mapper.resource("backup_schedule", "backup_schedule", controller=backup_schedules.Controller(), - path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("console", "consoles", controller=consoles.Controller(), - path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("image", "images", controller=images.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("flavor", "flavors", controller=flavors.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("shared_ip_group", "shared_ip_groups", - path_prefix="{account_id}/", collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index e77910fed..e71fc69e3 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -53,19 +53,15 @@ class AuthMiddleware(wsgi.Middleware): if not self.has_authentication(req): return self.authenticate(req) user = self.get_user_by_authentication(req) - account_name = req.path_info_peek() - + accounts = self.auth.get_projects(user=user) if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - if not account_name: - if self.auth.is_admin(user): - account_name = FLAGS.default_project - else: - return faults.Fault(webob.exc.HTTPUnauthorized()) - try: - account = self.auth.get_project(account_name) - except exception.NotFound: + if accounts: + #we are punting on this til auth is settled, + #and possibly til api v1.1 (mdragon) + account = accounts[0] + else: return faults.Fault(webob.exc.HTTPUnauthorized()) if not self.auth.is_admin(user) and \ @@ -85,7 +81,6 @@ class AuthMiddleware(wsgi.Middleware): # Unless the request is explicitly made against // don't # honor it path_info = req.path_info - account_name = None if len(path_info) > 1: return faults.Fault(webob.exc.HTTPUnauthorized()) @@ -95,10 +90,7 @@ class AuthMiddleware(wsgi.Middleware): except KeyError: return faults.Fault(webob.exc.HTTPUnauthorized()) - if ':' in username: - account_name, username = username.rsplit(':', 1) - - token, user = self._authorize_user(username, account_name, key, req) + token, user = self._authorize_user(username, key, req) if user and token: res = webob.Response() res.headers['X-Auth-Token'] = token.token_hash @@ -135,31 +127,15 @@ class AuthMiddleware(wsgi.Middleware): return self.auth.get_user(token.user_id) return None - def _authorize_user(self, username, account_name, key, req): + def _authorize_user(self, username, key, req): """Generates a new token and assigns it to a user. username - string - account_name - string key - string API key req - webob.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) - if account_name: - try: - account = self.auth.get_project(account_name) - except exception.NotFound: - return None, None - else: - # (dragondm) punt and try to determine account. - # this is something of a hack, but a user on 1 account is a - # common case, and is the way the current RS code works. - accounts = self.auth.get_projects(user=user) - if len(accounts) == 1: - account = accounts[0] - else: - #we can't tell what account they are logging in for. - return None, None if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, @@ -167,11 +143,7 @@ class AuthMiddleware(wsgi.Middleware): token_dict = {} token_dict['token_hash'] = token_hash token_dict['cdn_management_url'] = '' - # auth url + project (account) id, e.g. - # http://foo.org:8774/baz/v1.0/myacct/ - os_url = '%s%s%s/' % (req.url, - '' if req.url.endswith('/') else '/', - account.id) + os_url = req.url token_dict['server_management_url'] = os_url token_dict['storage_url'] = '' token_dict['user_id'] = user.id diff --git a/nova/auth/novarc.template b/nova/auth/novarc.template index 1c917ad44..cda2ecc28 100644 --- a/nova/auth/novarc.template +++ b/nova/auth/novarc.template @@ -11,5 +11,5 @@ export EUCALYPTUS_CERT=${NOVA_CERT} # euca-bundle-image seems to require this se alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_PRIVATE_KEY} --user 42 --ec2cert ${NOVA_CERT}" alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}" export NOVA_API_KEY="%(access)s" -export NOVA_USERNAME="%(project)s:%(user)s" +export NOVA_USERNAME="%(user)s" export NOVA_URL="%(os)s" diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py index b2e89824a..746f02f57 100644 --- a/nova/tests/api/openstack/test_accounts.py +++ b/nova/tests/api/openstack/test_accounts.py @@ -70,7 +70,7 @@ class AccountsTest(test.TestCase): super(AccountsTest, self).tearDown() def test_get_account(self): - req = webob.Request.blank('/v1.0/test1') + req = webob.Request.blank('/v1.0/accounts/test1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -80,7 +80,7 @@ class AccountsTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_account_delete(self): - req = webob.Request.blank('/v1.0/test1') + req = webob.Request.blank('/v1.0/accounts/test1') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertTrue('test1' not in fakes.FakeAuthManager.projects) @@ -89,7 +89,7 @@ class AccountsTest(test.TestCase): def test_account_create(self): body = dict(account=dict(description='test account', manager='guy1')) - req = webob.Request.blank('/v1.0/newacct') + req = webob.Request.blank('/v1.0/accounts/newacct') req.method = 'PUT' req.body = json.dumps(body) @@ -108,7 +108,7 @@ class AccountsTest(test.TestCase): def test_account_update(self): body = dict(account=dict(description='test account', manager='guy2')) - req = webob.Request.blank('/v1.0/test1') + req = webob.Request.blank('/v1.0/accounts/test1') req.method = 'PUT' req.body = json.dumps(body) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 7cb9e8450..4568cb9f5 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -50,7 +50,7 @@ class AdminAPITest(test.TestCase): def test_admin_enabled(self): FLAGS.allow_admin_api = True # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/testacct/flavors') + req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are available. @@ -58,7 +58,7 @@ class AdminAPITest(test.TestCase): def test_admin_disabled(self): FLAGS.allow_admin_api = False # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/testacct/flavors') + req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) # TODO: Confirm admin operations are unavailable. self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 8268a6fb9..49f90879d 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -78,7 +78,7 @@ class Test(test.TestCase): self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], - "http://foo/v1.0/test/") + "http://foo/v1.0/") self.assertEqual(result.headers['X-CDN-Management-Url'], "") self.assertEqual(result.headers['X-Storage-Url'], "") diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index ba0785b0e..8280a505f 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -42,11 +42,11 @@ class FlavorsTest(test.TestCase): super(FlavorsTest, self).tearDown() def test_get_flavor_list(self): - req = webob.Request.blank('/v1.0/testacct/flavors') + req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) def test_get_flavor_by_id(self): - req = webob.Request.blank('/v1.0/testacct/flavors/1') + req = webob.Request.blank('/v1.0/flavors/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 819ca001e..dbe507f7d 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -216,7 +216,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): super(ImageControllerWithGlanceServiceTest, self).tearDown() def test_get_image_index(self): - req = webob.Request.blank('/v1.0/testacct/images') + req = webob.Request.blank('/v1.0/images') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -228,7 +228,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): "image %s not in fixture index!" % str(image)) def test_get_image_details(self): - req = webob.Request.blank('/v1.0/testacct/images/detail') + req = webob.Request.blank('/v1.0/images/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index d592e06b0..705a2f800 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -150,7 +150,7 @@ class ServersTest(test.TestCase): super(ServersTest, self).tearDown() def test_get_server_by_id(self): - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -161,7 +161,7 @@ class ServersTest(test.TestCase): public = ["1.2.3.4"] new_return_server = return_server_with_addresses(private, public) self.stubs.Set(nova.db.api, 'instance_get', new_return_server) - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -173,7 +173,7 @@ class ServersTest(test.TestCase): self.assertEqual(addresses["private"][0], private) def test_get_server_list(self): - req = webob.Request.blank('/v1.0/testacct/servers') + req = webob.Request.blank('/v1.0/servers') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -224,7 +224,7 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={'hello': 'world', 'open': 'stack'}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers') + req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) @@ -233,7 +233,7 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_update_no_body(self): - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 422) @@ -251,7 +251,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) @@ -267,30 +267,30 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) def test_create_backup_schedules(self): - req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'POST' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): - req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): - req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/servers/1/backup_schedules') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): - req = webob.Request.blank('/v1.0/testacct/servers/detail') + req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -321,7 +321,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers_with_host) - req = webob.Request.blank('/v1.0/testacct/servers/detail') + req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -341,7 +341,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/pause') + req = webob.Request.blank('/v1.0/servers/1/pause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -353,7 +353,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/unpause') + req = webob.Request.blank('/v1.0/servers/1/unpause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -365,7 +365,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/suspend') + req = webob.Request.blank('/v1.0/servers/1/suspend') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -377,7 +377,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/resume') + req = webob.Request.blank('/v1.0/servers/1/resume') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -389,7 +389,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/reset_network') + req = webob.Request.blank('/v1.0/servers/1/reset_network') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -402,7 +402,7 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) req = webob.Request.blank( - '/v1.0/testacct/servers/1/inject_network_info') + '/v1.0/servers/1/inject_network_info') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -410,13 +410,13 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 202) def test_server_diagnostics(self): - req = webob.Request.blank("/v1.0/testacct/servers/1/diagnostics") + req = webob.Request.blank("/v1.0/servers/1/diagnostics") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_actions(self): - req = webob.Request.blank("/v1.0/testacct/servers/1/actions") + req = webob.Request.blank("/v1.0/servers/1/actions") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) @@ -425,7 +425,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/action') + req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -435,7 +435,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/action') + req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -445,14 +445,14 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/action') + req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) def test_delete_server_instance(self): - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'DELETE' self.server_delete_called = False diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py index bd32254cd..14c7897f0 100644 --- a/nova/tests/api/openstack/test_users.py +++ b/nova/tests/api/openstack/test_users.py @@ -72,7 +72,7 @@ class UsersTest(test.TestCase): super(UsersTest, self).tearDown() def test_get_user_list(self): - req = webob.Request.blank('/v1.0/testacct/users') + req = webob.Request.blank('/v1.0/users') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -80,7 +80,7 @@ class UsersTest(test.TestCase): self.assertEqual(len(res_dict['users']), 2) def test_get_user_by_id(self): - req = webob.Request.blank('/v1.0/testacct/users/guy2') + req = webob.Request.blank('/v1.0/users/guy2') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -91,7 +91,7 @@ class UsersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_user_delete(self): - req = webob.Request.blank('/v1.0/testacct/users/guy1') + req = webob.Request.blank('/v1.0/users/guy1') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertTrue('guy1' not in [u.id for u in @@ -103,7 +103,7 @@ class UsersTest(test.TestCase): access='acc3', secret='invasionIsInNormandy', admin=True)) - req = webob.Request.blank('/v1.0/testacct/users') + req = webob.Request.blank('/v1.0/users') req.method = 'POST' req.body = json.dumps(body) @@ -124,7 +124,7 @@ class UsersTest(test.TestCase): body = dict(user=dict(name='guy2', access='acc2', secret='invasionIsInNormandy')) - req = webob.Request.blank('/v1.0/testacct/users/guy2') + req = webob.Request.blank('/v1.0/users/guy2') req.method = 'PUT' req.body = json.dumps(body) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 51f13af48..6d869dc15 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -85,7 +85,7 @@ class ZonesTest(test.TestCase): super(ZonesTest, self).tearDown() def test_get_zone_list(self): - req = webob.Request.blank('/v1.0/testacct/zones') + req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -93,7 +93,7 @@ class ZonesTest(test.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): - req = webob.Request.blank('/v1.0/testacct/zones/1') + req = webob.Request.blank('/v1.0/zones/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -103,7 +103,7 @@ class ZonesTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_zone_delete(self): - req = webob.Request.blank('/v1.0/testacct/zones/1') + req = webob.Request.blank('/v1.0/zones/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -111,7 +111,7 @@ class ZonesTest(test.TestCase): def test_zone_create(self): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) - req = webob.Request.blank('/v1.0/testacct/zones') + req = webob.Request.blank('/v1.0/zones') req.method = 'POST' req.body = json.dumps(body) @@ -125,7 +125,7 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) - req = webob.Request.blank('/v1.0/testacct/zones/1') + req = webob.Request.blank('/v1.0/zones/1') req.method = 'PUT' req.body = json.dumps(body) -- cgit From e81294b94e3bc8708bd4777b685a7d302594557e Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Tue, 8 Mar 2011 18:53:20 +0100 Subject: Added ability to remove networks on nova-manage command --- bin/nova-manage | 6 +++++- nova/db/api.py | 7 +++++++ nova/db/sqlalchemy/api.py | 7 +++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 94b0d5946..8ddfea5c2 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -548,7 +548,11 @@ class NetworkCommands(object): def delete(self, fixed_range): """Deletes a network""" network = db.network_get_by_cidr(context.get_admin_context(), fixed_range) - + if network.project_id is not None: + raise ValueError(_('Network must be disassociated from project %s' + ' before delete' %network.project_id)) + db.network_delete_safe(context.get_admin_context(), network.id) + class ServiceCommands(object): """Enable and disable running services""" diff --git a/nova/db/api.py b/nova/db/api.py index 04f5fd72f..7ad99c1f4 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -488,6 +488,13 @@ def network_create_safe(context, values): """ return IMPL.network_create_safe(context, values) +def network_delete_safe(context, network_id): + """Delete network with key network_id + + This method assumes that the network is not associated with any project + """ + return IMPL.network_delete_safe(context, network_id) + def network_create_fixed_ips(context, network_id, num_vpn_clients): """Create the ips for the network, reserving sepecified ips.""" diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index c8f42425d..90730d325 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1042,6 +1042,13 @@ def network_create_safe(context, values): except IntegrityError: return None +@require_admin_context +def network_delete_safe(context, network_id): + session = get_session() + with session.begin(): + network_ref = network_get(context, network_id=network_id, session=session) + session.delete(network_ref) + @require_admin_context def network_disassociate(context, network_id): -- cgit From 1caceddf431a1ad1ef22235c2206bccf39fde5c5 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Tue, 8 Mar 2011 14:24:01 -0600 Subject: Nits --- nova/virt/xenapi/vmops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 7fe1f6ff0..86dbf251b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -90,7 +90,7 @@ class VMOps(object): vm = VMHelper.lookup(self._session, instance_name) if vm is not None: raise exception.Duplicate(_('Attempted to create' - ' non-unique name %s') % instance_name) + ' non-unique name %s') % instance_name) #ensure enough free memory is available if not VMHelper.ensure_free_mem(self._session, instance): @@ -104,7 +104,7 @@ class VMOps(object): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - vdi_ref = kernel = ramdisk = pv_kernel = None + kernel = ramdisk = pv_kernel = None # Are we building from a pre-existing disk? vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) -- cgit From 4517117a71c03526aca8f245a70760c45e5214c0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 8 Mar 2011 20:24:48 +0000 Subject: modify nova manage doc --- doc/source/runnova/nova.manage.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/source/runnova/nova.manage.rst b/doc/source/runnova/nova.manage.rst index 0e9a29b6b..0636e5752 100644 --- a/doc/source/runnova/nova.manage.rst +++ b/doc/source/runnova/nova.manage.rst @@ -182,6 +182,29 @@ Nova Floating IPs Displays a list of all floating IP addresses. +Nova Images +~~~~~~~~~~~ + +``nova-manage image image_register `` + + Registers an image with the image service. + +``nova-manage image kernel_register `` + + Registers a kernel with the image service. + +``nova-manage image ramdisk_register `` + + Registers a ramdisk with the image service. + +``nova-manage image all_register `` + + Registers an image kernel and ramdisk with the image service. + +``nova-manage image convert `` + + Converts all images in directory from the old (Bexar) format to the new format. + Concept: Flags -------------- -- cgit From 23d3be4b6f28359211e29212867157daeac9e142 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 8 Mar 2011 20:25:05 +0000 Subject: update code to work with new container and disk formats from glance --- bin/nova-manage | 45 ++++++++++++++++++++++++++++++--------------- nova/api/ec2/cloud.py | 9 ++++++--- nova/image/s3.py | 7 ++++++- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b97d8b81d..b8e0563c7 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -96,6 +96,7 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DECLARE('images_path', 'nova.image.local') flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) @@ -743,17 +744,20 @@ class ImageCommands(object): def __init__(self, *args, **kwargs): self.image_service = utils.import_object(FLAGS.image_service) - def _register(self, image_type, path, owner, name=None, - is_public='T', architecture='x86_64', - kernel_id=None, ramdisk_id=None): - meta = {'type': image_type, - 'is_public': True, + def _register(self, image_type, disk_format, container_format, + path, owner, name=None, is_public='T', + architecture='x86_64', kernel_id=None, ramdisk_id=None): + meta = {'is_public': True, 'name': name, + 'disk_format': disk_format, + 'container_format': container_format, 'properties': {'image_state': 'available', 'owner': owner, + 'type': image_type, 'architecture': architecture, 'image_location': 'local', 'is_public': (is_public == 'T')}} + print image_type, meta if kernel_id: meta['properties']['kernel_id'] = int(kernel_id) if ramdisk_id: @@ -781,28 +785,31 @@ class ImageCommands(object): architecture, kernel_id, ramdisk_id) def image_register(self, path, owner, name=None, is_public='T', - architecture='x86_64', kernel_id=None, ramdisk_id=None): + architecture='x86_64', kernel_id=None, ramdisk_id=None, + disk_format='ami', container_format='ami'): """Uploads an image into the image_service arguments: path owner [name] [is_public='T'] [architecture='x86_64'] - [kernel_id] [ramdisk_id]""" - return self._register('machine', path, owner, name, is_public, - architecture, kernel_id, ramdisk_id) + [kernel_id=None] [ramdisk_id=None] + [disk_format='ami'] [container_format='ami']""" + return self._register('machine', disk_format, container_format, path, + owner, name, is_public, architecture, + kernel_id, ramdisk_id) def kernel_register(self, path, owner, name=None, is_public='T', architecture='x86_64'): """Uploads a kernel into the image_service arguments: path owner [name] [is_public='T'] [architecture='x86_64'] """ - return self._register('kernel', path, owner, name, is_public, - architecture) + return self._register('kernel', 'aki', 'aki', path, owner, name, + is_public, architecture) def ramdisk_register(self, path, owner, name=None, is_public='T', architecture='x86_64'): """Uploads a ramdisk into the image_service arguments: path owner [name] [is_public='T'] [architecture='x86_64'] """ - return self._register('ramdisk', path, owner, name, is_public, - architecture) + return self._register('ramdisk', 'ari', 'ari', path, owner, name, + is_public, architecture) def _lookup(self, old_image_id): try: @@ -813,12 +820,19 @@ class ImageCommands(object): return image['id'] def _old_to_new(self, old): - new = {'type': old['type'], + mapping = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + container_format = mapping[old['type']] + disk_format = container_format + new = {'disk_format': disk_format, + 'container_format': container_format, 'is_public': True, 'name': old['imageId'], 'properties': {'image_state': old['imageState'], 'owner': old['imageOwnerId'], 'architecture': old['architecture'], + 'type': old['type'], 'image_location': old['imageLocation'], 'is_public': old['isPublic']}} if old.get('kernelId'): @@ -851,7 +865,8 @@ class ImageCommands(object): # to move the files out of the way before importing # so we aren't writing to the same directory. This # may fail if the dir was a mointpoint. - if directory == os.path.abspath(FLAGS.images_path): + if (FLAGS.image_service == 'nova.image.local.LocalImageService' + and directory == os.path.abspath(FLAGS.images_path)): new_dir = "%s_bak" % directory os.move(directory, new_dir) os.mkdir(directory) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6479c9445..6b79f7f53 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -867,7 +867,8 @@ class CloudController(object): def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - ec2_id = self._image_ec2_id(image.get('id'), image.get('type')) + image_type = image['properties'].get('type') + ec2_id = self._image_ec2_id(image.get('id'), image_type) name = image.get('name') if name: i['imageId'] = "%s (%s)" % (ec2_id, name) @@ -882,7 +883,7 @@ class CloudController(object): i['imageOwnerId'] = image['properties'].get('owner_id') i['imageLocation'] = image['properties'].get('image_location') i['imageState'] = image['properties'].get('image_state') - i['type'] = image.get('type') + i['type'] = image_type i['isPublic'] = str(image['properties'].get('is_public', '')) == 'True' i['architecture'] = image['properties'].get('architecture') return i @@ -915,7 +916,8 @@ class CloudController(object): image_location = kwargs['name'] metadata = {'properties': {'image_location': image_location}} image = self.image_service.create(context, metadata) - image_id = self._image_ec2_id(image['id'], image['type']) + image_id = self._image_ec2_id(image['id'], + image['properties']['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -954,6 +956,7 @@ class CloudController(object): raise exception.NotFound(_('Image %s not found') % image_id) internal_id = image['id'] del(image['id']) + raise Exception(image) image['properties']['is_public'] = (operation_type == 'add') return self.image_service.update(context, internal_id, image) diff --git a/nova/image/s3.py b/nova/image/s3.py index ab6eea8cf..bf104c29a 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -139,11 +139,13 @@ class S3ImageService(service.BaseImageService): manifest = key.get_contents_as_string() manifest = ElementTree.fromstring(manifest) + image_format = 'ami' image_type = 'machine' try: kernel_id = manifest.find("machine_configuration/kernel_id").text if kernel_id == 'true': + image_format = 'aki' image_type = 'kernel' kernel_id = None except: @@ -152,6 +154,7 @@ class S3ImageService(service.BaseImageService): try: ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text if ramdisk_id == 'true': + image_format = 'ari' image_type = 'ramdisk' ramdisk_id = None except: @@ -173,7 +176,9 @@ class S3ImageService(service.BaseImageService): properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id) properties['is_public'] = False - metadata.update({'type': image_type, + properties['type'] = image_type + metadata.update({'disk_format': image_format, + 'container_format': image_format, 'status': 'queued', 'is_public': True, 'properties': properties}) -- cgit From ec23b8e1205e969d449834b02984d01a8daf93dc Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 8 Mar 2011 20:28:11 +0000 Subject: update manpage --- doc/source/man/novamanage.rst | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/doc/source/man/novamanage.rst b/doc/source/man/novamanage.rst index 17ba91bef..1d8446f08 100644 --- a/doc/source/man/novamanage.rst +++ b/doc/source/man/novamanage.rst @@ -173,7 +173,10 @@ Nova Floating IPs ``nova-manage floating create `` Creates floating IP addresses for the named host by the given range. - floating delete Deletes floating IP addresses in the range given. + +``nova-manage floating delete `` + + Deletes floating IP addresses in the range given. ``nova-manage floating list`` @@ -193,7 +196,7 @@ Nova Flavor ``nova-manage flavor create <(optional) swap> <(optional) RXTX Quota> <(optional) RXTX Cap>`` creates a flavor with the following positional arguments: - * memory (expressed in megabytes) + * memory (expressed in megabytes) * vcpu(s) (integer) * local storage (expressed in gigabytes) * flavorid (unique integer) @@ -209,12 +212,33 @@ Nova Flavor Purges the flavor with the name . This removes this flavor from the database. - Nova Instance_type ~~~~~~~~~~~~~~~~~~ The instance_type command is provided as an alias for the flavor command. All the same subcommands and arguments from nova-manage flavor can be used. +Nova Images +~~~~~~~~~~~ + +``nova-manage image image_register `` + + Registers an image with the image service. + +``nova-manage image kernel_register `` + + Registers a kernel with the image service. + +``nova-manage image ramdisk_register `` + + Registers a ramdisk with the image service. + +``nova-manage image all_register `` + + Registers an image kernel and ramdisk with the image service. + +``nova-manage image convert `` + + Converts all images in directory from the old (Bexar) format to the new format. FILES ======== -- cgit From dd2f0019297d01fe5d6b3dae4efc72946191be75 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 8 Mar 2011 22:14:25 +0000 Subject: Use disk_format and container_format instead of image type --- nova/api/openstack/servers.py | 2 +- nova/virt/xenapi/vm_utils.py | 18 ++++++++++-------- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 14 +++++++++++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index c2bf42b72..85999764f 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -450,7 +450,7 @@ class Controller(wsgi.Controller): _("Cannot build from image %(image_id)s, status not active") % locals()) - if image['type'] != 'machine': + if image['disk_format'] != 'ami': return None, None try: diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 80b7540d4..ce081a2d6 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -467,19 +467,21 @@ class VMHelper(HelperBase): "%(image_id)s, instance %(instance_id)s") % locals()) def determine_from_glance(): - glance_type2nova_type = {'machine': ImageType.DISK, - 'raw': ImageType.DISK_RAW, - 'vhd': ImageType.DISK_VHD, - 'kernel': ImageType.KERNEL_RAMDISK, - 'ramdisk': ImageType.KERNEL_RAMDISK} + glance_disk_format2nova_type = { + 'ami': ImageType.DISK, + 'aki': ImageType.KERNEL_RAMDISK, + 'ari': ImageType.KERNEL_RAMDISK, + 'raw': ImageType.DISK_RAW, + 'vhd': ImageType.DISK_VHD} client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) meta = client.get_image_meta(instance.image_id) - type_ = meta['type'] + disk_format = meta['disk_format'] try: - return glance_type2nova_type[type_] + return glance_disk_format2nova_type[disk_format] except KeyError: raise exception.NotFound( - _("Unrecognized image type '%(type_)s'") % locals()) + _("Unrecognized disk_format '%(disk_format)s'") + % locals()) def determine_from_instance(): if instance.kernel_id: diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index aa12d432a..201b99fda 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -201,13 +201,21 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port): # to request conn.putrequest('PUT', '/images/%s' % image_id) - # TODO(sirp): make `store` configurable + # NOTE(sirp): There is some confusion around OVF. Here's a summary of + # where we currently stand: + # 1. OVF as a container format is misnamed. We really should be using + # OVA since that is the name for the container format; OVF is the + # standard applied to the manifest file contained within. + # 2. We're currently uploading a vanilla tarball. In order to be OVF/OVA + # compliant, we'll need to embed a minimal OVF manifest as the first + # file. headers = { 'content-type': 'application/octet-stream', 'transfer-encoding': 'chunked', - 'x-image-meta-is_public': 'True', + 'x-image-meta-is-public': 'True', 'x-image-meta-status': 'queued', - 'x-image-meta-type': 'vhd'} + 'x-image-meta-disk-format': 'vhd', + 'x-image-meta-container-format': 'ovf'} for header, value in headers.iteritems(): conn.putheader(header, value) conn.endheaders() -- cgit From 7a6833c883a04fd7920bff7367c9e28a35858d8d Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 8 Mar 2011 23:17:50 +0000 Subject: Accidentally left some bad data around --- nova/tests/test_xenapi.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 919a38c06..916555af3 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -376,6 +376,11 @@ class XenAPIMigrateInstance(test.TestCase): stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) + def tearDown(self): + super(XenAPIMigrateInstance, self).tearDown() + self.manager.delete_project(self.project) + self.manager.delete_user(self.user) + self.stubs.UnsetAll() def test_migrate_disk_and_power_off(self): instance = db.instance_create(self.values) -- cgit From 698398fdc2a05a0930591d3f3d386ad24a322359 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Tue, 8 Mar 2011 23:24:19 +0000 Subject: Pep8 fixes --- nova/tests/test_xenapi.py | 2 ++ nova/tests/xenapi/stubs.py | 1 + nova/virt/xenapi/vmops.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 916555af3..c26dc8639 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -376,6 +376,7 @@ class XenAPIMigrateInstance(test.TestCase): stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) + def tearDown(self): super(XenAPIMigrateInstance, self).tearDown() self.manager.delete_project(self.project) @@ -394,6 +395,7 @@ class XenAPIMigrateInstance(test.TestCase): conn = xenapi_conn.get_connection(False) conn.finish_resize(instance, dict(base_copy='hurr', cow='durr')) + class XenAPIDetermineDiskImageTestCase(test.TestCase): """ Unit tests for code that detects the ImageType diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index d8e358611..70d46a1fb 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -237,6 +237,7 @@ class FakeSessionForMigrationTests(fake.SessionBase): vm['is_a_template'] = False vm['is_control_domain'] = False + def stub_out_migration_methods(stubs): def fake_get_snapshot(self, instance): return 'foo', 'bar' diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 43c23806e..562ecd4d5 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -83,7 +83,7 @@ class VMOps(object): def spawn(self, instance): vdi_uuid = self.create_disk(instance) self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) - + def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" instance_name = instance.name -- cgit From 6207abe3068964c586d06bb0e3740b8bad922dca Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Tue, 8 Mar 2011 23:26:33 +0000 Subject: Fixing tests --- nova/tests/glance/stubs.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index 3ff8d7ce5..5872552ec 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -35,23 +35,28 @@ class FakeGlance(object): IMAGE_FIXTURES = { IMAGE_MACHINE: { 'image_meta': {'name': 'fakemachine', 'size': 0, - 'type': 'machine'}, + 'disk_format': 'ami', + 'container_format': 'ami'}, 'image_data': StringIO.StringIO('')}, IMAGE_KERNEL: { 'image_meta': {'name': 'fakekernel', 'size': 0, - 'type': 'kernel'}, + 'disk_format': 'aki', + 'container_format': 'aki'}, 'image_data': StringIO.StringIO('')}, IMAGE_RAMDISK: { 'image_meta': {'name': 'fakeramdisk', 'size': 0, - 'type': 'ramdisk'}, + 'disk_format': 'ari', + 'container_format': 'ari'}, 'image_data': StringIO.StringIO('')}, IMAGE_RAW: { 'image_meta': {'name': 'fakeraw', 'size': 0, - 'type': 'raw'}, + 'disk_format': 'raw', + 'container_format': 'bare'}, 'image_data': StringIO.StringIO('')}, IMAGE_VHD: { 'image_meta': {'name': 'fakevhd', 'size': 0, - 'type': 'vhd'}, + 'disk_format': 'vhd', + 'container_format': 'ovf'}, 'image_data': StringIO.StringIO('')}} def __init__(self, host, port=None, use_ssl=False): -- cgit From a4830f83afd78cdb96dc3e474eb4efc167de7737 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 8 Mar 2011 16:45:20 -0800 Subject: Sorted imports correctly --- bin/nova-api | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 85ca4eefd..06bb855cb 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -34,9 +34,9 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) -from nova import service from nova import flags from nova import log as logging +from nova import service from nova import utils from nova import version from nova import wsgi -- cgit From e8c8fd3f232371625f0924410c4c09c32339b113 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 8 Mar 2011 16:47:43 -0800 Subject: Renamed FLAG.paste_config -> FLAG.api_paste_config --- nova/service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/service.py b/nova/service.py index 5a8d58695..460f36f7a 100644 --- a/nova/service.py +++ b/nova/service.py @@ -56,7 +56,7 @@ flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') flags.DEFINE_string('osapi_listen', "0.0.0.0", 'IP address for OpenStack API to listen') flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') -flags.DEFINE_string('paste_config', "api-paste.ini", +flags.DEFINE_string('api_paste_config', "api-paste.ini", 'File name for the paste.deploy config for nova-api') @@ -240,10 +240,10 @@ class ApiService(WsgiService): @classmethod def create(cls, conf=None): if not conf: - conf = wsgi.paste_config_file(FLAGS.paste_config) + conf = wsgi.paste_config_file(FLAGS.api_paste_config) if not conf: message = (_("No paste configuration found for: %s"), - FLAGS.paste_config) + FLAGS.api_paste_config) raise exception.Error(message) api_endpoints = ['ec2', 'osapi'] service = cls(conf, api_endpoints) @@ -315,7 +315,7 @@ def _run_wsgi(paste_config_file, apis): getattr(FLAGS, "%s_listen" % api))) if len(apps) == 0: logging.error(_("No known API applications configured in %s."), - paste_config_file) + paste_config_file) return server = wsgi.Server() -- cgit From e4b176d41cca234082c28ba6d9188745f1d2b98a Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 9 Mar 2011 00:49:56 +0000 Subject: a few fixes for the tests --- .../versions/009_add_os_type_to_instances.py | 53 ---------------------- .../versions/010_add_os_type_to_instances.py | 53 ++++++++++++++++++++++ nova/tests/test_xenapi.py | 1 + nova/virt/xenapi/vmops.py | 2 +- 4 files changed, 55 insertions(+), 54 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py deleted file mode 100644 index 514b92b81..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py +++ /dev/null @@ -1,53 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from sqlalchemy.sql import text -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -instances_os_type = Column('os_type', - String(length=255, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False), - nullable=True) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - - instances.create_column(instances_os_type) - migrate_engine.execute(instances.update()\ - .where(instances.c.os_type == None)\ - .values(os_type='linux')) - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - - instances.drop_column('os_type') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py new file mode 100644 index 000000000..514b92b81 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py @@ -0,0 +1,53 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from sqlalchemy.sql import text +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) + migrate_engine.execute(instances.update()\ + .where(instances.c.os_type == None)\ + .values(os_type='linux')) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances.drop_column('os_type') diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 383819b00..cd125a301 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -440,6 +440,7 @@ class XenAPIMigrateInstance(test.TestCase): 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux' } stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index a6a9fbf95..aa4372c3d 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -261,7 +261,7 @@ class VMOps(object): template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, image_id) + self._session, instance, template_vdi_uuids, image_id) finally: if template_vm_ref: self._destroy(instance, template_vm_ref, -- cgit From 59fa70102a06dce9f86b9b29825245bc54c01598 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 8 Mar 2011 16:51:05 -0800 Subject: Added documentation about needed flags --- nova/service.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nova/service.py b/nova/service.py index 460f36f7a..af20db01c 100644 --- a/nova/service.py +++ b/nova/service.py @@ -221,7 +221,12 @@ class Service(object): class WsgiService(object): - """Base class for WSGI based services.""" + """Base class for WSGI based services. + + For each api you define, you must also define these flags: + :_listen: The address on which to listen + :_listen_port: The port on which to listen + """ def __init__(self, conf, apis): self.conf = conf -- cgit From a320b5df9f916adf8422ed312306c77570d392c2 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 00:30:05 -0500 Subject: execvp: almost passes tests --- nova/api/ec2/cloud.py | 2 +- nova/network/linux_net.py | 21 +++++++++++---------- nova/tests/test_network.py | 2 +- nova/tests/test_virt.py | 11 ++++++----- nova/utils.py | 19 +++++++++++++------ nova/virt/libvirt_conn.py | 11 ++++++----- nova/virt/xenapi/vm_utils.py | 6 ++---- nova/volume/driver.py | 5 +++-- 8 files changed, 43 insertions(+), 34 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0d22a3f46..b7d72d1c1 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -115,7 +115,7 @@ class CloudController(object): start = os.getcwd() os.chdir(FLAGS.ca_path) # TODO(vish): Do this with M2Crypto instead - utils.runthis(_("Generating root CA: %s"), "sh genrootca.sh") + utils.runthis(_("Generating root CA: %s"), "sh", "genrootca.sh") os.chdir(start) def _get_mpi_data(self, context, project_id): diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ad019a8c0..b66a1adb7 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -139,14 +139,14 @@ def init_host(): def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" _execute('sudo', 'ip', 'addr', 'add', floating_ip, - 'dev', FLAGS.public_interface), + 'dev', FLAGS.public_interface, check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" _execute('sudo', 'ip', 'addr', 'del', floating_ip, - 'dev', FLAGS.public_interface)) + 'dev', FLAGS.public_interface) def ensure_vlan_forward(public_ip, port, private_ip): @@ -213,7 +213,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'brctl', 'addbr', bridge) _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute('sudo', 'brctl', 'stp', bridge', 'off') + _execute('sudo', 'brctl', 'stp', bridge, 'off') _execute('sudo', 'ip', 'link', 'set', bridge, up) if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the @@ -223,7 +223,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): "%s/%s" % (net_attrs['gateway'], suffix), 'brd', - net-attrs['broadcast'], + net_attrs['broadcast'], 'dev', bridge, check_exit_code=False) @@ -257,7 +257,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) - out, err = _execute('sudo', 'brctl', 'addif, bridge, interface, + out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " @@ -265,11 +265,11 @@ def ensure_bridge(bridge, interface, net_attrs=None): raise exception.Error("Failed to add interface: %s" % err) if FLAGS.use_nova_chains: - (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward, + (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward', check_exit_code=False) if err != 'iptables: Chain already exists.\n': # NOTE(vish): chain didn't exist link chain - _execute('sudo', 'iptables, '-D', 'FORWARD', '-j', 'nova_forward', + _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') @@ -355,7 +355,7 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat', "/proc/%d/cmdline' + out, _err = _execute('cat', '/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: @@ -383,7 +383,7 @@ def _host_dhcp(fixed_ip_ref): def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", ' '.join(cmd)) + LOG.debug("FAKE NET: %s", " ".join(map(str, cmd))) return "fake", 0 else: return utils.execute(*cmd, **kwargs) @@ -396,7 +396,8 @@ def _device_exists(device): return not err -def _confirm_rule(chain, *cmd, append=False): +def _confirm_rule(chain, *cmd, **kwargs): + append=kwargs.get('append',False) """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 6d2d8b771..19099ff4c 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,7 +343,7 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = (binpath('nova-dhcpbridge'), 'add' + cmd = (binpath('nova-dhcpbridge'), 'add', instance_ref['mac_address'], private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..7f1ad002e 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -315,15 +315,16 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): - if cmd == 'sudo ip6tables-save -t filter': + def fake_iptables_execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None - if cmd == 'sudo iptables-save -t filter': + if cmd == ('sudo', 'iptables-save', '-t', 'filter'): return '\n'.join(self.in_rules), None - if cmd == 'sudo iptables-restore': + if cmd == ('sudo', 'iptables-restore'): self.out_rules = process_input.split('\n') return '', '' - if cmd == 'sudo ip6tables-restore': + if cmd == ('sudo', 'ip6tables-restore'): self.out6_rules = process_input.split('\n') return '', '' self.fw.execute = fake_iptables_execute diff --git a/nova/utils.py b/nova/utils.py index c96b85294..9b51f8b40 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -128,13 +128,20 @@ def fetchfile(url, target): execute("curl", "--fail", url, "-o", target) -def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): +def execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + addl_env=kwargs.get('addl_env', None) + check_exit_code=kwargs.get('check_exit_code', True) + stdin=kwargs.get('stdin', subprocess.PIPE) + stdout=kwargs.get('stdout', subprocess.PIPE) + stderr=kwargs.get('stderr', subprocess.PIPE) + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) result = None if process_input != None: result = obj.communicate(process_input) @@ -220,9 +227,9 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code=True): - LOG.debug(_("Running %s"), (cmd)) - rv, err = execute(cmd, check_exit_code=check_exit_code) +def runthis(prompt, *cmd, **kwargs): + LOG.debug(_("Running %s"), (" ".join(cmd))) + rv, err = execute(*cmd, **kwargs) def generate_uid(topic, size=8): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index a5256bbc2..76f31f91a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -540,8 +540,8 @@ class LibvirtConnection(object): if not os.path.exists(base): fn(target=base, *args, **kwargs) if cow: - utils.execute('qemu-img', 'create', '-f', 'qcow2', "'-o'', - "cluster_size=2M,backing_file=%s" % base, + utils.execute('qemu-img', 'create', '-f', 'qcow2', '-o', + 'cluster_size=2M,backing_file=%s' % base, target) else: utils.execute('cp', base, target) @@ -554,7 +554,7 @@ class LibvirtConnection(object): def _create_local(self, target, local_gb): """Create a blank image of specified size""" - utils.execute('truncate', target, '-s', "%dG" local_gb) + utils.execute('truncate', target, '-s', "%dG" % local_gb) # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): @@ -565,7 +565,7 @@ class LibvirtConnection(object): fname + suffix) # ensure directories exist and are writable - utils.execute('mkdir', '-p', basepath(suffix='') + utils.execute('mkdir', '-p', basepath(suffix='')) LOG.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') @@ -1245,7 +1245,8 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter') + current_filter, _ = self.execute('sudo', 'iptables-save', + '-t', 'filter') current_lines = current_filter.split('\n') new_filter = self.modify_rules(current_lines, 4) self.execute('sudo', 'iptables-restore', diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 8fdb658fb..6ba13f980 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -915,10 +915,8 @@ def _write_partition(virtual_size, dev): LOG.debug(_('Writing partition table %(primary_first)d %(primary_last)d' ' to %(dest)s...') % locals()) - def execute(*cmd, process_input=None, check_exit_code=True): - return utils.execute(*cmd, - process_input=process_input, - check_exit_code=check_exit_code) + def execute(*cmd, **kwargs): + return utils.execute(*cmd, **kwargs) execute('parted', '--script', dest, 'mklabel', 'msdos') execute('parted', '--script', dest, 'mkpart', 'primary', diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 220c9ef9d..e9bdf162f 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -104,7 +104,8 @@ class VolumeDriver(object): def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute('sudo', 'lvdisplay', '%s/%s" % + self._try_execute('sudo', 'lvdisplay', + '%s/%s' % (FLAGS.volume_group, volume['name'])) except Exception as e: @@ -550,7 +551,7 @@ class SheepdogDriver(VolumeDriver): else: sizestr = '%sG' % volume['size'] self._try_execute('qemu-img', 'create', - "sheepdog:%s" %s" % volume['name'], + "sheepdog:%s" % volume['name'], sizestr) def delete_volume(self, volume): -- cgit From 1d7358e70379607c9cce02307f4336efbd135a5d Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 01:26:53 -0500 Subject: execvp: unit tests pass --- nova/crypto.py | 2 +- nova/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/crypto.py b/nova/crypto.py index dd24723b8..20bb570a5 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -196,7 +196,7 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute('openssl', 'genrsa', '-out', keyfile, bits) + utils.execute('openssl', 'genrsa', '-out', keyfile, str(bits)) utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, '-batch', '-subj', subject) private_key = open(keyfile).read() diff --git a/nova/utils.py b/nova/utils.py index 9b51f8b40..7ddf056ea 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -135,6 +135,7 @@ def execute(*cmd, **kwargs): stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) + cmd=map(str,cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() -- cgit From 77da93886be61230dea5a4a4c4de036a57e62550 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 06:56:42 +0000 Subject: tests and semaphore fix for image caching --- nova/tests/test_virt.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++ nova/virt/libvirt_conn.py | 14 +++++++--- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..ec7498d72 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,12 +14,16 @@ # License for the specific language governing permissions and limitations # under the License. +import os + +import eventlet from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db from nova import flags +from nova import log as logging from nova import test from nova import utils from nova.api.ec2 import cloud @@ -30,6 +34,68 @@ FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +def _concurrency(wait, done, target): + wait.wait() + done.send() + + +class CacheConcurrencyTestCase(test.TestCase): + def setUp(self): + super(CacheConcurrencyTestCase, self).setUp() + + def fake_exists(fname): + basedir = os.path.join(FLAGS.instances_path, '_base') + if fname == basedir: + return True + return False + + def fake_execute(*args, **kwargs): + pass + + self.stubs.Set(os.path, 'exists', fake_exists) + self.stubs.Set(utils, 'execute', fake_execute) + + def test_same_fname_concurrency(self): + """Ensures that the same fname cache runs at a sequentially""" + conn = libvirt_conn.get_connection(False) + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertFalse(done2.ready()) + finally: + wait1.send() + done1.wait() + eventlet.sleep(0) + self.assertTrue(done2.ready()) + + def test_different_fname_concurrency(self): + """Ensures that two different fname caches are concurrent""" + conn = libvirt_conn.get_connection(False) + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname2', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname1', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertTrue(done2.ready()) + finally: + wait1.send() + eventlet.sleep(0) + + class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9f7315c17..1a1f146d4 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -44,9 +44,8 @@ import uuid from xml.dom import minidom -from eventlet import greenthread -from eventlet import event from eventlet import tpool +from eventlet import semaphore import IPy @@ -512,6 +511,8 @@ class LibvirtConnection(object): subprocess.Popen(cmd, shell=True) return {'token': token, 'host': host, 'port': port} + _image_semaphores = {} + def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): """Wrapper for a method that creates an image that caches the image. @@ -531,8 +532,13 @@ class LibvirtConnection(object): if not os.path.exists(base_dir): os.mkdir(base_dir) base = os.path.join(base_dir, fname) - if not os.path.exists(base): - fn(target=base, *args, **kwargs) + + if fname not in self._image_semaphores: + self._image_semaphores[fname] = semaphore.Semaphore() + with self._image_semaphores[fname]: + if not os.path.exists(base): + fn(target=base, *args, **kwargs) + if cow: utils.execute('qemu-img create -f qcow2 -o ' 'cluster_size=2M,backing_file=%s %s' -- cgit From ddeab2da30bb2f74109854d982c6681e78e7a4ce Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 07:35:58 +0000 Subject: make static method for testing without initializing libvirt --- nova/tests/test_virt.py | 4 ++-- nova/virt/libvirt_conn.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index ec7498d72..113632a0c 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -57,7 +57,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_same_fname_concurrency(self): """Ensures that the same fname cache runs at a sequentially""" - conn = libvirt_conn.get_connection(False) + conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, @@ -78,7 +78,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" - conn = libvirt_conn.get_connection(False) + conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 1a1f146d4..ecef7950a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -511,9 +511,10 @@ class LibvirtConnection(object): subprocess.Popen(cmd, shell=True) return {'token': token, 'host': host, 'port': port} - _image_semaphores = {} + _image_sems = {} - def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): + @staticmethod + def _cache_image(fn, target, fname, cow=False, *args, **kwargs): """Wrapper for a method that creates an image that caches the image. This wrapper will save the image into a common store and create a @@ -533,9 +534,9 @@ class LibvirtConnection(object): os.mkdir(base_dir) base = os.path.join(base_dir, fname) - if fname not in self._image_semaphores: - self._image_semaphores[fname] = semaphore.Semaphore() - with self._image_semaphores[fname]: + if fname not in LibvirtConnection._image_sems: + LibvirtConnection._image_sems[fname] = semaphore.Semaphore() + with LibvirtConnection._image_sems[fname]: if not os.path.exists(base): fn(target=base, *args, **kwargs) -- cgit From 7d31fe9ef316f49379818259a55a84deb5b850cd Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 9 Mar 2011 10:30:18 +0100 Subject: Stop assuming anything about the order in which the two processes are scheduled. --- nova/tests/test_misc.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 9f572b58e..a658e4978 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -71,30 +71,33 @@ class LockTestCase(test.TestCase): "got mangled") def test_synchronized(self): - rpipe, wpipe = os.pipe() + rpipe1, wpipe1 = os.pipe() + rpipe2, wpipe2 = os.pipe() + + @synchronized('testlock') + def f(rpipe, wpipe): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + pid = os.fork() if pid > 0: - os.close(wpipe) - - @synchronized('testlock') - def f(): - rfds, _, __ = select.select([rpipe], [], [], 1) - self.assertEquals(len(rfds), 0, "The other process, which was" - " supposed to be locked, " - "wrote on its end of the " - "pipe") - os.close(rpipe) - - f() + os.close(wpipe1) + os.close(rpipe2) + + f(rpipe1, wpipe2) else: - os.close(rpipe) + os.close(rpipe1) + os.close(wpipe2) - @synchronized('testlock') - def g(): - try: - os.write(wpipe, "foo") - except OSError, e: - self.assertEquals(e.errno, errno.EPIPE) - return - g() + f(rpipe2, wpipe1) os._exit(0) -- cgit From 0f45b59ca6f9502a3ae6578e2fca5a7d9575ae5e Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 9 Mar 2011 10:37:21 -0500 Subject: Added 'adminPass' to the serialization_metadata. --- nova/api/openstack/servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 9581b8477..bbedd7c63 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -96,7 +96,7 @@ class Controller(wsgi.Controller): 'application/xml': { "attributes": { "server": ["id", "imageId", "name", "flavorId", "hostId", - "status", "progress"]}}} + "status", "progress", "adminPass"]}}} def __init__(self): self.compute_api = compute.API() -- cgit From eadce208c55513ddbab550898e641b8ee55a67ec Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 9 Mar 2011 12:32:15 -0500 Subject: Fix spacing. --- nova/api/openstack/servers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index bbedd7c63..7222285e0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -84,11 +84,13 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) + def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ return dict(server=dict(id=inst['id'], name=inst['display_name'])) + class Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ -- cgit From e17876ec002f976572b6ac102dc113024669a45c Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 18:57:53 +0100 Subject: fixed lp715427 --- nova/network/manager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nova/network/manager.py b/nova/network/manager.py index b36dd59cf..39da031eb 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -563,6 +563,16 @@ class VlanManager(NetworkManager): # NOTE(vish): This makes ports unique accross the cloud, a more # robust solution would be to make them unique per ip net['vpn_public_port'] = vpn_start + index + network_ref = None + try: + network_ref = db.network_get_by_cidr(context, cidr) + except exception.NotFound: + pass + + if network_ref is not None: + raise ValueError(_('Network with cidr %s already exists' % + cidr)) + network_ref = self.db.network_create_safe(context, net) if network_ref: self._create_fixed_ips(context, network_ref['id']) -- cgit From 48c8b911899db4db36dfc2e0ddaf3410c3179071 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 19:03:58 +0100 Subject: fixed lp715427 --- bin/nova-manage | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 8ddfea5c2..45437d7e7 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -547,10 +547,11 @@ class NetworkCommands(object): def delete(self, fixed_range): """Deletes a network""" - network = db.network_get_by_cidr(context.get_admin_context(), fixed_range) + network = db.network_get_by_cidr(context.get_admin_context(), \ + fixed_range) if network.project_id is not None: - raise ValueError(_('Network must be disassociated from project %s' - ' before delete' %network.project_id)) + raise ValueError(_('Network must be disassociated from project %s' + ' before delete' % network.project_id)) db.network_delete_safe(context.get_admin_context(), network.id) class ServiceCommands(object): -- cgit From 429fdb1ee733a62052c67f4e42c62447fc716ec0 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Wed, 9 Mar 2011 18:10:45 +0000 Subject: removed uneeded **kw args leftover from removed account-in-url changes. --- nova/api/openstack/backup_schedules.py | 6 ++--- nova/api/openstack/consoles.py | 10 ++++---- nova/api/openstack/flavors.py | 6 ++--- nova/api/openstack/images.py | 12 +++++----- nova/api/openstack/servers.py | 42 +++++++++++++++++----------------- nova/api/openstack/shared_ip_groups.py | 12 +++++----- nova/api/openstack/users.py | 12 +++++----- nova/api/openstack/zones.py | 12 +++++----- 8 files changed, 56 insertions(+), 56 deletions(-) diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index a4d5939df..7abb5f884 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -40,15 +40,15 @@ class Controller(wsgi.Controller): def __init__(self): pass - def index(self, req, server_id, **kw): + def index(self, req, server_id): """ Returns the list of backup schedules for a given instance """ return _translate_keys({}) - def create(self, req, server_id, **kw): + def create(self, req, server_id): """ No actual update method required, since the existing API allows both create and update through a POST """ return faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id, **kw): + def delete(self, req, server_id, id): """ Deletes an existing backup schedule """ return faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 85b2a4140..9ebdbe710 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -55,7 +55,7 @@ class Controller(wsgi.Controller): self.console_api = console.API() super(Controller, self).__init__() - def index(self, req, server_id, **kw): + def index(self, req, server_id): """Returns a list of consoles for this instance""" consoles = self.console_api.get_consoles( req.environ['nova.context'], @@ -63,14 +63,14 @@ class Controller(wsgi.Controller): return dict(consoles=[_translate_keys(console) for console in consoles]) - def create(self, req, server_id, **kw): + def create(self, req, server_id): """Creates a new console""" #info = self._deserialize(req.body, req) self.console_api.create_console( req.environ['nova.context'], int(server_id)) - def show(self, req, server_id, id, **kw): + def show(self, req, server_id, id): """Shows in-depth information on a specific console""" try: console = self.console_api.get_console( @@ -81,11 +81,11 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return _translate_detail_keys(console) - def update(self, req, server_id, id, **kw): + def update(self, req, server_id, id): """You can't update a console""" raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id, **kw): + def delete(self, req, server_id, id): """Deletes a console""" try: self.console_api.delete_console(req.environ['nova.context'], diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index 1de67328b..f3d040ba3 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -34,17 +34,17 @@ class Controller(wsgi.Controller): "attributes": { "flavor": ["id", "name", "ram", "disk"]}}} - def index(self, req, **kw): + def index(self, req): """Return all flavors in brief.""" return dict(flavors=[dict(id=flavor['id'], name=flavor['name']) for flavor in self.detail(req)['flavors']]) - def detail(self, req, **kw): + def detail(self, req): """Return all flavors in detail.""" items = [self.show(req, id)['flavor'] for id in self._all_ids(req)] return dict(flavors=items) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given flavor id.""" ctxt = req.environ['nova.context'] values = db.instance_type_get_by_flavor_id(ctxt, id) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 5bc5b9978..cf85a496f 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -115,14 +115,14 @@ class Controller(wsgi.Controller): def __init__(self): self._service = utils.import_object(FLAGS.image_service) - def index(self, req, **kw): + def index(self, req): """Return all public images in brief""" items = self._service.index(req.environ['nova.context']) items = common.limited(items, req) items = [_filter_keys(item, ('id', 'name')) for item in items] return dict(images=items) - def detail(self, req, **kw): + def detail(self, req): """Return all public images in detail""" try: items = self._service.detail(req.environ['nova.context']) @@ -136,7 +136,7 @@ class Controller(wsgi.Controller): items = [_translate_status(item) for item in items] return dict(images=items) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given image id""" image_id = common.get_image_id_from_image_hash(self._service, req.environ['nova.context'], id) @@ -145,11 +145,11 @@ class Controller(wsgi.Controller): _convert_image_id_to_hash(image) return dict(image=image) - def delete(self, req, id, **kw): + def delete(self, req, id): # Only public images are supported for now. raise faults.Fault(exc.HTTPNotFound()) - def create(self, req, **kw): + def create(self, req): context = req.environ['nova.context'] env = self._deserialize(req.body, req) instance_id = env["image"]["serverId"] @@ -160,7 +160,7 @@ class Controller(wsgi.Controller): return dict(image=image_meta) - def update(self, req, id, **kw): + def update(self, req, id): # Users may not modify public images, and that's all that # we support for now. raise faults.Fault(exc.HTTPNotFound()) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 54060c2bb..c2bf42b72 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -105,11 +105,11 @@ class Controller(wsgi.Controller): self._image_service = utils.import_object(FLAGS.image_service) super(Controller, self).__init__() - def index(self, req, **kw): + def index(self, req): """ Returns a list of server names and ids for a given user """ return self._items(req, entity_maker=_translate_keys) - def detail(self, req, **kw): + def detail(self, req): """ Returns a list of server details for a given user """ return self._items(req, entity_maker=_translate_detail_keys) @@ -123,7 +123,7 @@ class Controller(wsgi.Controller): res = [entity_maker(inst)['server'] for inst in limited_list] return dict(servers=res) - def show(self, req, id, **kw): + def show(self, req, id): """ Returns server details by server id """ try: instance = self.compute_api.get(req.environ['nova.context'], id) @@ -131,7 +131,7 @@ class Controller(wsgi.Controller): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) - def delete(self, req, id, **kw): + def delete(self, req, id): """ Destroys a server """ try: self.compute_api.delete(req.environ['nova.context'], id) @@ -139,7 +139,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def create(self, req, **kw): + def create(self, req): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) if not env: @@ -180,7 +180,7 @@ class Controller(wsgi.Controller): onset_files=env.get('onset_files', [])) return _translate_keys(instances[0]) - def update(self, req, id, **kw): + def update(self, req, id): """ Updates the server name or password """ inst_dict = self._deserialize(req.body, req) if not inst_dict: @@ -202,7 +202,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() - def action(self, req, id, **kw): + def action(self, req, id): """Multi-purpose method used to reboot, rebuild, or resize a server""" @@ -267,7 +267,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def lock(self, req, id, **kw): + def lock(self, req, id): """ lock the instance with id admin only operation @@ -282,7 +282,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unlock(self, req, id, **kw): + def unlock(self, req, id): """ unlock the instance with id admin only operation @@ -297,7 +297,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_lock(self, req, id, **kw): + def get_lock(self, req, id): """ return the boolean state of (instance with id)'s lock @@ -311,7 +311,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def reset_network(self, req, id, **kw): + def reset_network(self, req, id): """ Reset networking on an instance (admin only). @@ -325,7 +325,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def inject_network_info(self, req, id, **kw): + def inject_network_info(self, req, id): """ Inject network info for an instance (admin only). @@ -339,7 +339,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def pause(self, req, id, **kw): + def pause(self, req, id): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: @@ -350,7 +350,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unpause(self, req, id, **kw): + def unpause(self, req, id): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] try: @@ -361,7 +361,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def suspend(self, req, id, **kw): + def suspend(self, req, id): """permit admins to suspend the server""" context = req.environ['nova.context'] try: @@ -372,7 +372,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def resume(self, req, id, **kw): + def resume(self, req, id): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] try: @@ -383,7 +383,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def rescue(self, req, id, **kw): + def rescue(self, req, id): """Permit users to rescue the server.""" context = req.environ["nova.context"] try: @@ -394,7 +394,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unrescue(self, req, id, **kw): + def unrescue(self, req, id): """Permit users to unrescue the server.""" context = req.environ["nova.context"] try: @@ -405,7 +405,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_ajax_console(self, req, id, **kw): + def get_ajax_console(self, req, id): """ Returns a url to an instance's ajaxterm console. """ try: self.compute_api.get_ajax_console(req.environ['nova.context'], @@ -414,12 +414,12 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def diagnostics(self, req, id, **kw): + def diagnostics(self, req, id): """Permit Admins to retrieve server diagnostics.""" ctxt = req.environ["nova.context"] return self.compute_api.get_diagnostics(ctxt, id) - def actions(self, req, id, **kw): + def actions(self, req, id): """Permit Admins to retrieve server actions.""" ctxt = req.environ["nova.context"] items = self.compute_api.get_actions(ctxt, id) diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index e3c917749..5d78f9377 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -40,26 +40,26 @@ class Controller(wsgi.Controller): 'attributes': { 'sharedIpGroup': []}}} - def index(self, req, **kw): + def index(self, req): """ Returns a list of Shared IP Groups for the user """ return dict(sharedIpGroups=[]) - def show(self, req, id, **kw): + def show(self, req, id): """ Shows in-depth information on a specific Shared IP Group """ return _translate_keys({}) - def update(self, req, id, **kw): + def update(self, req, id): """ You can't update a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, id, **kw): + def delete(self, req, id): """ Deletes a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def detail(self, req, **kw): + def detail(self, req): """ Returns a complete list of Shared IP Groups """ return _translate_detail_keys({}) - def create(self, req, **kw): + def create(self, req): """ Creates a new Shared IP group """ raise faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index ae3bf7791..83ebec964 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -50,28 +50,28 @@ class Controller(wsgi.Controller): if not context.is_admin: raise exception.NotAuthorized(_("Not admin user")) - def index(self, req, **kw): + def index(self, req): """Return all users in brief""" users = self.manager.get_users() users = common.limited(users, req) users = [_translate_keys(user) for user in users] return dict(users=users) - def detail(self, req, **kw): + def detail(self, req): """Return all users in detail""" return self.index(req) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given user id""" user = self.manager.get_user(id) return dict(user=_translate_keys(user)) - def delete(self, req, id, **kw): + def delete(self, req, id): self._check_admin(req.environ['nova.context']) self.manager.delete_user(id) return {} - def create(self, req, **kw): + def create(self, req): self._check_admin(req.environ['nova.context']) env = self._deserialize(req.body, req) is_admin = env['user'].get('admin') in ('T', 'True', True) @@ -81,7 +81,7 @@ class Controller(wsgi.Controller): user = self.manager.create_user(name, access, secret, is_admin) return dict(user=_translate_keys(user)) - def update(self, req, id, **kw): + def update(self, req, id): self._check_admin(req.environ['nova.context']) env = self._deserialize(req.body, req) is_admin = env['user'].get('admin') diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 30bf2b67b..d5206da20 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -43,35 +43,35 @@ class Controller(wsgi.Controller): "attributes": { "zone": ["id", "api_url"]}}} - def index(self, req, **kw): + def index(self, req): """Return all zones in brief""" items = db.zone_get_all(req.environ['nova.context']) items = common.limited(items, req) items = [_scrub_zone(item) for item in items] return dict(zones=items) - def detail(self, req, **kw): + def detail(self, req): """Return all zones in detail""" return self.index(req) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given zone id""" zone_id = int(id) zone = db.zone_get(req.environ['nova.context'], zone_id) return dict(zone=_scrub_zone(zone)) - def delete(self, req, id, **kw): + def delete(self, req, id): zone_id = int(id) db.zone_delete(req.environ['nova.context'], zone_id) return {} - def create(self, req, **kw): + def create(self, req): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) - def update(self, req, id, **kw): + def update(self, req, id): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone_id = int(id) -- cgit From e44f085ed464a3397e3bf89a3e5355e538c71a65 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 19:16:26 +0100 Subject: Fixed pep8 issues --- bin/nova-manage | 5 +++-- nova/db/api.py | 7 +++++-- nova/db/sqlalchemy/api.py | 7 +++++-- nova/network/manager.py | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 45437d7e7..7dfc3c045 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -552,8 +552,9 @@ class NetworkCommands(object): if network.project_id is not None: raise ValueError(_('Network must be disassociated from project %s' ' before delete' % network.project_id)) - db.network_delete_safe(context.get_admin_context(), network.id) - + db.network_delete_safe(context.get_admin_context(), network.id) + + class ServiceCommands(object): """Enable and disable running services""" diff --git a/nova/db/api.py b/nova/db/api.py index 7ad99c1f4..5c34a02e4 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -459,6 +459,7 @@ def network_associate(context, project_id): """Associate a free network to a project.""" return IMPL.network_associate(context, project_id) + def network_count(context): """Return the number of networks.""" return IMPL.network_count(context) @@ -488,9 +489,9 @@ def network_create_safe(context, values): """ return IMPL.network_create_safe(context, values) + def network_delete_safe(context, network_id): - """Delete network with key network_id - + """Delete network with key network_id. This method assumes that the network is not associated with any project """ return IMPL.network_delete_safe(context, network_id) @@ -531,10 +532,12 @@ def network_get_by_bridge(context, bridge): """Get a network by bridge or raise if it does not exist.""" return IMPL.network_get_by_bridge(context, bridge) + def network_get_by_cidr(context, cidr): """Get a network by cidr or raise if it does not exist""" return IMPL.network_get_by_cidr(context, cidr) + def network_get_by_instance(context, instance_id): """Get a network by instance id or raise if it does not exist.""" return IMPL.network_get_by_instance(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 90730d325..3a1162a17 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1042,12 +1042,14 @@ def network_create_safe(context, values): except IntegrityError: return None + @require_admin_context def network_delete_safe(context, network_id): session = get_session() with session.begin(): - network_ref = network_get(context, network_id=network_id, session=session) - session.delete(network_ref) + network_ref = network_get(context, network_id=network_id, \ + session=session) + session.delete(network_ref) @require_admin_context @@ -1134,6 +1136,7 @@ def network_get_by_cidr(context, cidr): cidr) return result + @require_admin_context def network_get_by_instance(_context, instance_id): session = get_session() diff --git a/nova/network/manager.py b/nova/network/manager.py index 39da031eb..3dfc48934 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -568,11 +568,11 @@ class VlanManager(NetworkManager): network_ref = db.network_get_by_cidr(context, cidr) except exception.NotFound: pass - + if network_ref is not None: raise ValueError(_('Network with cidr %s already exists' % cidr)) - + network_ref = self.db.network_create_safe(context, net) if network_ref: self._create_fixed_ips(context, network_ref['id']) -- cgit From 23369a63f4b74fb64bf57554a3fd8b15e3e2b49c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 14:31:23 -0500 Subject: Fixes uses of process_input --- nova/utils.py | 4 ++-- nova/virt/disk.py | 4 ++-- nova/virt/libvirt_conn.py | 11 ++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 7ddf056ea..0937522ec 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,7 +131,7 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): process_input=kwargs.get('process_input', None) addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', True) + check_exit_code=kwargs.get('check_exit_code', 0) stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) @@ -151,7 +151,7 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: + if check_exit_code is not None and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 203517275..a54cda003 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -175,8 +175,8 @@ def _inject_key_into_fs(key, fs): utils.execute('sudo', 'chown', 'root', sshdir) utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - # TODO:EWINDISCH: not sure about the following /w execv patch - utils.execute('sudo', 'tee', '-a', keyfile, '\n' + key.strip() + '\n') + utils.execute('sudo', 'tee', '-a', keyfile, + process_input='\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 76f31f91a..6b555ecbb 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -485,13 +485,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - - # TODO(ewindisch): broken /w execvp patch. - # subprocess lets us do this, but utils.execute - # abstracts it away from us - cmd = 'netcat', '0.0.0.0', port, '-w', '1', ' Date: Wed, 9 Mar 2011 14:53:44 -0500 Subject: Add password parameter to the set_admin_password call in the compute api. Updated servers password to use this parameter. --- nova/api/openstack/servers.py | 3 ++- nova/compute/api.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7222285e0..41166f810 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -183,7 +183,8 @@ class Controller(wsgi.Controller): password = "%s%s" % (server['server']['name'][:4], utils.generate_password(12)) server['server']['adminPass'] = password - self.compute_api.set_admin_password(context, server['server']['id']) + self.compute_api.set_admin_password(context, server['server']['id'], + password) return server def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index 33d25fc4b..a0bb2cf04 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -498,9 +498,10 @@ class API(base.Base): """Unrescue the given instance.""" self._cast_compute_message('unrescue_instance', context, instance_id) - def set_admin_password(self, context, instance_id): + def set_admin_password(self, context, instance_id, password=None): """Set the root/admin password for the given instance.""" - self._cast_compute_message('set_admin_password', context, instance_id) + self._cast_compute_message('set_admin_password', context, instance_id, + password) def inject_file(self, context, instance_id): """Write a file to the given instance.""" -- cgit From 3f723bcf54b4d779c66373dc8f69f43923dd586a Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 9 Mar 2011 15:08:11 -0500 Subject: renaming wsgi.Request.best_match to best_match_content_type; correcting calls to that function in code from trunk --- nova/api/direct.py | 2 +- nova/api/openstack/__init__.py | 2 +- nova/api/openstack/faults.py | 2 +- nova/api/openstack/servers.py | 2 +- nova/tests/api/openstack/common.py | 1 + nova/tests/api/test_wsgi.py | 18 +++++++++--------- nova/wsgi.py | 4 ++-- 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index 1d699f947..dfca250e0 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req.best_match()) + return self._serialize(result, req.best_match_content_type()) else: return result diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 6e1a2a06c..197fcc619 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -125,5 +125,5 @@ class Versions(wsgi.Application): "application/xml": { "attributes": dict(version=["status", "id"])}} - content_type = req.best_match() + content_type = req.best_match_content_type() return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 075fdb997..2fd733299 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -58,6 +58,6 @@ class Fault(webob.exc.HTTPException): # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} serializer = wsgi.Serializer(metadata) - content_type = req.best_match() + content_type = req.best_match_content_type() self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8dd078a31..25c667532 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -217,7 +217,7 @@ class Controller(wsgi.Controller): 'rebuild': self._action_rebuild, } - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) for key in actions.keys(): if key in input_dict: return actions[key](input_dict, req, id) diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py index 3f9c7d3cf..74bb8729a 100644 --- a/nova/tests/api/openstack/common.py +++ b/nova/tests/api/openstack/common.py @@ -28,6 +28,7 @@ def webob_factory(url): def web_request(url, method=None, body=None): req = webob.Request.blank("%s%s" % (base_url, url)) if method: + req.content_type = "application/json" req.method = method if body: req.body = json.dumps(body) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 7c0135656..b1a849cf9 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -139,48 +139,48 @@ class RequestTest(test.TestCase): def test_content_type_from_accept_xml(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml, application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = \ "application/json; q=0.3, application/xml; q=0.9" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") def test_content_type_from_query_extension(self): request = wsgi.Request.blank('/tests/123.xml') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") request = wsgi.Request.blank('/tests/123.json') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123.invalid') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") def test_content_type_accept_and_query_extension(self): request = wsgi.Request.blank('/tests/123.xml') request.headers["Accept"] = "application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") def test_content_type_accept_default(self): request = wsgi.Request.blank('/tests/123.unsupported') request.headers["Accept"] = "application/unsupported1" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") diff --git a/nova/wsgi.py b/nova/wsgi.py index c3e08522d..2d18da8fb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -85,7 +85,7 @@ class Server(object): class Request(webob.Request): - def best_match(self): + def best_match_content_type(self): """ Determine the most acceptable content-type based on the query extension then the Accept header @@ -354,7 +354,7 @@ class Controller(object): result = method(**arg_dict) if type(result) is dict: - content_type = req.best_match() + content_type = req.best_match_content_type() body = self._serialize(result, content_type) response = webob.Response() -- cgit From fc9840bae6200c8f89fb8a3ba0ab45663c872b3c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 15:33:20 -0500 Subject: execvp passes pep8 --- nova/console/xvp.py | 6 +++--- nova/crypto.py | 3 ++- nova/network/linux_net.py | 19 ++++++++++++------- nova/tests/test_virt.py | 2 +- nova/utils.py | 19 ++++++++++--------- nova/volume/driver.py | 4 ++-- 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/nova/console/xvp.py b/nova/console/xvp.py index 271dffa54..68d8c8565 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -134,9 +134,9 @@ class XVPConsoleProxy(object): logging.debug(_("Starting xvp")) try: utils.execute('xvp', - '-p',FLAGS.console_xvp_pid, - '-c',FLAGS.console_xvp_conf, - '-l',FLAGS.console_xvp_log) + '-p', FLAGS.console_xvp_pid, + '-c', FLAGS.console_xvp_conf, + '-l', FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) diff --git a/nova/crypto.py b/nova/crypto.py index 20bb570a5..717ea0041 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -120,7 +120,8 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen', '-y', '-f', '/dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', + # '/dev/stdin', private_key) return (private_key, public_key, fingerprint) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b66a1adb7..228a4d9ea 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -68,7 +68,8 @@ def metadata_forward(): _confirm_rule("PREROUTING", '-t', 'nat', '-s', '0.0.0.0/0', '-d', '169.254.169.254/32', '-p', 'tcp', '-m', 'tcp', '--dport', '80', '-j', 'DNAT', - '--to-destination', '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + '--to-destination', + '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): @@ -86,7 +87,8 @@ def init_host(): _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') - _execute('sudo', 'iptables', '-N', 'nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-N', 'nova_output', + check_exit_code=False) _execute('sudo', 'iptables', '-D', 'OUTPUT', '-j', 'nova_output', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'OUTPUT', '-j', 'nova_output') @@ -220,7 +222,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] out, err = _execute('sudo', 'ip', 'addr', 'add', - "%s/%s" % + "%s/%s" % (net_attrs['gateway'], suffix), 'brd', net_attrs['broadcast'], @@ -237,7 +239,8 @@ def ensure_bridge(bridge, interface, net_attrs=None): # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute('sudo', 'ip', 'link', 'set', 'dev', bridge, 'promisc', 'on') + _execute('sudo', 'ip', 'link', 'set', + 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge @@ -253,8 +256,10 @@ def ensure_bridge(bridge, interface, net_attrs=None): fields = line.split() if fields and fields[0] == "inet": params = ' '.join(fields[1:-1]) - _execute('sudo', 'ip', 'addr', 'del', params, 'dev', fields[-1]) - _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) + _execute('sudo', 'ip', 'addr', + 'del', params, 'dev', fields[-1]) + _execute('sudo', 'ip', 'addr', + 'add', params, 'dev', bridge) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, @@ -397,7 +402,7 @@ def _device_exists(device): def _confirm_rule(chain, *cmd, **kwargs): - append=kwargs.get('append',False) + append = kwargs.get('append', False) """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 7f1ad002e..dfa607f14 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -316,7 +316,7 @@ class IptablesFirewallTestCase(test.TestCase): # self.fw.add_instance(instance_ref) def fake_iptables_execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) + process_input = kwargs.get('process_input', None) if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None if cmd == ('sudo', 'iptables-save', '-t', 'filter'): diff --git a/nova/utils.py b/nova/utils.py index 0937522ec..3a4ec3c6a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -40,7 +40,7 @@ import netaddr from eventlet import event from eventlet import greenthread from eventlet.green import subprocess - +None from nova import exception from nova.exception import ProcessExecutionError from nova import log as logging @@ -129,13 +129,13 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) - addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', 0) - stdin=kwargs.get('stdin', subprocess.PIPE) - stdout=kwargs.get('stdout', subprocess.PIPE) - stderr=kwargs.get('stderr', subprocess.PIPE) - cmd=map(str,cmd) + process_input = kwargs.get('process_input', None) + addl_env = kwargs.get('addl_env', None) + check_exit_code = kwargs.get('check_exit_code', 0) + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + cmd = map(str, cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() @@ -151,7 +151,8 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code is not None and obj.returncode != check_exit_code: + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e9bdf162f..45cc800e7 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -112,7 +112,7 @@ class VolumeDriver(object): # If the volume isn't present, then don't attempt to delete return True - self._try_execute('sudo', 'lvremove', '-f',"%s/%s" % + self._try_execute('sudo', 'lvremove', '-f', "%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -256,7 +256,7 @@ class ISCSIDriver(VolumeDriver): self._sync_exec('sudo', 'ietadm', '--op', 'new', "--tid=%s" % iscsi_target, '--params', - "Name=%s" % iscsi-name, + "Name=%s" % iscsi_name, check_exit_code=False) self._sync_exec('sudo', 'ietadm', '--op', 'new', "--tid=%s" % iscsi_target, -- cgit From 75f7a73735957d5ddf04c7c9a23decf1a6fa7f9f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 9 Mar 2011 14:55:36 -0600 Subject: Added naming scheme comment --- nova/virt/xenapi_conn.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index b63a5f8c3..bfe290be3 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -49,6 +49,12 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. address for the nova-volume host :target_port: iSCSI Target Port, 3260 Default :iqn_prefix: IQN Prefix, e.g. 'iqn.2010-10.org.openstack' + +**Variable Naming Scheme** + +- suffix "_ref" for opaque references +- suffix "_uuid" for UUIDs +- suffix "_rec" for record objects """ import sys -- cgit From 3e61bf9963d7e98e8152d2eacfc4461d8cda309c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 21:43:35 +0000 Subject: remove the semaphore when there is no one waiting on it --- nova/tests/test_virt.py | 3 ++- nova/virt/libvirt_conn.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 113632a0c..56a271365 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -23,7 +23,6 @@ from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db from nova import flags -from nova import log as logging from nova import test from nova import utils from nova.api.ec2 import cloud @@ -70,11 +69,13 @@ class CacheConcurrencyTestCase(test.TestCase): eventlet.sleep(0) try: self.assertFalse(done2.ready()) + self.assertTrue('fname' in conn._image_sems) finally: wait1.send() done1.wait() eventlet.sleep(0) self.assertTrue(done2.ready()) + self.assertFalse('fname' in conn._image_sems) def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index ecef7950a..69249ed57 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -539,6 +539,8 @@ class LibvirtConnection(object): with LibvirtConnection._image_sems[fname]: if not os.path.exists(base): fn(target=base, *args, **kwargs) + if not LibvirtConnection._image_sems[fname].locked(): + del LibvirtConnection._image_sems[fname] if cow: utils.execute('qemu-img create -f qcow2 -o ' -- cgit From e8554da80ac916f168461cb48078488700081c02 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 16:44:48 -0500 Subject: execvp: cleanup. --- nova/crypto.py | 6 +++--- .../networking/etc/xensource/scripts/vif_rules.py | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index 717ea0041..2a8d4abca 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,7 +105,7 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen', '-q', '-b', '%d' % bits, '-N', '', + utils.execute('ssh-keygen', '-q', '-b', bits, '-N', '', '-f', keyfile) (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', '%s.pub' % (keyfile)) @@ -147,9 +147,9 @@ def revoke_cert(project_id, file_name): os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', - '%s' % file_name) + file_name) utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', - '-out', '%s' % FLAGS.crl_file) + '-out', FLAGS.crl_file) os.chdir(start) diff --git a/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py b/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py index 2c34f7b1d..d2b2d61e6 100755 --- a/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py +++ b/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py @@ -52,7 +52,7 @@ def main(dom_id, command, only_this_vif=None): apply_iptables_rules(command, params) -def execute(command, return_stdout=False): +def execute(*command, return_stdout=False): devnull = open(os.devnull, 'w') proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=devnull) @@ -110,26 +110,26 @@ def apply_arptables_rules(command, params): def apply_ebtables_rules(command, params): ebtables = lambda *rule: execute("/sbin/ebtables", *rule) - ebtables('-D', 'FORWARD', '-p', '0806', '-o', '%(VIF)s' % params, - '--arp-ip-dst', '%(IP)s' % params, + ebtables('-D', 'FORWARD', '-p', '0806', '-o', params['VIF'], + '--arp-ip-dst', params['IP'], '-j', 'ACCEPT') ebtables('-D', 'FORWARD', '-p', '0800', '-o', - '%(VIF)s' % params, '--ip-dst', '%(IP)s' % params, + params['VIF'], '--ip-dst', params['IP'], '-j', 'ACCEPT') if command == 'online': ebtables('-A', 'FORWARD', '-p', '0806', - '-o', '%(VIF)s' % params - '--arp-ip-dst', '%(IP)s' % params, + '-o', params['VIF'], + '--arp-ip-dst', params['IP'], '-j', 'ACCEPT') ebtables('-A', 'FORWARD', '-p', '0800', - '-o', '%(VIF)s' % params, - '--ip-dst', '%(IP)s' % params, + '-o', params['VIF'], + '--ip-dst', params['IP'], '-j', 'ACCEPT') - ebtables('-D', 'FORWARD', '-s', '!', '%(MAC)s' % params, - '-i', '%(VIF)s' % params, '-j', 'DROP') + ebtables('-D', 'FORWARD', '-s', '!', params['MAC'], + '-i', params['VIF'], '-j', 'DROP') if command == 'online': - ebtables('-I', 'FORWARD', '1', '-s', '!', '%(MAC)s' % params, + ebtables('-I', 'FORWARD', '1', '-s', '!', params['MAC'], '-i', '%(VIF)s', '-j', 'DROP') -- cgit From fb4785b85c1bef4179140cfb85ce01eca9fb5da5 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 9 Mar 2011 21:46:27 +0000 Subject: fix the copyright notice in migration --- .../sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py index 514b92b81..eb3066894 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py @@ -1,8 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. +# Copyright 2010 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain -- cgit From 203e23ebebc73a98dc8e8497fd2b28d3a6bf01da Mon Sep 17 00:00:00 2001 From: Ken Pepple Date: Wed, 9 Mar 2011 14:13:52 -0800 Subject: initializing instance power state on launch to 0 (fixes EC2 API bug) --- nova/compute/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index a0bb2cf04..2358b562c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -165,6 +165,7 @@ class API(base.Base): 'image_id': image_id, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', + 'state': "0", 'state_description': 'scheduling', 'user_id': context.user_id, 'project_id': context.project_id, -- cgit From 5f6a58c7c2a7359f67bc4e2c2eb6bb9cc0a9ff01 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 17:22:54 -0500 Subject: execvp: fix docs --- doc/ext/nova_autodoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ext/nova_autodoc.py b/doc/ext/nova_autodoc.py index 5429bb656..3dd992d84 100644 --- a/doc/ext/nova_autodoc.py +++ b/doc/ext/nova_autodoc.py @@ -8,5 +8,6 @@ from nova import utils def setup(app): rootdir = os.path.abspath(app.srcdir + '/..') print "**Autodocumenting from %s" % rootdir - rv = utils.execute('cd %s && ./generate_autodoc_index.sh' % rootdir) + os.chdir(rootdir) + rv = utils.execute('./generate_autodoc_index.sh') print rv[0] -- cgit From 9822af58162dc520c4a17646a013560e422efcf9 Mon Sep 17 00:00:00 2001 From: Ken Pepple Date: Wed, 9 Mar 2011 14:54:57 -0800 Subject: maybe a int instead ? --- nova/compute/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 2358b562c..5334acfcf 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -165,7 +165,7 @@ class API(base.Base): 'image_id': image_id, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', - 'state': "0", + 'state': 0, 'state_description': 'scheduling', 'user_id': context.user_id, 'project_id': context.project_id, -- cgit From 21937b48fcac81fa108f37f307b1b2e969bb7b4f Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 00:01:15 +0000 Subject: Replace session.execute() calls performing raw UPDATE statements with SQLAlchemy code, with the exception of fixed_ip_disassociate_all_by_timeout() --- nova/db/sqlalchemy/api.py | 97 ++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 5e498fc6f..22c85106d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -701,14 +701,18 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.execute('update instances set deleted=1,' - 'deleted_at=:at where id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) - session.execute('update security_group_instance_association ' - 'set deleted=1,deleted_at=:at where instance_id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': models.Instance.updated_at + 0}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(instance_id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupInstanceAssociation. + updated_at + 0)}) @require_context @@ -950,9 +954,11 @@ def key_pair_destroy_all_by_user(context, user_id): authorize_user_context(context, user_id) session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update key_pairs set deleted=1 where user_id=:id', - {'id': user_id}) + session.query(models.KeyPair).\ + filter_by(user_id=user_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': models.KeyPair.updated_at + 0}) @require_context @@ -1063,7 +1069,9 @@ def network_disassociate(context, network_id): @require_admin_context def network_disassociate_all(context): session = get_session() - session.execute('update networks set project_id=NULL') + session.query(models.Network).\ + update({'project_id': None, + 'updated_at': models.Network.updated_at + 0}) @require_context @@ -1433,15 +1441,17 @@ def volume_data_get_for_project(context, project_id): def volume_destroy(context, volume_id): session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update volumes set deleted=1 where id=:id', - {'id': volume_id}) - session.execute('update export_devices set volume_id=NULL ' - 'where volume_id=:id', - {'id': volume_id}) - session.execute('update iscsi_targets set volume_id=NULL ' - 'where volume_id=:id', - {'id': volume_id}) + session.query(models.Volume).\ + filter_by(id=volume_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': models.Volume.updated_at + 0}) + session.query(models.ExportDevice).\ + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) + session.query(models.IscsiTarget).\ + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) @require_admin_context @@ -1661,17 +1671,26 @@ def security_group_create(context, values): def security_group_destroy(context, security_group_id): session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update security_groups set deleted=1 where id=:id', - {'id': security_group_id}) - session.execute('update security_group_instance_association ' - 'set deleted=1,deleted_at=:at ' - 'where security_group_id=:id', - {'id': security_group_id, - 'at': datetime.datetime.utcnow()}) - session.execute('update security_group_rules set deleted=1 ' - 'where group_id=:id', - {'id': security_group_id}) + session.query(models.SecurityGroup).\ + filter_by(id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + models.SecurityGroup.updated_at + 0}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(security_group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupInstanceAssocation. + updated_at + 0)}) + session.query(models.SecurityGroupIngressRule).\ + filter_by(group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupIngressRule. + updated_at + 0)}) @require_context @@ -1679,9 +1698,17 @@ def security_group_destroy_all(context, session=None): if not session: session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update security_groups set deleted=1') - session.execute('update security_group_rules set deleted=1') + session.query(models.SecurityGroup).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + models.SecurityGroup.updated_at + 0}) + session.query(models.SecurityGroupIngressRule).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupIngressRule. + updated_at + 0)}) ################### -- cgit From a83b4879f38d11634d405d0efe977d482abdc344 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 10 Mar 2011 05:02:24 +0000 Subject: minor fixes from review --- nova/image/glance.py | 2 +- nova/image/s3.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index fb383f5e6..15fca69b8 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -74,7 +74,7 @@ class GlanceImageService(service.BaseImageService): if name == cantidate.get('name'): image = cantidate break - if image == None: + if image is None: raise exception.NotFound return image diff --git a/nova/image/s3.py b/nova/image/s3.py index bf104c29a..bbc54c263 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -77,17 +77,17 @@ class S3ImageService(service.BaseImageService): # FIXME(vish): detail doesn't filter so we do it manually return self._filter(context, images) - @staticmethod - def _is_visible(context, image): + @classmethod + def _is_visible(cls, context, image): return (context.is_admin or context.project_id == image['properties']['owner_id'] or image['properties']['is_public'] == 'True') - @staticmethod - def _filter(context, images): + @classmethod + def _filter(cls, context, images): filtered = [] for image in images: - if not S3ImageService._is_visible(context, image): + if not cls._is_visible(context, image): continue filtered.append(image) return filtered @@ -148,7 +148,7 @@ class S3ImageService(service.BaseImageService): image_format = 'aki' image_type = 'kernel' kernel_id = None - except: + except Exception: kernel_id = None try: @@ -157,12 +157,12 @@ class S3ImageService(service.BaseImageService): image_format = 'ari' image_type = 'ramdisk' ramdisk_id = None - except: + except Exception: ramdisk_id = None try: arch = manifest.find("machine_configuration/architecture").text - except: + except Exception: arch = 'x86_64' properties = metadata['properties'] @@ -235,7 +235,7 @@ class S3ImageService(service.BaseImageService): @staticmethod def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, - cloud_private_key, decrypted_filename): + cloud_private_key, decrypted_filename): key, err = utils.execute( 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, process_input=encrypted_key, -- cgit From 1fa41c5c621f3190c8c2b1c3d885c95b6b627b23 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 09:52:19 +0100 Subject: s/s.getuid()/os.getuid()/ --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e25e4af4f..44b07213a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -463,7 +463,7 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo', 'chown', s.getuid(), console_log) + utils.execute('sudo', 'chown', os.getuid(), console_log) if FLAGS.libvirt_type == 'xen': # Xen is special -- cgit From 9e77a0c6f6b43494e0eb87a16f33cd566f0746d2 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 09:55:45 +0100 Subject: Split dnsmasq and radvd commands into their respective argv's. --- nova/network/linux_net.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 228a4d9ea..9fd6c82de 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -424,30 +424,30 @@ def _remove_rule(chain, *cmd): def _dnsmasq_cmd(net): """Builds dnsmasq command""" - cmd = ['sudo -E dnsmasq', - ' --strict-order', - ' --bind-interfaces', - ' --conf-file=', - ' --domain=%s' % FLAGS.dhcp_domain, - ' --pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), - ' --listen-address=%s' % net['gateway'], - ' --except-interface=lo', - ' --dhcp-range=%s,static,120s' % net['dhcp_start'], - ' --dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), - ' --dhcp-script=%s' % FLAGS.dhcpbridge, - ' --leasefile-ro'] + cmd = ['sudo', '-E', 'dnsmasq', + '--strict-order', + '--bind-interfaces', + '--conf-file=', + '--domain=%s' % FLAGS.dhcp_domain, + '--pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), + '--listen-address=%s' % net['gateway'], + '--except-interface=lo', + '--dhcp-range=%s,static,120s' % net['dhcp_start'], + '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), + '--dhcp-script=%s' % FLAGS.dhcpbridge, + '--leasefile-ro'] if FLAGS.dns_server: - cmd.append(' -h -R --server=%s' % FLAGS.dns_server) - return ''.join(cmd) + cmd += ['-h', '-R', '--server=%s' % FLAGS.dns_server] + return cmd def _ra_cmd(net): """Builds radvd command""" - cmd = ['sudo -E radvd', -# ' -u nobody', - ' -C %s' % _ra_file(net['bridge'], 'conf'), - ' -p %s' % _ra_file(net['bridge'], 'pid')] - return ''.join(cmd) + cmd = ['sudo', '-E', 'radvd', +# '-u', 'nobody', + '-C', '%s' % _ra_file(net['bridge'], 'conf'), + '-p', '%s' % _ra_file(net['bridge'], 'pid')] + return cmd def _stop_dnsmasq(network): -- cgit From e575f5ddd46055f2e491606052493b6d648506f6 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 10:16:07 +0100 Subject: Pass argv of dnsmasq and radvd to execute as individual args, not as a list. --- nova/network/linux_net.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 9fd6c82de..e64c052f9 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -330,7 +330,7 @@ def update_dhcp(context, network_id): env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, 'DNSMASQ_INTERFACE': network_ref['bridge']} command = _dnsmasq_cmd(network_ref) - _execute(command, addl_env=env) + _execute(*command, addl_env=env) def update_ra(context, network_id): @@ -370,7 +370,7 @@ interface %s else: LOG.debug(_("Pid %d is stale, relaunching radvd"), pid) command = _ra_cmd(network_ref) - _execute(command) + _execute(*command) db.network_update(context, network_id, {"ra_server": utils.get_my_linklocal(network_ref['bridge'])}) -- cgit From 6601d52bfa501ac1ae266647be19fac2f6792efc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 11:35:42 +0100 Subject: Make nova.image.s3 catch up with the new execute syntax. --- nova/image/s3.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/nova/image/s3.py b/nova/image/s3.py index bbc54c263..85a2c651c 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -236,25 +236,32 @@ class S3ImageService(service.BaseImageService): @staticmethod def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename): - key, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_key, - check_exit_code=False) + key, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt private key: %s") % err) - iv, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_iv, - check_exit_code=False) + iv, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt initialization " "vector: %s") % err) - _out, err = utils.execute( - 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' - % (encrypted_filename, key, iv, decrypted_filename), - check_exit_code=False) + _out, err = utils.execute('openssl', 'enc', + '-d', '-aes-128-cbc', + '-in', '%s' % (encrypted_filename,), + '-K', '%s' % (key,), + '-iv', '%s' % (iv,), + '-out', '%s' % (decrypted_filename,), + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt image file " "%(image_file)s: %(err)s") % -- cgit From bd3411f88532619b760aa8f51379db2f9c1cf5d0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 11:59:50 +0100 Subject: More execvp fallout --- nova/objectstore/image.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 8013cbd9c..c90b5b54b 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -253,25 +253,34 @@ class Image(object): @staticmethod def decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename): - key, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_key, - check_exit_code=False) + key, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt private key: %s") % err) - iv, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_iv, - check_exit_code=False) + iv, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt initialization " "vector: %s") % err) - _out, err = utils.execute( - 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' - % (encrypted_filename, key, iv, decrypted_filename), - check_exit_code=False) + _out, err = utils.execute('openssl', + 'enc', + '-d', + '-aes-128-cbc', + '-in', '%s' % (encrypted_filename,), + '-K', '%s' % (key,), + '-iv', '%s' % (iv,), + '-out', '%s' % (decrypted_filename,), + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt image file " "%(image_file)s: %(err)s") % -- cgit -- cgit From b64cf7352a24d8ced69aa408f7ceadd9da71da14 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 15:47:09 +0100 Subject: One more thing.. --- nova/network/linux_net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index e64c052f9..c0bd76adf 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -216,7 +216,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) _execute('sudo', 'brctl', 'stp', bridge, 'off') - _execute('sudo', 'ip', 'link', 'set', bridge, up) + _execute('sudo', 'ip', 'link', 'set', bridge, 'up') if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly -- cgit From b38af111532717cbe9f4bef1d3c3d58e7082c8b9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 16:25:18 +0100 Subject: Another little detail.. --- nova/virt/disk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/disk.py b/nova/virt/disk.py index a54cda003..5d499c42c 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -51,7 +51,7 @@ def extend(image, size): return utils.execute('truncate', '-s', size, image) # NOTE(vish): attempts to resize filesystem - utils.execute('e2fsck', '-fp', mage, check_exit_code=False) + utils.execute('e2fsck', '-fp', image, check_exit_code=False) utils.execute('resize2fs', image, check_exit_code=False) -- cgit From 801212a0ff04ddc33719d17b8c8ca847db5b1228 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Thu, 10 Mar 2011 15:47:55 +0000 Subject: Use a FLAGS.default_os_type if available --- nova/virt/xenapi/vm_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a1b85284f..8dd246178 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -41,9 +41,11 @@ from nova.virt.xenapi import HelperBase from nova.virt.xenapi.volume_utils import StorageError -FLAGS = flags.FLAGS LOG = logging.getLogger("nova.virt.xenapi.vm_utils") +FLAGS = flags.FLAGS +flags.DEFINE_string('default_os_type', 'linux', 'Default OS type') + XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, 'Running': power_state.RUNNING, @@ -347,7 +349,7 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) - os_type = instance.os_type and instance.os_type or 'linux' + os_type = instance.os_type or FLAGS.default_os_type params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, -- cgit From b361153a160ba1d61ed1d52de419cd27a8b4feda Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 16:42:13 +0000 Subject: Correct a misspelling --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 22c85106d..2b60ab7d5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1682,7 +1682,7 @@ def security_group_destroy(context, security_group_id): update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), 'updated_at': - (models.SecurityGroupInstanceAssocation. + (models.SecurityGroupInstanceAssociation. updated_at + 0)}) session.query(models.SecurityGroupIngressRule).\ filter_by(group_id=security_group_id).\ -- cgit From 4ead485ab69ee1e92635857ba73133a9e1d3bbcb Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 10 Mar 2011 12:06:09 -0600 Subject: Cleaned up vmops --- nova/virt/xenapi/vm_utils.py | 26 +++++------ nova/virt/xenapi/vmops.py | 104 ++++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 69 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index ce081a2d6..4ad820bcd 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -90,7 +90,7 @@ class VMHelper(HelperBase): get_instance_type(instance.instance_type) mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) - rec = { + vm_rec = { 'name_label': instance.name, 'name_description': '', 'is_a_template': False, @@ -122,23 +122,23 @@ class VMHelper(HelperBase): #Complete VM configuration record according to the image type #non-raw/raw with PV kernel/raw in HVM mode if instance.kernel_id: - rec['PV_bootloader'] = '' - rec['PV_kernel'] = kernel - rec['PV_ramdisk'] = ramdisk - rec['PV_args'] = 'root=/dev/xvda1' - rec['PV_bootloader_args'] = '' - rec['PV_legacy_args'] = '' + vm_rec['PV_bootloader'] = '' + vm_rec['PV_kernel'] = kernel + vm_rec['PV_ramdisk'] = ramdisk + vm_rec['PV_args'] = 'root=/dev/xvda1' + vm_rec['PV_bootloader_args'] = '' + vm_rec['PV_legacy_args'] = '' else: if pv_kernel: - rec['PV_args'] = 'noninteractive' - rec['PV_bootloader'] = 'pygrub' + vm_rec['PV_args'] = 'noninteractive' + vm_rec['PV_bootloader'] = 'pygrub' else: - rec['HVM_boot_policy'] = 'BIOS order' - rec['HVM_boot_params'] = {'order': 'dc'} - rec['platform'] = {'acpi': 'true', 'apic': 'true', + vm_rec['HVM_boot_policy'] = 'BIOS order' + vm_rec['HVM_boot_params'] = {'order': 'dc'} + vm_rec['platform'] = {'acpi': 'true', 'apic': 'true', 'pae': 'true', 'viridian': 'true'} LOG.debug(_('Created VM %s...'), instance.name) - vm_ref = session.call_xenapi('VM.create', rec) + vm_ref = session.call_xenapi('VM.create', vm_rec) instance_name = instance.name LOG.debug(_('Created VM %(instance_name)s as %(vm_ref)s.') % locals()) return vm_ref diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 562ecd4d5..5375df5b4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -56,10 +56,10 @@ class VMOps(object): def list_instances(self): """List VM instances""" vms = [] - for vm in self._session.get_xenapi().VM.get_all(): - rec = self._session.get_xenapi().VM.get_record(vm) - if not rec["is_a_template"] and not rec["is_control_domain"]: - vms.append(rec["name_label"]) + for vm_ref in self._session.get_xenapi().VM.get_all(): + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + if not vm_rec["is_a_template"] and not vm_rec["is_control_domain"]: + vms.append(vm_rec["name_label"]) return vms def _start(self, instance, vm_ref=None): @@ -371,8 +371,8 @@ class VMOps(object): def reboot(self, instance): """Reboot VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.clean_reboot', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.clean_reboot', vm_ref) self._session.wait_for_task(task, instance.id) def set_admin_password(self, instance, new_pass): @@ -571,26 +571,27 @@ class VMOps(object): def pause(self, instance, callback): """Pause VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.pause', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.pause', vm_ref) self._wait_with_callback(instance.id, task, callback) def unpause(self, instance, callback): """Unpause VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.unpause', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.unpause', vm_ref) self._wait_with_callback(instance.id, task, callback) def suspend(self, instance, callback): """suspend the specified instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.suspend', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.suspend', vm_ref) self._wait_with_callback(instance.id, task, callback) def resume(self, instance, callback): """resume the specified instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.resume', vm, False, True) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.resume', vm_ref, False, + True) self._wait_with_callback(instance.id, task, callback) def rescue(self, instance, callback): @@ -605,22 +606,18 @@ class VMOps(object): raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) - vm = self._get_vm_opaque_ref(instance) - self._shutdown(instance, vm) - self._acquire_bootlock(vm) + vm_ref = self._get_vm_opaque_ref(instance) + self._shutdown(instance, vm_ref) + self._acquire_bootlock(vm_ref) instance._rescue = True self.spawn(instance) - rescue_vm = self._get_vm_opaque_ref(instance) + rescue_vm_ref = self._get_vm_opaque_ref(instance) - vbd = self._session.get_xenapi().VM.get_VBDs(vm)[0] + vbd = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] vdi_ref = self._session.get_xenapi().VBD.get_record(vbd)["VDI"] - vbd_ref = VMHelper.create_vbd( - self._session, - rescue_vm, - vdi_ref, - 1, - False) + vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, vdi_ref, + 1, False) self._session.call_xenapi("Async.VBD.plug", vbd_ref) @@ -637,7 +634,7 @@ class VMOps(object): raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) - original_vm = self._get_vm_opaque_ref(instance) + original_vm_ref = self._get_vm_opaque_ref(instance) vbds = self._session.get_xenapi().VM.get_VBDs(rescue_vm) instance._rescue = False @@ -662,20 +659,20 @@ class VMOps(object): task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm) self._session.wait_for_task(task2, instance.id) - self._release_bootlock(original_vm) - self._start(instance, original_vm) + self._release_bootlock(original_vm_ref) + self._start(instance, original_vm_ref) def get_info(self, instance): """Return data about VM instance""" - vm = self._get_vm_opaque_ref(instance) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_info(rec) + vm_ref = self._get_vm_opaque_ref(instance) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + return VMHelper.compile_info(vm_rec) def get_diagnostics(self, instance): """Return data about VM diagnostics""" - vm = self._get_vm_opaque_ref(instance) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_diagnostics(self._session, rec) + vm_ref = self._get_vm_opaque_ref(instance) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + return VMHelper.compile_diagnostics(self._session, vm_rec) def get_console_output(self, instance): """Return snapshot of console""" @@ -698,9 +695,9 @@ class VMOps(object): # at this stage even though they aren't implemented because these will # be needed for multi-nic and there was no sense writing it for single # network/single IP and then having to turn around and re-write it - vm_opaque_ref = self._get_vm_opaque_ref(instance.id) + vm_ref = self._get_vm_opaque_ref(instance.id) logging.debug(_("injecting network info to xenstore for vm: |%s|"), - vm_opaque_ref) + vm_ref) admin_context = context.get_admin_context() IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) networks = db.network_get_all_by_instance(admin_context, @@ -731,11 +728,10 @@ class VMOps(object): 'ips': [ip_dict(ip) for ip in network_IPs], 'ip6s': [ip6_dict(ip) for ip in network_IPs]} - self.write_to_param_xenstore(vm_opaque_ref, {location: mapping}) + self.write_to_param_xenstore(vm_ref, {location: mapping}) try: - self.write_to_xenstore(vm_opaque_ref, location, - mapping['location']) + self.write_to_xenstore(vm_ref, location, mapping['location']) except KeyError: # catch KeyError for domid if instance isn't running pass @@ -747,8 +743,8 @@ class VMOps(object): Creates vifs for an instance """ - vm_opaque_ref = self._get_vm_opaque_ref(instance.id) - logging.debug(_("creating vif(s) for vm: |%s|"), vm_opaque_ref) + vm_ref = self._get_vm_opaque_ref(instance.id) + logging.debug(_("creating vif(s) for vm: |%s|"), vm_ref) if networks is None: networks = db.network_get_all_by_instance(admin_context, instance['id']) @@ -768,12 +764,8 @@ class VMOps(object): except AttributeError: device = "0" - VMHelper.create_vif( - self._session, - vm_opaque_ref, - network_ref, - instance.mac_address, - device) + VMHelper.create_vif(self._session, vm_ref, network_ref, + instance.mac_address, device) def reset_network(self, instance): """ @@ -837,9 +829,9 @@ class VMOps(object): Any errors raised by the plugin will in turn raise a RuntimeError here. """ instance_id = vm.id - vm = self._get_vm_opaque_ref(vm) - rec = self._session.get_xenapi().VM.get_record(vm) - args = {'dom_id': rec['domid'], 'path': path} + vm_ref = self._get_vm_opaque_ref(vm) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + args = {'dom_id': vm_rec['domid'], 'path': path} args.update(addl_args) try: task = self._session.async_call_plugin(plugin, method, args) @@ -919,9 +911,9 @@ class VMOps(object): value for 'keys' is passed, the returned dict is filtered to only return the values for those keys. """ - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) data = self._session.call_xenapi_request('VM.get_xenstore_data', - (vm, )) + (vm_ref, )) ret = {} if keys is None: keys = data.keys() @@ -939,11 +931,11 @@ class VMOps(object): """Takes a key/value pair and adds it to the xenstore parameter record for the given vm instance. If the key exists in xenstore, it is overwritten""" - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) self.remove_from_param_xenstore(instance_or_vm, key) jsonval = json.dumps(val) self._session.call_xenapi_request('VM.add_to_xenstore_data', - (vm, key, jsonval)) + (vm_ref, key, jsonval)) def write_to_param_xenstore(self, instance_or_vm, mapping): """Takes a dict and writes each key/value pair to the xenstore @@ -958,14 +950,14 @@ class VMOps(object): them from the xenstore parameter record data for the given VM. If the key doesn't exist, the request is ignored. """ - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) if isinstance(key_or_keys, basestring): keys = [key_or_keys] else: keys = key_or_keys for key in keys: self._session.call_xenapi_request('VM.remove_from_xenstore_data', - (vm, key)) + (vm_ref, key)) def clear_param_xenstore(self, instance_or_vm): """Removes all data from the xenstore parameter record for this VM.""" -- cgit From 03e5b8f7c4e1afc6637774acb3d28100035cd323 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 20:04:21 +0000 Subject: Partial revert of one conversion due to phantom magic exception from SQLAlchemy in unrelated code; convert all deletes --- nova/db/sqlalchemy/api.py | 65 ++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2b60ab7d5..31adb33ee 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -701,11 +701,21 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.query(models.Instance).\ - filter_by(id=instance_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': models.Instance.updated_at + 0}) + session.execute('update instances set deleted=1,' + 'deleted_at=:at where id=:id', + {'id': instance_id, + 'at': datetime.datetime.utcnow()}) + # NOTE(klmitch): for some reason, using the SQLAlchemy code + # here instead of the direct SQL update above causes the + # test_run_terminate_timestamps test (and only that one) to + # fail with an obscure TypeError exception from deep within + # SQLAlchemy; the nearest nova function in the traceback is + # instance_get() + # session.query(models.Instance).\ + # filter_by(id=instance_id).\ + # update({'deleted': 1, + # 'deleted_at': datetime.datetime.utcnow(), + # 'updated_at': models.Instance.updated_at + 0}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(instance_id=instance_id).\ update({'deleted': 1, @@ -1837,12 +1847,15 @@ def user_create(_context, values): def user_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association ' - 'where user_id=:id', {'id': id}) - session.execute('delete from user_role_association ' - 'where user_id=:id', {'id': id}) - session.execute('delete from user_project_role_association ' - 'where user_id=:id', {'id': id}) + session.query(models.UserProjectAssociation).\ + filter_by(user_id=id).\ + delete() + session.query(models.UserRoleAssociation).\ + filter_by(user_id=id).\ + delete() + session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=id).\ + delete() user_ref = user_get(context, id, session=session) session.delete(user_ref) @@ -1933,10 +1946,12 @@ def project_update(context, project_id, values): def project_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association ' - 'where project_id=:id', {'id': id}) - session.execute('delete from user_project_role_association ' - 'where project_id=:id', {'id': id}) + session.query(models.UserProjectAssociation).\ + filter_by(project_id=id).\ + delete() + session.query(models.UserProjectRoleAssociation).\ + filter_by(project_id=id).\ + delete() project_ref = project_get(context, id, session=session) session.delete(project_ref) @@ -1961,11 +1976,11 @@ def user_get_roles_for_project(context, user_id, project_id): def user_remove_project_role(context, user_id, project_id, role): session = get_session() with session.begin(): - session.execute('delete from user_project_role_association where ' - 'user_id=:user_id and project_id=:project_id and ' - 'role=:role', {'user_id': user_id, - 'project_id': project_id, - 'role': role}) + session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(role=role).\ + delete() def user_remove_role(context, user_id, role): @@ -2116,8 +2131,9 @@ def console_delete(context, console_id): session = get_session() with session.begin(): # consoles are meant to be transient. (mdragon) - session.execute('delete from consoles ' - 'where id=:id', {'id': console_id}) + session.query(models.Console).\ + filter_by(id=console_id).\ + delete() def console_get_by_pool_instance(context, pool_id, instance_id): @@ -2273,8 +2289,9 @@ def zone_update(context, zone_id, values): def zone_delete(context, zone_id): session = get_session() with session.begin(): - session.execute('delete from zones ' - 'where id=:id', {'id': zone_id}) + session.query(models.Zone).\ + filter_by(id=zone_id).\ + delete() @require_admin_context -- cgit From 11f2d788fd63c66af0e992f7b75b61273c059bcb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 21:31:47 +0100 Subject: PEP8 --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 3008a512e..87e726394 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -166,9 +166,9 @@ def execute(*cmd, **kwargs): stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one greenthread.sleep(0) return result except ProcessExecutionError: -- cgit From bd06f0ac0d0d3e3c9d7b296c5fe4bb8a0dd44c89 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 20:36:36 +0000 Subject: Last un-magiced session.execute() replaced with SQLAlchemy code... --- nova/db/sqlalchemy/api.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 31adb33ee..88125aaf5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -579,16 +579,17 @@ def fixed_ip_disassociate_all_by_timeout(_context, host, time): session = get_session() # NOTE(vish): The nested select is because sqlite doesn't support # JOINs in UPDATEs. - result = session.execute('UPDATE fixed_ips SET instance_id = NULL, ' - 'leased = 0 ' - 'WHERE network_id IN (SELECT id FROM networks ' - 'WHERE host = :host) ' - 'AND updated_at < :time ' - 'AND instance_id IS NOT NULL ' - 'AND allocated = 0', - {'host': host, - 'time': time}) - return result.rowcount + inner_q = session.query(models.Network.id).\ + filter_by(host=host).\ + subquery() + result = session.query(models.FixedIp).\ + filter(models.FixedIp.network_id.in_(inner_q)).\ + filter(models.FixedIp.updated_at < time).\ + filter(models.FixedIp.instance_id != None).\ + filter_by(allocated=0).\ + update({'instance_id': None, + 'leased': 0}) + return result @require_admin_context -- cgit From be66b329d5b94ffbfb782355ef342eadbaed72a5 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 10 Mar 2011 22:14:53 +0000 Subject: Fix a fer nits jaypipes found in review. --- nova/api/openstack/accounts.py | 18 +++++++++++++++--- nova/api/openstack/users.py | 4 ++-- nova/tests/api/openstack/test_accounts.py | 6 +++--- nova/tests/api/openstack/test_auth.py | 15 ++++----------- nova/tests/api/openstack/test_users.py | 2 +- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 3b90d2776..dd88c3390 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -21,6 +21,7 @@ from nova import log as logging from nova import wsgi from nova.auth import manager +from nova.api.openstack import faults FLAGS = flags.FLAGS LOG = logging.getLogger('nova.api.openstack') @@ -44,11 +45,17 @@ class Controller(wsgi.Controller): self.manager = manager.AuthManager() def _check_admin(self, context): - """ We cannot depend on the db layer to check for admin access - for the auth manager, so we do it here """ + """We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here""" if not context.is_admin: raise exception.NotAuthorized(_("Not admin user.")) + def index(self, req): + raise faults.Fault(exc.HTTPNotImplemented()) + + def detail(self, req): + raise faults.Fault(exc.HTTPNotImplemented()) + def show(self, req, id): """Return data about the given account id""" account = self.manager.get_project(id) @@ -59,8 +66,13 @@ class Controller(wsgi.Controller): self.manager.delete_project(id) return {} + def create(self, req): + """We use update with create-or-update semantics + because the id comes from an external source""" + raise faults.Fault(exc.HTTPNotImplemented()) + def update(self, req, id): - """ This is really create or update. """ + """This is really create or update.""" self._check_admin(req.environ['nova.context']) env = self._deserialize(req.body, req) description = env['account'].get('description') diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index 83ebec964..5bb20a718 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -45,8 +45,8 @@ class Controller(wsgi.Controller): self.manager = manager.AuthManager() def _check_admin(self, context): - """ We cannot depend on the db layer to check for admin access - for the auth manager, so we do it here """ + """We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here""" if not context.is_admin: raise exception.NotAuthorized(_("Not admin user")) diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py index 746f02f57..78fceb47c 100644 --- a/nova/tests/api/openstack/test_accounts.py +++ b/nova/tests/api/openstack/test_accounts.py @@ -14,9 +14,10 @@ # under the License. +import json + import stubout import webob -import json import nova.api import nova.api.openstack.auth @@ -47,8 +48,7 @@ class AccountsTest(test.TestCase): fake_init) self.stubs.Set(nova.api.openstack.accounts.Controller, '_check_admin', fake_admin_check) - fakes.FakeAuthManager.auth_data = {} - fakes.FakeAuthManager.projects = {} + fakes.FakeAuthManager.clear_fakes() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 49f90879d..437a79ec5 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,9 +51,7 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -67,9 +65,7 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -86,7 +82,7 @@ class Test(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/test/fake') + req = webob.Request.blank('/v1.0/fake') req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') @@ -180,9 +176,6 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') @@ -194,7 +187,7 @@ class TestLimiter(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/test/fake') + req = webob.Request.blank'/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py index 14c7897f0..1edefe713 100644 --- a/nova/tests/api/openstack/test_users.py +++ b/nova/tests/api/openstack/test_users.py @@ -13,10 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +import json import stubout import webob -import json import nova.api import nova.api.openstack.auth -- cgit From f251ef70bf83eebce0f851f8a1b052174be1d615 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 10 Mar 2011 22:20:51 +0000 Subject: fix minor typo --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 437a79ec5..ff8d42a14 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -187,7 +187,7 @@ class TestLimiter(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank'/v1.0/fake') + req = webob.Request.blank('/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) -- cgit From 7e95a65ccec2336176f389d614a85c9e70da374d Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 10 Mar 2011 22:33:45 +0000 Subject: re-added a test change I removed thinking it was related to removed code. It wasn't :> --- nova/tests/api/openstack/test_auth.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index ff8d42a14..aaaa4e415 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -65,7 +65,9 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -176,7 +178,9 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' -- cgit From b9a479ffc8e9db0c1888047d7f3df99b3b57b2ec Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 10 Mar 2011 21:44:01 -0500 Subject: Make linux_net ensure_bridge commands that add and remove ip addr's from devices/bridges work with with the latest utils.execute method (execvp). --- nova/network/linux_net.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 9f9d282b6..e69ed2f75 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -513,11 +513,9 @@ def ensure_bridge(bridge, interface, net_attrs=None): for line in out.split("\n"): fields = line.split() if fields and fields[0] == "inet": - params = ' '.join(fields[1:-1]) - _execute('sudo', 'ip', 'addr', - 'del', params, 'dev', fields[-1]) - _execute('sudo', 'ip', 'addr', - 'add', params, 'dev', bridge) + params = fields[1:-1] + _execute(*_ip_bridge_cmd('del', params, fields[-1])) + _execute(*_ip_bridge_cmd('add', params, bridge)) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, @@ -739,3 +737,12 @@ def _ra_pid_for(bridge): if os.path.exists(pid_file): with open(pid_file, 'r') as f: return int(f.read()) + + +def _ip_bridge_cmd(action, params, device): + """Build commands to add/del ips to bridges/devices""" + + cmd = ['sudo', 'ip', 'addr', action] + cmd.extend(params) + cmd.extend(['dev', device]) + return cmd -- cgit From 46d1f6a8c888c1f6fdf12cf26df67eada1e8505b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 11:24:22 +0100 Subject: Use self.instances.pop in unfilter_instance to make the check/removal atomic. Move the semaphore grab outside the for loop in refresh_security_group_rules to avoid reading a value from self.instances, blocking waiting for the semaphore, having the instance be removed in the mean time, and then add its rules back. --- nova/virt/libvirt_conn.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b74ed25f9..d82b33ddd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1238,13 +1238,12 @@ class IptablesFirewallDriver(FirewallDriver): pass def unfilter_instance(self, instance): - if instance['id'] in self.instances: - del self.instances[instance['id']] + if self.instances.pop(instance['id'], False): self.remove_filters_for_instance(instance) self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' - 'filtered'), instance['id']) + 'filtered'), instance['id']) def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance @@ -1387,11 +1386,11 @@ class IptablesFirewallDriver(FirewallDriver): pass def refresh_security_group_rules(self, security_group): - for instance in self.instances.values(): - # We use the semaphore to make sure noone applies the rule set - # after we've yanked the existing rules but before we've put in - # the new ones. - with self.iptables.semaphore: + # We use the semaphore to make sure noone applies the rule set + # after we've yanked the existing rules but before we've put in + # the new ones. + with self.iptables.semaphore: + for instance in self.instances.values(): self.remove_filters_for_instance(instance) self.add_filters_for_instance(instance) self.iptables.apply() -- cgit From 36b5f7d9cf377ce2a4dcdad07e7e14062cd3ec4d Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 11:22:23 -0600 Subject: Further vmops cleanup --- nova/virt/xenapi/vmops.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 5375df5b4..f012fa446 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -536,10 +536,10 @@ class VMOps(object): """ instance_id = instance.id LOG.info(_("Destroying VM for Instance %(instance_id)s") % locals()) - vm = VMHelper.lookup(self._session, instance.name) - return self._destroy(instance, vm, shutdown=True) + vm_ref = VMHelper.lookup(self._session, instance.name) + return self._destroy(instance, vm_ref, shutdown=True) - def _destroy(self, instance, vm, shutdown=True, + def _destroy(self, instance, vm_ref, shutdown=True, destroy_kernel_ramdisk=True): """ Destroys VM instance by performing: @@ -549,17 +549,17 @@ class VMOps(object): 3. Destroying kernel and ramdisk files (if necessary) 4. Destroying that actual VM record """ - if vm is None: + if vm_ref is None: LOG.warning(_("VM is not present, skipping destroy...")) return if shutdown: - self._shutdown(instance, vm) + self._shutdown(instance, vm_ref) - self._destroy_vdis(instance, vm) + self._destroy_vdis(instance, vm_ref) if destroy_kernel_ramdisk: - self._destroy_kernel_ramdisk(instance, vm) - self._destroy_vm(instance, vm) + self._destroy_kernel_ramdisk(instance, vm_ref) + self._destroy_vm(instance, vm_ref) def _wait_with_callback(self, instance_id, task, callback): ret = None -- cgit From cfc7d21b959bc929295868aeb3e84ea56afbfd9c Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Fri, 11 Mar 2011 17:41:22 +0000 Subject: Discovered literal_column(), which does exactly what I need --- nova/db/sqlalchemy/api.py | 49 +++++++++++++++-------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 88125aaf5..431cf6e8e 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -34,6 +34,7 @@ from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload_all from sqlalchemy.sql import exists from sqlalchemy.sql import func +from sqlalchemy.sql.expression import literal_column FLAGS = flags.FLAGS @@ -702,28 +703,16 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.execute('update instances set deleted=1,' - 'deleted_at=:at where id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) - # NOTE(klmitch): for some reason, using the SQLAlchemy code - # here instead of the direct SQL update above causes the - # test_run_terminate_timestamps test (and only that one) to - # fail with an obscure TypeError exception from deep within - # SQLAlchemy; the nearest nova function in the traceback is - # instance_get() - # session.query(models.Instance).\ - # filter_by(id=instance_id).\ - # update({'deleted': 1, - # 'deleted_at': datetime.datetime.utcnow(), - # 'updated_at': models.Instance.updated_at + 0}) + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(instance_id=instance_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupInstanceAssociation. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -969,7 +958,7 @@ def key_pair_destroy_all_by_user(context, user_id): filter_by(user_id=user_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': models.KeyPair.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -1082,7 +1071,7 @@ def network_disassociate_all(context): session = get_session() session.query(models.Network).\ update({'project_id': None, - 'updated_at': models.Network.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -1456,7 +1445,7 @@ def volume_destroy(context, volume_id): filter_by(id=volume_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': models.Volume.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) session.query(models.ExportDevice).\ filter_by(volume_id=volume_id).\ update({'volume_id': None}) @@ -1686,22 +1675,17 @@ def security_group_destroy(context, security_group_id): filter_by(id=security_group_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - models.SecurityGroup.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(security_group_id=security_group_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupInstanceAssociation. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ filter_by(group_id=security_group_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupIngressRule. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -1712,14 +1696,11 @@ def security_group_destroy_all(context, session=None): session.query(models.SecurityGroup).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - models.SecurityGroup.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupIngressRule. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) ################### -- cgit From 195926d0635c0217edccf1cd763425163d3e92e7 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Fri, 11 Mar 2011 19:22:31 +0000 Subject: Minor stylistic updates affecting indentation --- nova/db/sqlalchemy/api.py | 132 +++++++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 89745aa95..08bc8fe2f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -581,15 +581,15 @@ def fixed_ip_disassociate_all_by_timeout(_context, host, time): # NOTE(vish): The nested select is because sqlite doesn't support # JOINs in UPDATEs. inner_q = session.query(models.Network.id).\ - filter_by(host=host).\ - subquery() + filter_by(host=host).\ + subquery() result = session.query(models.FixedIp).\ - filter(models.FixedIp.network_id.in_(inner_q)).\ - filter(models.FixedIp.updated_at < time).\ - filter(models.FixedIp.instance_id != None).\ - filter_by(allocated=0).\ - update({'instance_id': None, - 'leased': 0}) + filter(models.FixedIp.network_id.in_(inner_q)).\ + filter(models.FixedIp.updated_at < time).\ + filter(models.FixedIp.instance_id != None).\ + filter_by(allocated=0).\ + update({'instance_id': None, + 'leased': 0}) return result @@ -704,15 +704,15 @@ def instance_destroy(context, instance_id): session = get_session() with session.begin(): session.query(models.Instance).\ - filter_by(id=instance_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ - filter_by(instance_id=instance_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(instance_id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -955,10 +955,10 @@ def key_pair_destroy_all_by_user(context, user_id): session = get_session() with session.begin(): session.query(models.KeyPair).\ - filter_by(user_id=user_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(user_id=user_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1079,8 +1079,8 @@ def network_disassociate(context, network_id): def network_disassociate_all(context): session = get_session() session.query(models.Network).\ - update({'project_id': None, - 'updated_at': literal_column('updated_at')}) + update({'project_id': None, + 'updated_at': literal_column('updated_at')}) @require_context @@ -1463,16 +1463,16 @@ def volume_destroy(context, volume_id): session = get_session() with session.begin(): session.query(models.Volume).\ - filter_by(id=volume_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(id=volume_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.ExportDevice).\ - filter_by(volume_id=volume_id).\ - update({'volume_id': None}) + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) session.query(models.IscsiTarget).\ - filter_by(volume_id=volume_id).\ - update({'volume_id': None}) + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) @require_admin_context @@ -1693,20 +1693,20 @@ def security_group_destroy(context, security_group_id): session = get_session() with session.begin(): session.query(models.SecurityGroup).\ - filter_by(id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ - filter_by(security_group_id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(security_group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ - filter_by(group_id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1715,13 +1715,13 @@ def security_group_destroy_all(context, session=None): session = get_session() with session.begin(): session.query(models.SecurityGroup).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) ################### @@ -1851,14 +1851,14 @@ def user_delete(context, id): session = get_session() with session.begin(): session.query(models.UserProjectAssociation).\ - filter_by(user_id=id).\ - delete() + filter_by(user_id=id).\ + delete() session.query(models.UserRoleAssociation).\ - filter_by(user_id=id).\ - delete() + filter_by(user_id=id).\ + delete() session.query(models.UserProjectRoleAssociation).\ - filter_by(user_id=id).\ - delete() + filter_by(user_id=id).\ + delete() user_ref = user_get(context, id, session=session) session.delete(user_ref) @@ -1950,11 +1950,11 @@ def project_delete(context, id): session = get_session() with session.begin(): session.query(models.UserProjectAssociation).\ - filter_by(project_id=id).\ - delete() + filter_by(project_id=id).\ + delete() session.query(models.UserProjectRoleAssociation).\ - filter_by(project_id=id).\ - delete() + filter_by(project_id=id).\ + delete() project_ref = project_get(context, id, session=session) session.delete(project_ref) @@ -1980,10 +1980,10 @@ def user_remove_project_role(context, user_id, project_id, role): session = get_session() with session.begin(): session.query(models.UserProjectRoleAssociation).\ - filter_by(user_id=user_id).\ - filter_by(project_id=project_id).\ - filter_by(role=role).\ - delete() + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(role=role).\ + delete() def user_remove_role(context, user_id, role): @@ -2135,8 +2135,8 @@ def console_delete(context, console_id): with session.begin(): # consoles are meant to be transient. (mdragon) session.query(models.Console).\ - filter_by(id=console_id).\ - delete() + filter_by(id=console_id).\ + delete() def console_get_by_pool_instance(context, pool_id, instance_id): @@ -2293,8 +2293,8 @@ def zone_delete(context, zone_id): session = get_session() with session.begin(): session.query(models.Zone).\ - filter_by(id=zone_id).\ - delete() + filter_by(id=zone_id).\ + delete() @require_admin_context -- cgit From 0a130010c26e4ef9ba5b9917ff47766de7805ab9 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Fri, 11 Mar 2011 14:24:03 -0500 Subject: process_input for tee. fixes: 733439 --- nova/virt/disk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 5d499c42c..9abe44cc3 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -189,4 +189,4 @@ def _inject_net_into_fs(net, fs): utils.execute('sudo', 'chown', 'root:root', netdir) utils.execute('sudo', 'chmod', 755, netdir) netfile = os.path.join(netdir, 'interfaces') - utils.execute('sudo', 'tee', netfile, net) + utils.execute('sudo', 'tee', netfile, process_input=net) -- cgit From 2ac7fa75c02c885fc9d4dfacba8318aadbdbfceb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 23:34:26 +0100 Subject: Indentation adjustment (cosmetical). --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index d82b33ddd..678331eed 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1243,7 +1243,7 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' - 'filtered'), instance['id']) + 'filtered'), instance['id']) def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance -- cgit From a9d71273742f440af5687650dd9cd72d827a6bef Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 23:36:28 +0100 Subject: Make the fallback value None instead of False --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 678331eed..d2061a0ca 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1238,7 +1238,7 @@ class IptablesFirewallDriver(FirewallDriver): pass def unfilter_instance(self, instance): - if self.instances.pop(instance['id'], False): + if self.instances.pop(instance['id'], None): self.remove_filters_for_instance(instance) self.iptables.apply() else: -- cgit From b3f5a4d5a8e513fe65a3b1dde9b36fd1388afb67 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Fri, 11 Mar 2011 22:55:56 +0000 Subject: Remove vish comment --- nova/db/sqlalchemy/api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 08bc8fe2f..71b85d659 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -578,8 +578,6 @@ def fixed_ip_disassociate(context, address): @require_admin_context def fixed_ip_disassociate_all_by_timeout(_context, host, time): session = get_session() - # NOTE(vish): The nested select is because sqlite doesn't support - # JOINs in UPDATEs. inner_q = session.query(models.Network.id).\ filter_by(host=host).\ subquery() -- cgit From da76b3d67b2c2e864025c4ba201b63e1dee2ff1f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 16:58:18 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a42cabfe4..866eb5d62 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -101,7 +101,7 @@ class VMHelper(HelperBase): get_instance_type(instance.instance_type) mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) - vm_rec = { + rec = { 'actions_after_crash': 'destroy', 'actions_after_reboot': 'restart', 'actions_after_shutdown': 'destroy', @@ -145,21 +145,21 @@ class VMHelper(HelperBase): vm_rec['platform']['nx'] = 'false' if instance.kernel_id: # 1. Kernel explicitly passed in, use that - vm_rec['PV_args'] = 'root=/dev/xvda1' - vm_rec['PV_kernel'] = kernel - vm_rec['PV_ramdisk'] = ramdisk + rec['PV_args'] = 'root=/dev/xvda1' + rec['PV_kernel'] = kernel + rec['PV_ramdisk'] = ramdisk else: # 2. Use kernel within the image - vm_rec['PV_args'] = 'clocksource=jiffies' - vm_rec['PV_bootloader'] = 'pygrub' + rec['PV_args'] = 'clocksource=jiffies' + rec['PV_bootloader'] = 'pygrub' else: # 3. Using hardware virtualization - vm_rec['platform']['nx'] = 'true' - vm_rec['HVM_boot_params'] = {'order': 'dc'} - vm_rec['HVM_boot_policy'] = 'BIOS order' + rec['platform']['nx'] = 'true' + rec['HVM_boot_params'] = {'order': 'dc'} + rec['HVM_boot_policy'] = 'BIOS order' LOG.debug(_('Created VM %s...'), instance.name) - vm_ref = session.call_xenapi('VM.create', vm_rec) + vm_ref = session.call_xenapi('VM.create', rec) instance_name = instance.name LOG.debug(_('Created VM %(instance_name)s as %(vm_ref)s.') % locals()) return vm_ref -- cgit From 80bc32659e41f496bb1bfefbdd6ca63de7ff9f98 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:11:25 -0600 Subject: Review feedback --- nova/virt/xenapi/vmops.py | 61 +++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 190c2022d..3ccdf9d80 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -434,7 +434,7 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def _shutdown(self, instance, vm, hard=True): + def _shutdown(self, instance, vm_ref, hard=True): """Shutdown an instance""" state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: @@ -448,31 +448,33 @@ class VMOps(object): try: task = None if hard: - task = self._session.call_xenapi("Async.VM.hard_shutdown", vm) + task = self._session.call_xenapi("Async.VM.hard_shutdown", + vm_ref) else: - task = self._session.call_xenapi('Async.VM.clean_shutdown', vm) + task = self._session.call_xenapi("Async.VM.clean_shutdown", + vm_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_vdis(self, instance, vm): - """Destroys all VDIs associated with a VM """ + def _destroy_vdis(self, instance, vm_ref): + """Destroys all VDIs associated with a VM""" instance_id = instance.id LOG.debug(_("Destroying VDIs for Instance %(instance_id)s") % locals()) - vdis = VMHelper.lookup_vm_vdis(self._session, vm) + vdi_refs = VMHelper.lookup_vm_vdis(self._session, vm_ref) - if not vdis: + if not vdi_refs: return - for vdi in vdis: + for vdi_ref in vdi_refs: try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_kernel_ramdisk(self, instance, vm): + def _destroy_kernel_ramdisk(self, instance, vm_ref): """ Three situations can occur: @@ -499,8 +501,8 @@ class VMOps(object): "both" % locals())) # 3. We have both kernel and ramdisk - (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk( - self._session, vm) + (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk(self._session, + vm_ref) LOG.debug(_("Removing kernel/ramdisk files")) @@ -511,11 +513,11 @@ class VMOps(object): LOG.debug(_("kernel/ramdisk files removed")) - def _destroy_vm(self, instance, vm): - """Destroys a VM record """ + def _destroy_vm(self, instance, vm_ref): + """Destroys a VM record""" instance_id = instance.id try: - task = self._session.call_xenapi('Async.VM.destroy', vm) + task = self._session.call_xenapi('Async.VM.destroy', vm_ref) self._session.wait_for_task(task, instance_id) except self.XenAPI.Failure, exc: LOG.exception(exc) @@ -596,8 +598,9 @@ class VMOps(object): - spawn a rescue VM (the vm name-label will be instance-N-rescue) """ - rescue_vm = VMHelper.lookup(self._session, instance.name + "-rescue") - if rescue_vm: + rescue_vm_ref = VMHelper.lookup(self._session, + instance.name + "-rescue") + if rescue_vm_ref: raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) @@ -609,8 +612,8 @@ class VMOps(object): self.spawn(instance) rescue_vm_ref = self._get_vm_opaque_ref(instance) - vbd = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] - vdi_ref = self._session.get_xenapi().VBD.get_record(vbd)["VDI"] + vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] + vdi_ref = self._session.get_xenapi().VBD.get_record(vbd_ref)["VDI"] vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, vdi_ref, 1, False) @@ -623,35 +626,37 @@ class VMOps(object): - release the bootlock to allow the instance VM to start """ - rescue_vm = VMHelper.lookup(self._session, instance.name + "-rescue") + rescue_vm_ref = VMHelper.lookup(self._session, + instance.name + "-rescue") - if not rescue_vm: + if not rescue_vm_ref: raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) original_vm_ref = self._get_vm_opaque_ref(instance) - vbds = self._session.get_xenapi().VM.get_VBDs(rescue_vm) + vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) instance._rescue = False - for vbd_ref in vbds: + for vbd_ref in vbd_refs: vbd = self._session.get_xenapi().VBD.get_record(vbd_ref) if vbd["userdevice"] == "1": VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) - task1 = self._session.call_xenapi("Async.VM.hard_shutdown", rescue_vm) + task1 = self._session.call_xenapi("Async.VM.hard_shutdown", + rescue_vm_ref) self._session.wait_for_task(task1, instance.id) - vdis = VMHelper.lookup_vm_vdis(self._session, rescue_vm) - for vdi in vdis: + vdi_refs = VMHelper.lookup_vm_vdis(self._session, rescue_vm_ref) + for vdi_ref in vdi_refs: try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure: continue - task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm) + task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm_ref) self._session.wait_for_task(task2, instance.id) self._release_bootlock(original_vm_ref) -- cgit From ab37248cc2e40b06e1d349833da01494a9ca3641 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:13:10 -0600 Subject: oops --- nova/virt/xenapi/vm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 866eb5d62..8dd246178 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -142,7 +142,7 @@ class VMHelper(HelperBase): # Complete VM configuration record according to the image type # non-raw/raw with PV kernel/raw in HVM mode if use_pv_kernel: - vm_rec['platform']['nx'] = 'false' + rec['platform']['nx'] = 'false' if instance.kernel_id: # 1. Kernel explicitly passed in, use that rec['PV_args'] = 'root=/dev/xvda1' -- cgit From f9706b5080786a4d3e530f3e8bdb69147e9f5086 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:35:37 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 31 ++++++++++++++++--------------- nova/virt/xenapi/vmops.py | 6 +++--- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 8dd246178..7aa3f3c3b 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -633,37 +633,38 @@ class VMHelper(HelperBase): return is_pv @classmethod - def lookup(cls, session, i): + def lookup(cls, session, name_label): """Look the instance i up, and returns it if available""" - vms = session.get_xenapi().VM.get_by_name_label(i) - n = len(vms) + vm_refs = session.get_xenapi().VM.get_by_name_label(name_label) + n = len(vm_refs) if n == 0: return None elif n > 1: - raise exception.Duplicate(_('duplicate name found: %s') % i) + raise exception.Duplicate(_('duplicate name found: %s') % + name_label) else: - return vms[0] + return vm_refs[0] @classmethod - def lookup_vm_vdis(cls, session, vm): + def lookup_vm_vdis(cls, session, vm_ref): """Look for the VDIs that are attached to the VM""" # Firstly we get the VBDs, then the VDIs. # TODO(Armando): do we leave the read-only devices? - vbds = session.get_xenapi().VM.get_VBDs(vm) - vdis = [] - if vbds: - for vbd in vbds: + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + vdi_refs = [] + if vbd_refs: + for vbd_ref in vbd_refs: try: - vdi = session.get_xenapi().VBD.get_VDI(vbd) + vdi_ref = session.get_xenapi().VBD.get_VDI(vbd_ref) # Test valid VDI - record = session.get_xenapi().VDI.get_record(vdi) + record = session.get_xenapi().VDI.get_record(vdi_ref) LOG.debug(_('VDI %s is still available'), record['uuid']) except cls.XenAPI.Failure, exc: LOG.exception(exc) else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis + vdi_refs.append(vdi_ref) + if len(vdi_refs) > 0: + return vdi_refs else: return None diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3ccdf9d80..0faec1169 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -614,10 +614,10 @@ class VMOps(object): vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] vdi_ref = self._session.get_xenapi().VBD.get_record(vbd_ref)["VDI"] - vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, vdi_ref, - 1, False) + rescue_vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, + vdi_ref, 1, False) - self._session.call_xenapi("Async.VBD.plug", vbd_ref) + self._session.call_xenapi("Async.VBD.plug", rescue_vbd_ref) def unrescue(self, instance, callback): """Unrescue the specified instance -- cgit From cdd8790426d3eb77712f5a19f99211b12a9ad9c5 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:48:44 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 10 +++++----- nova/virt/xenapi/vmops.py | 6 +++--- nova/virt/xenapi/volume_utils.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 7aa3f3c3b..1a872345d 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -202,13 +202,13 @@ class VMHelper(HelperBase): @classmethod def find_vbd_by_number(cls, session, vm_ref, number): """Get the VBD reference from the device number""" - vbds = session.get_xenapi().VM.get_VBDs(vm_ref) - if vbds: - for vbd in vbds: + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + if vbd_refs: + for vbd_ref in vbd_refs: try: - vbd_rec = session.get_xenapi().VBD.get_record(vbd) + vbd_rec = session.get_xenapi().VBD.get_record(vbd_ref) if vbd_rec['userdevice'] == str(number): - return vbd + return vbd_ref except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('VBD not found in instance %s') % vm_ref) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 0faec1169..382915b0c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -55,12 +55,12 @@ class VMOps(object): def list_instances(self): """List VM instances""" - vms = [] + vm_refs = [] for vm_ref in self._session.get_xenapi().VM.get_all(): vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) if not vm_rec["is_a_template"] and not vm_rec["is_control_domain"]: - vms.append(vm_rec["name_label"]) - return vms + vm_refs.append(vm_rec["name_label"]) + return vm_refs def _start(self, instance, vm_ref=None): """Power on a VM instance""" diff --git a/nova/virt/xenapi/volume_utils.py b/nova/virt/xenapi/volume_utils.py index d5ebd29d5..72284ac02 100644 --- a/nova/virt/xenapi/volume_utils.py +++ b/nova/virt/xenapi/volume_utils.py @@ -117,16 +117,16 @@ class VolumeHelper(HelperBase): def introduce_vdi(cls, session, sr_ref): """Introduce VDI in the host""" try: - vdis = session.get_xenapi().SR.get_VDIs(sr_ref) + vdi_refs = session.get_xenapi().SR.get_VDIs(sr_ref) except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('Unable to introduce VDI on SR %s') % sr_ref) try: - vdi_rec = session.get_xenapi().VDI.get_record(vdis[0]) + vdi_rec = session.get_xenapi().VDI.get_record(vdi_refs[0]) except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('Unable to get record' - ' of VDI %s on') % vdis[0]) + ' of VDI %s on') % vdi_refs[0]) else: try: return session.get_xenapi().VDI.introduce( -- cgit From 48196abaf7e9c47bfda3f744e0be9bc242004b72 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 18:00:34 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 56 ++++++++++++++++++++++---------------------- nova/virt/xenapi/vmops.py | 8 +++---- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 1a872345d..4d55937e3 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -443,29 +443,29 @@ class VMHelper(HelperBase): vdi_size += MBR_SIZE_BYTES name_label = get_name_label_for_image(image) - vdi = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) + vdi_ref = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) - with_vdi_attached_here(session, vdi, False, + with_vdi_attached_here(session, vdi_ref, False, lambda dev: _stream_disk(dev, image_type, virtual_size, image_file)) if image_type == ImageType.KERNEL_RAMDISK: #we need to invoke a plugin for copying VDI's #content into proper path - LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi) + LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi_ref) fn = "copy_kernel_vdi" args = {} - args['vdi-ref'] = vdi + args['vdi-ref'] = vdi_ref #let the plugin copy the correct number of bytes args['image-size'] = str(vdi_size) task = session.async_call_plugin('glance', fn, args) filename = session.wait_for_task(task, instance_id) #remove the VDI as it is not needed anymore - session.get_xenapi().VDI.destroy(vdi) - LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi) + session.get_xenapi().VDI.destroy(vdi_ref) + LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi_ref) return filename else: - return session.get_xenapi().VDI.get_uuid(vdi) + return session.get_xenapi().VDI.get_uuid(vdi_ref) @classmethod def determine_disk_image_type(cls, instance): @@ -840,16 +840,16 @@ def safe_find_sr(session): def find_sr(session): """Return the storage repository to hold VM images""" host = session.get_xenapi_host() - srs = session.get_xenapi().SR.get_all() - for sr in srs: - sr_rec = session.get_xenapi().SR.get_record(sr) + sr_refs = session.get_xenapi().SR.get_all() + for sr_ref in sr_refs: + sr_rec = session.get_xenapi().SR.get_record(sr_ref) if not ('i18n-key' in sr_rec['other_config'] and sr_rec['other_config']['i18n-key'] == 'local-storage'): continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.get_xenapi().PBD.get_record(pbd) + for pbd_ref in sr_rec['PBDs']: + pbd_rec = session.get_xenapi().PBD.get_record(pbd_ref) if pbd_rec['host'] == host: - return sr + return sr_ref return None @@ -874,11 +874,11 @@ def remap_vbd_dev(dev): return remapped_dev -def with_vdi_attached_here(session, vdi, read_only, f): +def with_vdi_attached_here(session, vdi_ref, read_only, f): this_vm_ref = get_this_vm_ref(session) vbd_rec = {} vbd_rec['VM'] = this_vm_ref - vbd_rec['VDI'] = vdi + vbd_rec['VDI'] = vdi_ref vbd_rec['userdevice'] = 'autodetect' vbd_rec['bootable'] = False vbd_rec['mode'] = read_only and 'RO' or 'RW' @@ -889,14 +889,14 @@ def with_vdi_attached_here(session, vdi, read_only, f): vbd_rec['qos_algorithm_type'] = '' vbd_rec['qos_algorithm_params'] = {} vbd_rec['qos_supported_algorithms'] = [] - LOG.debug(_('Creating VBD for VDI %s ... '), vdi) - vbd = session.get_xenapi().VBD.create(vbd_rec) - LOG.debug(_('Creating VBD for VDI %s done.'), vdi) + LOG.debug(_('Creating VBD for VDI %s ... '), vdi_ref) + vbd_ref = session.get_xenapi().VBD.create(vbd_rec) + LOG.debug(_('Creating VBD for VDI %s done.'), vdi_ref) try: - LOG.debug(_('Plugging VBD %s ... '), vbd) - session.get_xenapi().VBD.plug(vbd) - LOG.debug(_('Plugging VBD %s done.'), vbd) - orig_dev = session.get_xenapi().VBD.get_device(vbd) + LOG.debug(_('Plugging VBD %s ... '), vbd_ref) + session.get_xenapi().VBD.plug(vbd_ref) + LOG.debug(_('Plugging VBD %s done.'), vbd_ref) + orig_dev = session.get_xenapi().VBD.get_device(vbd_ref) LOG.debug(_('VBD %(vbd)s plugged as %(orig_dev)s') % locals()) dev = remap_vbd_dev(orig_dev) if dev != orig_dev: @@ -904,13 +904,13 @@ def with_vdi_attached_here(session, vdi, read_only, f): 'remapping to %(dev)s') % locals()) return f(dev) finally: - LOG.debug(_('Destroying VBD for VDI %s ... '), vdi) - vbd_unplug_with_retry(session, vbd) - ignore_failure(session.get_xenapi().VBD.destroy, vbd) - LOG.debug(_('Destroying VBD for VDI %s done.'), vdi) + LOG.debug(_('Destroying VBD for VDI %s ... '), vdi_ref) + vbd_unplug_with_retry(session, vbd_ref) + ignore_failure(session.get_xenapi().VBD.destroy, vbd_ref) + LOG.debug(_('Destroying VBD for VDI %s done.'), vdi_ref) -def vbd_unplug_with_retry(session, vbd): +def vbd_unplug_with_retry(session, vbd_ref): """Call VBD.unplug on the given VBD, with a retry if we get DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're seeing the device still in use, even when all processes using the device @@ -918,7 +918,7 @@ def vbd_unplug_with_retry(session, vbd): # FIXME(sirp): We can use LoopingCall here w/o blocking sleep() while True: try: - session.get_xenapi().VBD.unplug(vbd) + session.get_xenapi().VBD.unplug(vbd_ref) LOG.debug(_('VBD.unplug successful first time.')) return except VMHelper.XenAPI.Failure, e: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 382915b0c..fcb290d03 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -87,8 +87,8 @@ class VMOps(object): def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is not None: + vm_ref = VMHelper.lookup(self._session, instance_name) + if vm_ref is not None: raise exception.Duplicate(_('Attempted to create' ' non-unique name %s') % instance_name) @@ -639,8 +639,8 @@ class VMOps(object): instance._rescue = False for vbd_ref in vbd_refs: - vbd = self._session.get_xenapi().VBD.get_record(vbd_ref) - if vbd["userdevice"] == "1": + _vbd_ref = self._session.get_xenapi().VBD.get_record(vbd_ref) + if _vbd_ref["userdevice"] == "1": VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) -- cgit From b944cbcbf023ca321edcc511354b56aa5b07d438 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 18:03:19 -0600 Subject: oops --- nova/virt/xenapi/vm_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 4d55937e3..f07b57796 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -897,10 +897,10 @@ def with_vdi_attached_here(session, vdi_ref, read_only, f): session.get_xenapi().VBD.plug(vbd_ref) LOG.debug(_('Plugging VBD %s done.'), vbd_ref) orig_dev = session.get_xenapi().VBD.get_device(vbd_ref) - LOG.debug(_('VBD %(vbd)s plugged as %(orig_dev)s') % locals()) + LOG.debug(_('VBD %(vbd_ref)s plugged as %(orig_dev)s') % locals()) dev = remap_vbd_dev(orig_dev) if dev != orig_dev: - LOG.debug(_('VBD %(vbd)s plugged into wrong dev, ' + LOG.debug(_('VBD %(vbd_ref)s plugged into wrong dev, ' 'remapping to %(dev)s') % locals()) return f(dev) finally: -- cgit From de1197cfee200782a5a1d07fb40138d4f103890e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sun, 13 Mar 2011 10:49:56 +0100 Subject: Fix instructions for setting up the initial database. --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index a880a9c2f..cb4d18614 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -276,7 +276,7 @@ def _db_error(caught_exception): print caught_exception print _("The above error may show that the database has not " "been created.\nPlease create a database using " - "nova-manage sync db before running this command.") + "'nova-manage db sync' before running this command.") exit(1) -- cgit