From 71b7298788045d4832dd8ec44cba3785955aa847 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Tue, 6 Sep 2011 15:21:15 +0000 Subject: Implement deferred delete of instances --- nova/api/openstack/contrib/deferred_delete.py | 76 +++++++++++++++++++++++++++ nova/compute/api.py | 50 ++++++++++++++++-- nova/compute/manager.py | 59 ++++++++++++++++++--- nova/compute/task_states.py | 2 + nova/virt/driver.py | 8 +++ nova/virt/xenapi/vmops.py | 10 ++++ nova/virt/xenapi_conn.py | 8 +++ 7 files changed, 204 insertions(+), 9 deletions(-) create mode 100644 nova/api/openstack/contrib/deferred_delete.py diff --git a/nova/api/openstack/contrib/deferred_delete.py b/nova/api/openstack/contrib/deferred_delete.py new file mode 100644 index 000000000..54d8aac2a --- /dev/null +++ b/nova/api/openstack/contrib/deferred_delete.py @@ -0,0 +1,76 @@ +# Copyright 2011 Openstack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The deferred instance delete extension.""" + +import webob +from webob import exc + +from nova import compute +from nova import exception +from nova import log as logging +from nova.api.openstack import common +from nova.api.openstack import extensions +from nova.api.openstack import faults +from nova.api.openstack import servers + + +LOG = logging.getLogger("nova.api.contrib.deferred-delete") + + +class Deferred_delete(extensions.ExtensionDescriptor): + def __init__(self): + super(Deferred_delete, self).__init__() + self.compute_api = compute.API() + + def _restore(self, input_dict, req, instance_id): + "Restore a previously deleted instance." + + context = req.environ["nova.context"] + self.compute_api.restore(context, instance_id) + return webob.Response(status_int=202) + + def _force_delete(self, input_dict, req, instance_id): + "Force delete of instance before deferred cleanup." + + context = req.environ["nova.context"] + self.compute_api.force_delete(context, instance_id) + return webob.Response(status_int=202) + + def get_name(self): + return "DeferredDelete" + + def get_alias(self): + return "os-deferred-delete" + + def get_description(self): + return "Instance deferred delete" + + def get_namespace(self): + return "http://docs.openstack.org/ext/deferred-delete/api/v1.1" + + def get_updated(self): + return "2011-09-01T00:00:00+00:00" + + def get_actions(self): + """Return the actions the extension adds, as required by contract.""" + actions = [ + extensions.ActionExtension("servers", "restore", + self._restore), + extensions.ActionExtension("servers", "forceDelete", + self._force_delete), + ] + + return actions diff --git a/nova/compute/api.py b/nova/compute/api.py index e045ef3de..5a73e6d2b 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -49,6 +49,8 @@ FLAGS = flags.FLAGS flags.DECLARE('vncproxy_topic', 'nova.vnc') flags.DEFINE_integer('find_host_timeout', 30, 'Timeout after NN seconds when looking for a host.') +flags.DEFINE_integer('delete_instance_interval', 43200, + 'Time in seconds to wait to scrub deleted instances.') def generate_default_hostname(instance): @@ -759,17 +761,59 @@ class API(base.Base): if not _is_able_to_shutdown(instance, instance_id): return + host = instance['host'] + if FLAGS.delete_instance_interval and host: + self.update(context, + instance_id, + vm_state=vm_states.DELETED, + task_state=task_states.QUEUED_DELETE, + deleted_at=utils.utcnow()) + + self._cast_compute_message('power_off_instance', context, + instance_id, host) + else: + self.update(context, + instance_id, + task_state=task_states.DELETING) + + if host: + self._cast_compute_message('terminate_instance', context, + instance_id, host) + else: + terminate_volumes(self.db, context, instance_id) + self.db.instance_destroy(context, instance_id) + + @scheduler_api.reroute_compute("restore") + def restore(self, context, instance_id): + """Restore a previously deleted (but not reclaimed) instance.""" + instance = self._get_instance(context, instance_id, 'restore') + + self.update(context, + instance_id, + vm_state=vm_states.ACTIVE, + task_state=None, + deleted_at=None) + + # FIXME: How to handle no host? + host = instance['host'] + if host: + self._cast_compute_message('power_on_instance', context, + instance_id, host) + + @scheduler_api.reroute_compute("force_delete") + def force_delete(self, context, instance_id): + """Force delete a previously deleted (but not reclaimed) instance.""" + instance = self._get_instance(context, instance_id, 'force delete') + self.update(context, instance_id, task_state=task_states.DELETING) + # FIXME: How to handle no host? host = instance['host'] if host: self._cast_compute_message('terminate_instance', context, instance_id, host) - else: - terminate_volumes(self.db, context, instance_id) - self.db.instance_destroy(context, instance_id) @scheduler_api.reroute_compute("stop") def stop(self, context, instance_id): diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 0477db745..58625fc8b 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -35,12 +35,13 @@ terminating it. """ +import datetime +import functools import os import socket import sys import tempfile import time -import functools from eventlet import greenthread @@ -174,7 +175,7 @@ class ComputeManager(manager.SchedulerDependentManager): 'nova-compute restart.'), locals()) self.reboot_instance(context, instance['id']) elif drv_state == power_state.RUNNING: - # Hyper-V and VMWareAPI drivers will raise and exception + # Hyper-V and VMWareAPI drivers will raise an exception try: net_info = self._get_instance_nw_info(context, instance) self.driver.ensure_filtering_rules_for_instance(instance, @@ -485,10 +486,8 @@ class ComputeManager(manager.SchedulerDependentManager): if action_str == 'Terminating': terminate_volumes(self.db, context, instance_id) - @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) - @checks_instance_lock - def terminate_instance(self, context, instance_id): - """Terminate an instance on this host.""" + def _delete_instance(self, context, instance_id): + """Delete an instance on this host.""" self._shutdown_instance(context, instance_id, 'Terminating') instance = self.db.instance_get(context.elevated(), instance_id) self._instance_update(context, @@ -504,6 +503,12 @@ class ComputeManager(manager.SchedulerDependentManager): 'compute.instance.delete', notifier.INFO, usage_info) + @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) + @checks_instance_lock + def terminate_instance(self, context, instance_id): + """Terminate an instance on this host.""" + self._delete_instance(context, instance_id) + @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) @checks_instance_lock def stop_instance(self, context, instance_id): @@ -514,6 +519,28 @@ class ComputeManager(manager.SchedulerDependentManager): vm_state=vm_states.STOPPED, task_state=None) + @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) + @checks_instance_lock + def power_off_instance(self, context, instance_id): + """Power off an instance on this host.""" + instance = self.db.instance_get(context, instance_id) + self.driver.power_off(instance) + current_power_state = self._get_power_state(context, instance) + self._instance_update(context, + instance_id, + power_state=current_power_state) + + @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) + @checks_instance_lock + def power_on_instance(self, context, instance_id): + """Power on an instance on this host.""" + instance = self.db.instance_get(context, instance_id) + self.driver.power_on(instance) + current_power_state = self._get_power_state(context, instance) + self._instance_update(context, + instance_id, + power_state=current_power_state) + @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) @checks_instance_lock def rebuild_instance(self, context, instance_id, **kwargs): @@ -1659,6 +1686,13 @@ class ComputeManager(manager.SchedulerDependentManager): LOG.warning(_("Error during power_state sync: %s"), unicode(ex)) error_list.append(ex) + try: + self._reclaim_queued_deletes(context) + except Exception as ex: + LOG.warning(_("Error during reclamation of queued deletes: %s"), + unicode(ex)) + error_list.append(ex) + return error_list def _report_driver_status(self): @@ -1708,3 +1742,16 @@ class ComputeManager(manager.SchedulerDependentManager): self._instance_update(context, db_instance["id"], power_state=vm_power_state) + + def _reclaim_queued_deletes(self, context): + # Find all instances that are queued for deletion + instances = self.db.instance_get_all_by_host(context, self.host) + + curtime = utils.utcnow() + for instance in instances: + LOG.info('name: %r, state: %r, deleted_at: %r' % (instance['name'], instance['task_state'], instance['deleted_at'])) + if instance['task_state'] == task_states.QUEUED_DELETE and \ + (curtime - instance['deleted_at'] >= + datetime.timedelta(seconds=FLAGS.delete_instance_interval)): + LOG.info('Deleting %s' % instance['name']) + self._delete_instance(context, instance['id']) diff --git a/nova/compute/task_states.py b/nova/compute/task_states.py index e3315a542..0337b2641 100644 --- a/nova/compute/task_states.py +++ b/nova/compute/task_states.py @@ -54,6 +54,8 @@ RESUMING = 'resuming' RESCUING = 'rescuing' UNRESCUING = 'unrescuing' +QUEUED_DELETE = 'queued_delete' + DELETING = 'deleting' STOPPING = 'stopping' STARTING = 'starting' diff --git a/nova/virt/driver.py b/nova/virt/driver.py index d05b51bd9..69d50717a 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -286,6 +286,14 @@ class ComputeDriver(object): # TODO(Vek): Need to pass context in for access to auth_token raise NotImplementedError() + def power_off(self, instance): + """Power off the specified instance.""" + raise NotImplementedError() + + def power_on(self, instance): + """Power on the specified instance""" + raise NotImplementedError() + def update_available_resource(self, ctxt, host): """Updates compute manager resource info on ComputeNode table. diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c5f105f40..83ec96b8a 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -988,6 +988,16 @@ class VMOps(object): self._release_bootlock(original_vm_ref) self._start(instance, original_vm_ref) + def power_off(self, instance): + """Power off the specified instance.""" + vm_ref = self._get_vm_opaque_ref(instance) + self._shutdown(instance, vm_ref, hard=True) + + def power_on(self, instance): + """Power on the specified instance.""" + vm_ref = self._get_vm_opaque_ref(instance) + self._start(instance, vm_ref) + def poll_rescued_instances(self, timeout): """Look for expirable rescued instances. diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 0d23e7689..69a17f721 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -250,6 +250,14 @@ class XenAPIConnection(driver.ComputeDriver): """Unrescue the specified instance""" self._vmops.unrescue(instance, _callback) + def power_off(self, instance): + """Power off the specified instance""" + self._vmops.power_off(instance) + + def power_on(self, instance): + """Power on the specified instance""" + self._vmops.power_on(instance) + def poll_rescued_instances(self, timeout): """Poll for rescued instances""" self._vmops.poll_rescued_instances(timeout) -- cgit From 3d37846624839d239e35b6c91aa4357844585d36 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Tue, 6 Sep 2011 19:12:02 +0000 Subject: Include new extension --- nova/tests/api/openstack/test_extensions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 31443242b..e12a8b969 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -86,6 +86,7 @@ class ExtensionControllerTest(test.TestCase): self.flags(osapi_extensions_path=ext_path) self.ext_list = [ "Createserverext", + "DeferredDelete", "FlavorExtraSpecs", "Floating_ips", "Fox In Socks", -- cgit From 124f577387f8cec5098ea783db32b80e4d677c59 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Tue, 6 Sep 2011 19:12:22 +0000 Subject: PEP8 cleanups --- nova/compute/manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 58625fc8b..6c3d7044f 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1744,14 +1744,14 @@ class ComputeManager(manager.SchedulerDependentManager): power_state=vm_power_state) def _reclaim_queued_deletes(self, context): - # Find all instances that are queued for deletion + "Reclaim instances that are queued for deletion." + instances = self.db.instance_get_all_by_host(context, self.host) + queue_time = datetime.timedelta(seconds=FLAGS.delete_instance_interval) curtime = utils.utcnow() for instance in instances: - LOG.info('name: %r, state: %r, deleted_at: %r' % (instance['name'], instance['task_state'], instance['deleted_at'])) if instance['task_state'] == task_states.QUEUED_DELETE and \ - (curtime - instance['deleted_at'] >= - datetime.timedelta(seconds=FLAGS.delete_instance_interval)): + (curtime - instance['deleted_at']) >= queue_time: LOG.info('Deleting %s' % instance['name']) self._delete_instance(context, instance['id']) -- cgit From c11fa8152afce89bddeca0ca92693e00c08a34af Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Tue, 6 Sep 2011 19:12:36 +0000 Subject: Default to 0 seconds (off) --- nova/compute/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 6037903c4..cb6cb6ab2 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -49,7 +49,7 @@ FLAGS = flags.FLAGS flags.DECLARE('vncproxy_topic', 'nova.vnc') flags.DEFINE_integer('find_host_timeout', 30, 'Timeout after NN seconds when looking for a host.') -flags.DEFINE_integer('delete_instance_interval', 43200, +flags.DEFINE_integer('delete_instance_interval', 0, 'Time in seconds to wait to scrub deleted instances.') -- cgit From 8f523e90d5c5da15fc72d33ec9abdd47798f5c7c Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 15:31:38 +0000 Subject: Tests for deferred delete, restore and forceDelete --- nova/tests/integrated/test_servers.py | 123 ++++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 14 deletions(-) diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index 2cf604d06..b56fd7692 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -28,15 +28,14 @@ LOG = logging.getLogger('nova.tests.integrated') class ServersTest(integrated_helpers._IntegratedTestBase): - def _wait_for_creation(self, server): - retries = 0 - while server['status'] == 'BUILD': - time.sleep(1) + def _wait_for_state_change(self, server, status): + for i in xrange(0, 50): server = self.api.get_server(server['id']) print server - retries = retries + 1 - if retries > 5: + if server['status'] != status: break + time.sleep(.1) + return server def test_get_servers(self): @@ -102,7 +101,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): server_ids = [server['id'] for server in servers] self.assertTrue(created_server_id in server_ids) - found_server = self._wait_for_creation(found_server) + found_server = self._wait_for_state_change(found_server, 'BUILD') # It should be available... # TODO(justinsb): Mock doesn't yet do this... @@ -114,12 +113,103 @@ class ServersTest(integrated_helpers._IntegratedTestBase): self._delete_server(created_server_id) - def _delete_server(self, server_id): + def test_deferred_delete(self): + """Creates and deletes a server.""" + self.flags(stub_network=True, delete_instance_interval=1) + + # Create server + server = self._build_minimal_create_server_request() + + created_server = self.api.post_server({'server': server}) + LOG.debug("created_server: %s" % created_server) + self.assertTrue(created_server['id']) + created_server_id = created_server['id'] + + # Wait for it to finish being created + found_server = self._wait_for_state_change(created_server, 'BUILD') + + # It should be available... + # TODO(justinsb): Mock doesn't yet do this... + self.assertEqual('ACTIVE', found_server['status']) + # Delete the server - self.api.delete_server(server_id) + self.api.delete_server(created_server_id) + + # Wait for queued deletion + found_server = self._wait_for_state_change(found_server, 'ACTIVE') + self.assertEqual('DELETED', found_server['status']) + + # Wait for real deletion + self._wait_for_deletion(created_server_id) + + def test_deferred_delete_restore(self): + """Creates, deletes and restores a server.""" + self.flags(stub_network=True, delete_instance_interval=1) + + # Create server + server = self._build_minimal_create_server_request() + created_server = self.api.post_server({'server': server}) + LOG.debug("created_server: %s" % created_server) + self.assertTrue(created_server['id']) + created_server_id = created_server['id'] + + # Wait for it to finish being created + found_server = self._wait_for_state_change(created_server, 'BUILD') + + # It should be available... + # TODO(justinsb): Mock doesn't yet do this... + self.assertEqual('ACTIVE', found_server['status']) + + # Delete the server + self.api.delete_server(created_server_id) + + # Wait for queued deletion + found_server = self._wait_for_state_change(found_server, 'ACTIVE') + self.assertEqual('DELETED', found_server['status']) + + # Restore server + self.api.post_server_action(created_server_id, {'restore': {}}) + + # Wait for server to become active again + found_server = self._wait_for_state_change(found_server, 'DELETED') + self.assertEqual('ACTIVE', found_server['status']) + + def test_deferred_delete_force(self): + """Creates, deletes and force deletes a server.""" + self.flags(stub_network=True, delete_instance_interval=1) + + # Create server + server = self._build_minimal_create_server_request() + + created_server = self.api.post_server({'server': server}) + LOG.debug("created_server: %s" % created_server) + self.assertTrue(created_server['id']) + created_server_id = created_server['id'] + + # Wait for it to finish being created + found_server = self._wait_for_state_change(created_server, 'BUILD') + + # It should be available... + # TODO(justinsb): Mock doesn't yet do this... + self.assertEqual('ACTIVE', found_server['status']) + + # Delete the server + self.api.delete_server(created_server_id) + + # Wait for queued deletion + found_server = self._wait_for_state_change(found_server, 'ACTIVE') + self.assertEqual('DELETED', found_server['status']) + + # Force delete server + self.api.post_server_action(created_server_id, {'forceDelete': {}}) + + # Wait for real deletion + self._wait_for_deletion(created_server_id) + + def _wait_for_deletion(self, server_id): # Wait (briefly) for deletion - for _retries in range(5): + for _retries in range(500): try: found_server = self.api.get_server(server_id) except client.OpenStackApiNotFoundException: @@ -132,11 +222,16 @@ class ServersTest(integrated_helpers._IntegratedTestBase): # TODO(justinsb): Mock doesn't yet do accurate state changes #if found_server['status'] != 'deleting': # break - time.sleep(1) + time.sleep(.1) # Should be gone self.assertFalse(found_server) + def _delete_server(self, server_id): + # Delete the server + self.api.delete_server(server_id) + self._wait_for_deletion(server_id) + def test_create_server_with_metadata(self): """Creates a server with metadata.""" @@ -194,7 +289,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): self.assertTrue(created_server['id']) created_server_id = created_server['id'] - created_server = self._wait_for_creation(created_server) + created_server = self._wait_for_state_change(created_server, 'BUILD') # rebuild the server with metadata post = {} @@ -228,7 +323,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): self.assertTrue(created_server['id']) created_server_id = created_server['id'] - created_server = self._wait_for_creation(created_server) + created_server = self._wait_for_state_change(created_server, 'BUILD') # rebuild the server with metadata post = {} @@ -274,7 +369,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): self.assertTrue(created_server['id']) created_server_id = created_server['id'] - created_server = self._wait_for_creation(created_server) + created_server = self._wait_for_state_change(created_server, 'BUILD') # rebuild the server with metadata post = {} -- cgit From 3aa265623cf42a1e94aedbc1ec17656302a36761 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 22:32:54 +0000 Subject: Restart compute with a lower periodic_interval to make test run faster --- nova/tests/integrated/integrated_helpers.py | 8 ++++---- nova/tests/integrated/test_servers.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py index 343190427..964ddfa37 100644 --- a/nova/tests/integrated/integrated_helpers.py +++ b/nova/tests/integrated/integrated_helpers.py @@ -70,10 +70,10 @@ class _IntegratedTestBase(test.TestCase): self.stubs.Set(nova.image, 'get_image_service', fake_get_image_service) # set up services - self.start_service('compute') - self.start_service('volume') - self.start_service('network') - self.start_service('scheduler') + self.compute = self.start_service('compute') + self.volume = self.start_service('volume') + self.network = self.start_service('network') + self.scheduler = self.start_service('scheduler') self._start_api_service() diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index b56fd7692..9ca9f022e 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -38,6 +38,15 @@ class ServersTest(integrated_helpers._IntegratedTestBase): return server + def _restart_compute_service(self, periodic_interval=None): + """restart compute service. NOTE: fake driver forgets all instances.""" + self.compute.kill() + if periodic_interval: + self.compute = self.start_service( + 'compute', periodic_interval=periodic_interval) + else: + self.compute = self.start_service('compute') + def test_get_servers(self): """Simple check that listing servers works.""" servers = self.api.get_servers() @@ -117,6 +126,9 @@ class ServersTest(integrated_helpers._IntegratedTestBase): """Creates and deletes a server.""" self.flags(stub_network=True, delete_instance_interval=1) + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + # Create server server = self._build_minimal_create_server_request() @@ -209,7 +221,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): def _wait_for_deletion(self, server_id): # Wait (briefly) for deletion - for _retries in range(500): + for _retries in range(50): try: found_server = self.api.get_server(server_id) except client.OpenStackApiNotFoundException: -- cgit From d046c059ed421cfe901c00d53414dfe20be66c97 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 22:33:59 +0000 Subject: PEP8 cleanup --- nova/tests/api/openstack/test_servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 2ef687709..d063a60c2 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -3615,7 +3615,7 @@ class TestGetKernelRamdiskFromImage(test.TestCase): self.assertRaises(exception.NotFound, self._get_k_r, image_meta) def test_ami_no_ramdisk(self): - """If an ami is missing a ramdisk, return kernel ID and None for + """If an ami is missing a ramdisk, return kernel ID and None for ramdisk ID """ image_meta = {'id': 1, 'status': 'active', 'container_format': 'ami', -- cgit From 2655520793dfcfa5b4e779c15abfd2490b193a94 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 22:39:15 +0000 Subject: delete_instance_interval -> reclaim_instance_interval --- nova/compute/api.py | 4 +--- nova/compute/manager.py | 5 ++++- nova/tests/integrated/test_servers.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 24c8a6354..7c8b3cbfe 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -49,8 +49,6 @@ FLAGS = flags.FLAGS flags.DECLARE('vncproxy_topic', 'nova.vnc') flags.DEFINE_integer('find_host_timeout', 30, 'Timeout after NN seconds when looking for a host.') -flags.DEFINE_integer('delete_instance_interval', 0, - 'Time in seconds to wait to scrub deleted instances.') def generate_default_hostname(instance): @@ -762,7 +760,7 @@ class API(base.Base): return host = instance['host'] - if FLAGS.delete_instance_interval and host: + if FLAGS.reclaim_instance_interval and host: self.update(context, instance_id, vm_state=vm_states.DELETED, diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6c3d7044f..314fd4983 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -84,6 +84,8 @@ flags.DEFINE_integer("rescue_timeout", 0, " Set to 0 to disable.") flags.DEFINE_integer('host_state_interval', 120, 'Interval in seconds for querying the host status') +flags.DEFINE_integer('reclaim_instance_interval', 0, + 'Interval in seconds for reclaiming deleted instances') LOG = logging.getLogger('nova.compute.manager') @@ -1748,7 +1750,8 @@ class ComputeManager(manager.SchedulerDependentManager): instances = self.db.instance_get_all_by_host(context, self.host) - queue_time = datetime.timedelta(seconds=FLAGS.delete_instance_interval) + queue_time = datetime.timedelta( + seconds=FLAGS.reclaim_instance_interval) curtime = utils.utcnow() for instance in instances: if instance['task_state'] == task_states.QUEUED_DELETE and \ diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index 9ca9f022e..6ea165b82 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -124,7 +124,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): def test_deferred_delete(self): """Creates and deletes a server.""" - self.flags(stub_network=True, delete_instance_interval=1) + self.flags(stub_network=True, reclaim_instance_interval=1) # enforce periodic tasks run in short time to avoid wait for 60s. self._restart_compute_service(periodic_interval=0.3) @@ -156,7 +156,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): def test_deferred_delete_restore(self): """Creates, deletes and restores a server.""" - self.flags(stub_network=True, delete_instance_interval=1) + self.flags(stub_network=True, reclaim_instance_interval=1) # Create server server = self._build_minimal_create_server_request() @@ -189,7 +189,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): def test_deferred_delete_force(self): """Creates, deletes and force deletes a server.""" - self.flags(stub_network=True, delete_instance_interval=1) + self.flags(stub_network=True, reclaim_instance_interval=1) # Create server server = self._build_minimal_create_server_request() -- cgit From 7b73ecf540b9bd14bdc97306018eed170b0959eb Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 22:49:24 +0000 Subject: Add local hostname to fix Authors test --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 5c8df80e0..6b0df4019 100644 --- a/.mailmap +++ b/.mailmap @@ -24,6 +24,7 @@ + -- cgit From 4968aade5ce77d1068a014baf5d520b94fd668dc Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 22:59:13 +0000 Subject: Make sure instance is deleted before allowing restore or forceDelete --- nova/compute/api.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index 7c8b3cbfe..6ff8d96a1 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -92,6 +92,18 @@ def _is_able_to_shutdown(instance, instance_id): return True +def _is_delete_queued(instance, instance_id): + vm_state = instance["vm_state"] + task_state = instance["task_state"] + + if vm_state != vm_states.DELETED: + LOG.warn(_("Instance %(instance_id)s is not in a 'deleted' state. It " + "is currently %(vm_state)s. Action aborted.") % locals()) + return False + + return True + + class API(base.Base): """API for interacting with the compute manager.""" @@ -786,6 +798,9 @@ class API(base.Base): """Restore a previously deleted (but not reclaimed) instance.""" instance = self._get_instance(context, instance_id, 'restore') + if not _is_delete_queued(instance, instance_id): + return + self.update(context, instance_id, vm_state=vm_states.ACTIVE, @@ -803,6 +818,9 @@ class API(base.Base): """Force delete a previously deleted (but not reclaimed) instance.""" instance = self._get_instance(context, instance_id, 'force delete') + if not _is_delete_queued(instance, instance_id): + return + self.update(context, instance_id, task_state=task_states.DELETING) -- cgit From da40f02dc7ef4e568ef872628402f11f666e3e2b Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 23:03:18 +0000 Subject: Check task_state for queued delete --- nova/compute/api.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 6ff8d96a1..99166b0af 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -92,13 +92,14 @@ def _is_able_to_shutdown(instance, instance_id): return True -def _is_delete_queued(instance, instance_id): +def _is_queued_delete(instance, instance_id): vm_state = instance["vm_state"] task_state = instance["task_state"] - if vm_state != vm_states.DELETED: - LOG.warn(_("Instance %(instance_id)s is not in a 'deleted' state. It " - "is currently %(vm_state)s. Action aborted.") % locals()) + if task_state != task_states.QUEUED_DELETE: + LOG.warn(_("Instance %(instance_id)s is not in a 'queued deleted' " + "state. It is currently %(task_state)s. Action aborted.") % + locals()) return False return True @@ -798,7 +799,7 @@ class API(base.Base): """Restore a previously deleted (but not reclaimed) instance.""" instance = self._get_instance(context, instance_id, 'restore') - if not _is_delete_queued(instance, instance_id): + if not _is_queued_delete(instance, instance_id): return self.update(context, @@ -818,7 +819,7 @@ class API(base.Base): """Force delete a previously deleted (but not reclaimed) instance.""" instance = self._get_instance(context, instance_id, 'force delete') - if not _is_delete_queued(instance, instance_id): + if not _is_queued_delete(instance, instance_id): return self.update(context, -- cgit From d55394af12ab9507d0284f222c1acb66206b187b Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 7 Sep 2011 23:11:38 +0000 Subject: Ensure restore and forceDelete don't do anything unless the server is waiting to be reclaimed --- nova/tests/integrated/test_servers.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index 6ea165b82..bffbe08fc 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -123,7 +123,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): self._delete_server(created_server_id) def test_deferred_delete(self): - """Creates and deletes a server.""" + """Creates, deletes and waits for server to be reclaimed.""" self.flags(stub_network=True, reclaim_instance_interval=1) # enforce periodic tasks run in short time to avoid wait for 60s. @@ -144,6 +144,20 @@ class ServersTest(integrated_helpers._IntegratedTestBase): # TODO(justinsb): Mock doesn't yet do this... self.assertEqual('ACTIVE', found_server['status']) + # Cannot restore unless instance is deleted + self.api.post_server_action(created_server_id, {'restore': {}}) + + # Check it's still active + found_server = self.api.get_server(created_server_id) + self.assertEqual('ACTIVE', found_server['status']) + + # Cannot forceDelete unless instance is deleted + self.api.post_server_action(created_server_id, {'forceDelete': {}}) + + # Check it's still active + found_server = self.api.get_server(created_server_id) + self.assertEqual('ACTIVE', found_server['status']) + # Delete the server self.api.delete_server(created_server_id) -- cgit From 35eb007a07a14f2bf389e53e771114217f1b018e Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Thu, 8 Sep 2011 13:53:31 +0000 Subject: Cleanup some comments --- nova/tests/integrated/test_servers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index bffbe08fc..e9c79aa13 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -141,7 +141,6 @@ class ServersTest(integrated_helpers._IntegratedTestBase): found_server = self._wait_for_state_change(created_server, 'BUILD') # It should be available... - # TODO(justinsb): Mock doesn't yet do this... self.assertEqual('ACTIVE', found_server['status']) # Cannot restore unless instance is deleted @@ -184,7 +183,6 @@ class ServersTest(integrated_helpers._IntegratedTestBase): found_server = self._wait_for_state_change(created_server, 'BUILD') # It should be available... - # TODO(justinsb): Mock doesn't yet do this... self.assertEqual('ACTIVE', found_server['status']) # Delete the server @@ -217,7 +215,6 @@ class ServersTest(integrated_helpers._IntegratedTestBase): found_server = self._wait_for_state_change(created_server, 'BUILD') # It should be available... - # TODO(justinsb): Mock doesn't yet do this... self.assertEqual('ACTIVE', found_server['status']) # Delete the server -- cgit From 15fe2208e858434c6a6f8ee329055da2ce77b348 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Thu, 8 Sep 2011 13:53:51 +0000 Subject: Cleanup some more comments --- nova/compute/api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 99166b0af..81747d09c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -808,7 +808,6 @@ class API(base.Base): task_state=None, deleted_at=None) - # FIXME: How to handle no host? host = instance['host'] if host: self._cast_compute_message('power_on_instance', context, @@ -826,7 +825,6 @@ class API(base.Base): instance_id, task_state=task_states.DELETING) - # FIXME: How to handle no host? host = instance['host'] if host: self._cast_compute_message('terminate_instance', context, -- cgit From 3279898ffcd66870b8523e5281993311a513f0f9 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Fri, 9 Sep 2011 18:33:36 +0000 Subject: Use triple quotes for docstrings to be consistent --- nova/api/openstack/contrib/deferred_delete.py | 4 ++-- nova/compute/manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/contrib/deferred_delete.py b/nova/api/openstack/contrib/deferred_delete.py index 54d8aac2a..0dbee682e 100644 --- a/nova/api/openstack/contrib/deferred_delete.py +++ b/nova/api/openstack/contrib/deferred_delete.py @@ -36,14 +36,14 @@ class Deferred_delete(extensions.ExtensionDescriptor): self.compute_api = compute.API() def _restore(self, input_dict, req, instance_id): - "Restore a previously deleted instance." + """Restore a previously deleted instance.""" context = req.environ["nova.context"] self.compute_api.restore(context, instance_id) return webob.Response(status_int=202) def _force_delete(self, input_dict, req, instance_id): - "Force delete of instance before deferred cleanup." + """Force delete of instance before deferred cleanup.""" context = req.environ["nova.context"] self.compute_api.force_delete(context, instance_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 314fd4983..5c1cbd5f7 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1746,7 +1746,7 @@ class ComputeManager(manager.SchedulerDependentManager): power_state=vm_power_state) def _reclaim_queued_deletes(self, context): - "Reclaim instances that are queued for deletion." + """Reclaim instances that are queued for deletion.""" instances = self.db.instance_get_all_by_host(context, self.host) -- cgit From 2351c06f50d4556e564a7b0abb1653805e661330 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Fri, 9 Sep 2011 18:38:01 +0000 Subject: Make whitespace consistent --- nova/api/openstack/contrib/deferred_delete.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/contrib/deferred_delete.py b/nova/api/openstack/contrib/deferred_delete.py index 0dbee682e..13ee5511e 100644 --- a/nova/api/openstack/contrib/deferred_delete.py +++ b/nova/api/openstack/contrib/deferred_delete.py @@ -67,10 +67,10 @@ class Deferred_delete(extensions.ExtensionDescriptor): def get_actions(self): """Return the actions the extension adds, as required by contract.""" actions = [ - extensions.ActionExtension("servers", "restore", - self._restore), - extensions.ActionExtension("servers", "forceDelete", - self._force_delete), + extensions.ActionExtension("servers", "restore", + self._restore), + extensions.ActionExtension("servers", "forceDelete", + self._force_delete), ] return actions -- cgit From cc86ca1ddc3c5b33d1469619ae491bf09c1ac6b5 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Mon, 12 Sep 2011 22:59:46 +0000 Subject: PEP8 cleanup --- nova/tests/api/openstack/test_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index c63d1203a..10c7663ff 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -1182,7 +1182,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): # Snapshot for User 1 server_ref = 'http://localhost/v1.1/servers/42' snapshot_properties = {'instance_ref': server_ref, 'user_id': 'fake'} - statuses = ('queued', 'saving', 'active','killed', + statuses = ('queued', 'saving', 'active', 'killed', 'deleted', 'pending_delete') for status in statuses: add_fixture(id=image_id, name='%s snapshot' % status, -- cgit From a5b339fb75e1e5f525a758ea1fb2fb35d1b9044a Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 14 Sep 2011 22:31:00 +0000 Subject: Cleanup state management to use vm_state instead of task_state Add schedule_delete() method so delete() actually does what it says it does --- nova/api/ec2/cloud.py | 1 + nova/api/openstack/common.py | 3 +++ nova/api/openstack/servers.py | 10 ++++++-- nova/compute/api.py | 55 ++++++++++++++++++++++++++++++------------- nova/compute/manager.py | 8 ++++--- nova/compute/task_states.py | 4 ++-- nova/compute/vm_states.py | 1 + 7 files changed, 58 insertions(+), 24 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0efb90d6e..ee9d658e8 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -89,6 +89,7 @@ _STATE_DESCRIPTION_MAP = { vm_states.BUILDING: 'pending', vm_states.REBUILDING: 'pending', vm_states.DELETED: 'terminated', + vm_states.SOFT_DELETE: 'terminated', vm_states.STOPPED: 'stopped', vm_states.MIGRATING: 'migrate', vm_states.RESIZING: 'resize', diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index a836a584c..66e18c557 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -78,6 +78,9 @@ _STATE_MAP = { vm_states.DELETED: { 'default': 'DELETED', }, + vm_states.SOFT_DELETE: { + 'default': 'DELETED', + }, } diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 5affd1f33..c81deb3ac 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -169,6 +169,12 @@ class Controller(object): server['server']['adminPass'] = extra_values['password'] return server + def _delete(self, context, id): + if FLAGS.reclaim_instance_interval: + self.compute_api.soft_delete(context, id) + else: + self.compute_api.delete(context, id) + @scheduler_api.redirect_handler def update(self, req, id, body): """Update server then pass on to version-specific controller""" @@ -566,7 +572,7 @@ class ControllerV10(Controller): def delete(self, req, id): """ Destroys a server """ try: - self.compute_api.delete(req.environ['nova.context'], id) + self._delete(req.environ['nova.context'], id) except exception.NotFound: raise exc.HTTPNotFound() return webob.Response(status_int=202) @@ -644,7 +650,7 @@ class ControllerV11(Controller): def delete(self, req, id): """ Destroys a server """ try: - self.compute_api.delete(req.environ['nova.context'], id) + self._delete(req.environ['nova.context'], id) except exception.NotFound: raise exc.HTTPNotFound() diff --git a/nova/compute/api.py b/nova/compute/api.py index ba314efb6..1a34b41a6 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -96,9 +96,9 @@ def _is_queued_delete(instance, instance_id): vm_state = instance["vm_state"] task_state = instance["task_state"] - if task_state != task_states.QUEUED_DELETE: - LOG.warn(_("Instance %(instance_id)s is not in a 'queued deleted' " - "state. It is currently %(task_state)s. Action aborted.") % + if vm_state != vm_states.SOFT_DELETE: + LOG.warn(_("Instance %(instance_id)s is not in a 'soft delete' " + "state. It is currently %(vm_state)s. Action aborted.") % locals()) return False @@ -764,36 +764,54 @@ class API(base.Base): {'instance_id': instance_id, 'action_str': action_str}) raise - @scheduler_api.reroute_compute("delete") - def delete(self, context, instance_id): + @scheduler_api.reroute_compute("soft_delete") + def soft_delete(self, context, instance_id): """Terminate an instance.""" - LOG.debug(_("Going to try to terminate %s"), instance_id) - instance = self._get_instance(context, instance_id, 'terminating') + LOG.debug(_("Going to try to soft delete %s"), instance_id) + instance = self._get_instance(context, instance_id, 'soft delete') if not _is_able_to_shutdown(instance, instance_id): return + # NOTE(jerdfelt): The compute daemon handles reclaiming instances + # that are in soft delete. If there is no host assigned, there is + # no daemon to reclaim, so delete it immediately. host = instance['host'] - if FLAGS.reclaim_instance_interval and host: + if host: self.update(context, instance_id, - vm_state=vm_states.DELETED, - task_state=task_states.QUEUED_DELETE, + vm_state=vm_states.SOFT_DELETE, + task_state=task_states.POWERING_OFF, deleted_at=utils.utcnow()) self._cast_compute_message('power_off_instance', context, - instance_id, host) + instance_id, host) else: + LOG.warning(_("No host for instance %s, deleting immediately"), + instance_id) + terminate_volumes(self.db, context, instance_id) + self.db.instance_destroy(context, instance_id) + + @scheduler_api.reroute_compute("delete") + def delete(self, context, instance_id): + """Terminate an instance.""" + LOG.debug(_("Going to try to terminate %s"), instance_id) + instance = self._get_instance(context, instance_id, 'delete') + + if not _is_able_to_shutdown(instance, instance_id): + return + + host = instance['host'] + if host: self.update(context, instance_id, task_state=task_states.DELETING) - if host: - self._cast_compute_message('terminate_instance', context, - instance_id, host) - else: - terminate_volumes(self.db, context, instance_id) - self.db.instance_destroy(context, instance_id) + self._cast_compute_message('terminate_instance', context, + instance_id, host) + else: + terminate_volumes(self.db, context, instance_id) + self.db.instance_destroy(context, instance_id) @scheduler_api.reroute_compute("restore") def restore(self, context, instance_id): @@ -811,6 +829,9 @@ class API(base.Base): host = instance['host'] if host: + self.update(context, + instance_id, + task_state=task_states.POWERING_ON) self._cast_compute_message('power_on_instance', context, instance_id, host) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 91236dd3c..3692be922 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -531,7 +531,8 @@ class ComputeManager(manager.SchedulerDependentManager): current_power_state = self._get_power_state(context, instance) self._instance_update(context, instance_id, - power_state=current_power_state) + power_state=current_power_state, + task_state=None) @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) @checks_instance_lock @@ -542,7 +543,8 @@ class ComputeManager(manager.SchedulerDependentManager): current_power_state = self._get_power_state(context, instance) self._instance_update(context, instance_id, - power_state=current_power_state) + power_state=current_power_state, + task_state=None) @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id()) @checks_instance_lock @@ -1755,7 +1757,7 @@ class ComputeManager(manager.SchedulerDependentManager): seconds=FLAGS.reclaim_instance_interval) curtime = utils.utcnow() for instance in instances: - if instance['task_state'] == task_states.QUEUED_DELETE and \ + if instance['vm_state'] == vm_states.SOFT_DELETE and \ (curtime - instance['deleted_at']) >= queue_time: LOG.info('Deleting %s' % instance['name']) self._delete_instance(context, instance['id']) diff --git a/nova/compute/task_states.py b/nova/compute/task_states.py index 0337b2641..b52140bf8 100644 --- a/nova/compute/task_states.py +++ b/nova/compute/task_states.py @@ -50,12 +50,12 @@ PAUSING = 'pausing' UNPAUSING = 'unpausing' SUSPENDING = 'suspending' RESUMING = 'resuming' +POWERING_OFF = 'powering-off' +POWERING_ON = 'powering-on' RESCUING = 'rescuing' UNRESCUING = 'unrescuing' -QUEUED_DELETE = 'queued_delete' - DELETING = 'deleting' STOPPING = 'stopping' STARTING = 'starting' diff --git a/nova/compute/vm_states.py b/nova/compute/vm_states.py index 6f16c1f09..f219bf7f4 100644 --- a/nova/compute/vm_states.py +++ b/nova/compute/vm_states.py @@ -32,6 +32,7 @@ SUSPENDED = 'suspended' RESCUED = 'rescued' DELETED = 'deleted' STOPPED = 'stopped' +SOFT_DELETE = 'soft-delete' MIGRATING = 'migrating' RESIZING = 'resizing' -- cgit From 54d4970b9f6d6f1c52d94e5ee5d5905699bc9fc6 Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Wed, 14 Sep 2011 22:38:48 +0000 Subject: In the unlikely case of an instance losing a host, make sure we still delete the instance when a forceDelete is done --- nova/compute/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index 1a34b41a6..6048e44bc 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -851,6 +851,9 @@ class API(base.Base): if host: self._cast_compute_message('terminate_instance', context, instance_id, host) + else: + terminate_volumes(self.db, context, instance_id) + self.db.instance_destroy(context, instance_id) @scheduler_api.reroute_compute("stop") def stop(self, context, instance_id): -- cgit From 43604520de242b46d8d6bdab1fada84bac57b4dc Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Sep 2011 15:17:03 -0700 Subject: remove keystone --- etc/nova/api-paste.ini | 30 ------------------------------ nova/api/auth.py | 31 ------------------------------- nova/api/ec2/__init__.py | 48 ------------------------------------------------ 3 files changed, 109 deletions(-) diff --git a/etc/nova/api-paste.ini b/etc/nova/api-paste.ini index cd24efb13..8555f6ce5 100644 --- a/etc/nova/api-paste.ini +++ b/etc/nova/api-paste.ini @@ -22,15 +22,11 @@ use = egg:Paste#urlmap pipeline = logrequest ec2noauth cloudrequest authorizer ec2executor # NOTE(vish): use the following pipeline for deprecated auth #pipeline = logrequest authenticate cloudrequest authorizer ec2executor -# NOTE(vish): use the following pipeline for keystone -# pipeline = logrequest totoken authtoken keystonecontext cloudrequest authorizer ec2executor [pipeline:ec2admin] pipeline = logrequest ec2noauth adminrequest authorizer ec2executor # NOTE(vish): use the following pipeline for deprecated auth #pipeline = logrequest authenticate adminrequest authorizer ec2executor -# NOTE(vish): use the following pipeline for keystone -#pipeline = logrequest totoken authtoken keystonecontext adminrequest authorizer ec2executor [pipeline:ec2metadata] pipeline = logrequest ec2md @@ -44,9 +40,6 @@ paste.filter_factory = nova.api.ec2:RequestLogging.factory [filter:ec2lockout] paste.filter_factory = nova.api.ec2:Lockout.factory -[filter:totoken] -paste.filter_factory = nova.api.ec2:ToToken.factory - [filter:ec2noauth] paste.filter_factory = nova.api.ec2:NoAuth.factory @@ -87,15 +80,11 @@ use = egg:Paste#urlmap pipeline = faultwrap noauth ratelimit osapiapp10 # NOTE(vish): use the following pipeline for deprecated auth # pipeline = faultwrap auth ratelimit osapiapp10 -# NOTE(vish): use the following pipeline for keystone -#pipeline = faultwrap authtoken keystonecontext ratelimit osapiapp10 [pipeline:openstackapi11] pipeline = faultwrap noauth ratelimit extensions osapiapp11 # NOTE(vish): use the following pipeline for deprecated auth # pipeline = faultwrap auth ratelimit extensions osapiapp11 -# NOTE(vish): use the following pipeline for keystone -# pipeline = faultwrap authtoken keystonecontext ratelimit extensions osapiapp11 [filter:faultwrap] paste.filter_factory = nova.api.openstack:FaultWrapper.factory @@ -123,22 +112,3 @@ pipeline = faultwrap osversionapp [app:osversionapp] paste.app_factory = nova.api.openstack.versions:Versions.factory - -########## -# Shared # -########## - -[filter:keystonecontext] -paste.filter_factory = nova.api.auth:KeystoneContext.factory - -[filter:authtoken] -paste.filter_factory = keystone.middleware.auth_token:filter_factory -service_protocol = http -service_host = 127.0.0.1 -service_port = 808 -auth_host = 127.0.0.1 -auth_port = 5001 -auth_protocol = http -auth_uri = http://127.0.0.1:5000/ -admin_token = 999888777666 - diff --git a/nova/api/auth.py b/nova/api/auth.py index cd0d38b3f..a94f28739 100644 --- a/nova/api/auth.py +++ b/nova/api/auth.py @@ -43,34 +43,3 @@ class InjectContext(wsgi.Middleware): def __call__(self, req): req.environ['nova.context'] = self.context return self.application - - -class KeystoneContext(wsgi.Middleware): - """Make a request context from keystone headers""" - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - try: - user_id = req.headers['X_USER'] - except KeyError: - return webob.exc.HTTPUnauthorized() - # get the roles - roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')] - project_id = req.headers['X_TENANT'] - # Get the auth token - auth_token = req.headers.get('X_AUTH_TOKEN', - req.headers.get('X_STORAGE_TOKEN')) - - # Build a context, including the auth_token... - remote_address = getattr(req, 'remote_address', '127.0.0.1') - remote_address = req.remote_addr - if FLAGS.use_forwarded_for: - remote_address = req.headers.get('X-Forwarded-For', remote_address) - ctx = context.RequestContext(user_id, - project_id, - roles=roles, - auth_token=auth_token, - remote_address=remote_address) - - req.environ['nova.context'] = ctx - return self.application diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 3b217e62e..57097fdf4 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -142,54 +142,6 @@ class Lockout(wsgi.Middleware): return res -class ToToken(wsgi.Middleware): - """Authenticate an EC2 request with keystone and convert to token.""" - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - # Read request signature and access id. - try: - signature = req.params['Signature'] - access = req.params['AWSAccessKeyId'] - except KeyError: - raise webob.exc.HTTPBadRequest() - - # Make a copy of args for authentication and signature verification. - auth_params = dict(req.params) - # Not part of authentication args - auth_params.pop('Signature') - - # Authenticate the request. - creds = {'ec2Credentials': {'access': access, - 'signature': signature, - 'host': req.host, - 'verb': req.method, - 'path': req.path, - 'params': auth_params, - }} - creds_json = utils.dumps(creds) - headers = {'Content-Type': 'application/json'} - o = urlparse(FLAGS.keystone_ec2_url) - if o.scheme == "http": - conn = httplib.HTTPConnection(o.netloc) - else: - conn = httplib.HTTPSConnection(o.netloc) - conn.request('POST', o.path, body=creds_json, headers=headers) - response = conn.getresponse().read() - conn.close() - - # NOTE(vish): We could save a call to keystone by - # having keystone return token, tenant, - # user, and roles from this call. - result = utils.loads(response) - # TODO(vish): check for errors - - token_id = result['auth']['token']['id'] - # Authenticated! - req.headers['X-Auth-Token'] = token_id - return self.application - - class NoAuth(wsgi.Middleware): """Add user:project as 'nova.context' to WSGI environ.""" -- cgit From 7e7880ea2c088c330a4e27c70ef23915c5b81d4a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 20 Sep 2011 15:52:03 -0700 Subject: remove keystone url flag --- nova/api/ec2/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 57097fdf4..14bf8676a 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -46,9 +46,6 @@ flags.DEFINE_integer('lockout_minutes', 15, 'Number of minutes to lockout if triggered.') flags.DEFINE_integer('lockout_window', 15, 'Number of minutes for lockout window.') -flags.DEFINE_string('keystone_ec2_url', - 'http://localhost:5000/v2.0/ec2tokens', - 'URL to get token from ec2 request.') flags.DECLARE('use_forwarded_for', 'nova.api.auth') -- cgit From e86dc6b69f775c283a61932a456b50af15073ed7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 21 Sep 2011 02:09:37 -0700 Subject: fix moving of ips on flatdhcp bridge --- nova/network/linux_net.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ad7c5776b..cefccc69e 100755 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -472,22 +472,30 @@ def initialize_gateway_device(dev, network_ref): # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly - suffix = network_ref['cidr'].rpartition('/')[2] - out, err = _execute('ip', 'addr', 'add', - '%s/%s' % - (network_ref['dhcp_server'], suffix), - 'brd', - network_ref['broadcast'], - 'dev', - dev, - run_as_root=True, - check_exit_code=False) - if err and err != 'RTNETLINK answers: File exists\n': - raise exception.Error('Failed to add ip: %s' % err) - if FLAGS.send_arp_for_ha: - _execute('arping', '-U', network_ref['gateway'], - '-A', '-I', dev, - '-c', 1, run_as_root=True, check_exit_code=False) + full_ip = '%s/%s' % (network_ref['dhcp_server'], + network_ref['cidr'].rpartition('/')[2]) + new_ip_params = [[ full_ip, 'brd', network_ref['broadcast'] ]] + old_ip_params = [] + out, err = _execute('ip', 'addr', 'show', 'dev', dev, + 'scope', 'global', run_as_root=True) + for line in out.split('\n'): + fields = line.split() + if fields and fields[0] == 'inet': + ip_params = fields[1:-1] + old_ip_params.append(ip_params) + if ip_params[0] != full_ip: + new_ip_params.append(ip_params) + if old_ip_params[0][0] != full_ip: + for ip_params in old_ip_params: + _execute(*_ip_bridge_cmd('del', ip_params, dev), + run_as_root=True) + for ip_params in new_ip_params: + _execute(*_ip_bridge_cmd('add', ip_params, dev), + run_as_root=True) + if FLAGS.send_arp_for_ha: + _execute('arping', '-U', network_ref['dhcp_server'], + '-A', '-I', dev, + '-c', 1, run_as_root=True, check_exit_code=False) if(FLAGS.use_ipv6): _execute('ip', '-f', 'inet6', 'addr', 'change', network_ref['cidr_v6'], -- cgit From 2810a7aca4fc7d9773d77e5186f87dcf77f60ef4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 21 Sep 2011 05:34:56 -0700 Subject: add tests and fix bug when no ip was set --- nova/network/linux_net.py | 2 +- nova/tests/test_linux_net.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index cefccc69e..0d150c78f 100755 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -485,7 +485,7 @@ def initialize_gateway_device(dev, network_ref): old_ip_params.append(ip_params) if ip_params[0] != full_ip: new_ip_params.append(ip_params) - if old_ip_params[0][0] != full_ip: + if not old_ip_params or old_ip_params[0][0] != full_ip: for ip_params in old_ip_params: _execute(*_ip_bridge_cmd('del', ip_params, dev), run_as_root=True) diff --git a/nova/tests/test_linux_net.py b/nova/tests/test_linux_net.py index 99577b88e..e8f2d7c63 100755 --- a/nova/tests/test_linux_net.py +++ b/nova/tests/test_linux_net.py @@ -345,3 +345,58 @@ class LinuxNetworkTestCase(test.TestCase): expected = ("10.0.0.1,fake_instance00.novalocal,192.168.0.100") actual = self.driver._host_dhcp(fixed_ips[0]) self.assertEquals(actual, expected) + + def _test_initialize_gateway(self, existing, expected): + self.flags(fake_network=False) + executes = [] + def fake_execute(*args, **kwargs): + executes.append(args) + if args[0] == 'ip' and args[1] == 'addr' and args[2] == 'show': + return existing, "" + self.stubs.Set(utils, 'execute', fake_execute) + network = {'dhcp_server': '192.168.1.1', + 'cidr': '192.168.1.0/24', + 'broadcast': '192.168.1.255', + 'cidr_v6': '2001:db8::/64'} + self.driver.initialize_gateway_device('eth0', network) + self.assertEqual(executes, expected) + + def test_initialize_gateway_moves_wrong_ip(self): + existing = ("2: eth0: " + " mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000\n" + " link/ether de:ad:be:ef:be:ef brd ff:ff:ff:ff:ff:ff\n" + " inet 192.168.0.1/24 brd 192.168.0.255 scope global eth0\n" + " inet6 dead::beef:dead:beef:dead/64 scope link\n" + " valid_lft forever preferred_lft forever\n") + expected = [('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), + ('ip', 'addr', 'del', '192.168.0.1/24', 'brd', '192.168.0.255', 'scope', 'global', 'dev', 'eth0'), + ('ip', 'addr', 'add', '192.168.1.1/24', 'brd', '192.168.1.255', 'dev', 'eth0'), + ('ip', 'addr', 'add', '192.168.0.1/24', 'brd', '192.168.0.255', 'scope', 'global', 'dev', 'eth0'), + ('ip', '-f', 'inet6', 'addr', 'change', '2001:db8::/64', 'dev', 'eth0'), + ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on')] + self._test_initialize_gateway(existing, expected) + + def test_initialize_gateway_no_move_right_ip(self): + existing = ("2: eth0: " + " mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000\n" + " link/ether de:ad:be:ef:be:ef brd ff:ff:ff:ff:ff:ff\n" + " inet 192.168.1.1/24 brd 192.168.1.255 scope global eth0\n" + " inet 192.168.0.1/24 brd 192.168.0.255 scope global eth0\n" + " inet6 dead::beef:dead:beef:dead/64 scope link\n" + " valid_lft forever preferred_lft forever\n") + expected = [('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), + ('ip', '-f', 'inet6', 'addr', 'change', '2001:db8::/64', 'dev', 'eth0'), + ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on')] + self._test_initialize_gateway(existing, expected) + + def test_initialize_gateway_add_if_blank(self): + existing = ("2: eth0: " + " mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000\n" + " link/ether de:ad:be:ef:be:ef brd ff:ff:ff:ff:ff:ff\n" + " inet6 dead::beef:dead:beef:dead/64 scope link\n" + " valid_lft forever preferred_lft forever\n") + expected = [('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), + ('ip', 'addr', 'add', '192.168.1.1/24', 'brd', '192.168.1.255', 'dev', 'eth0'), + ('ip', '-f', 'inet6', 'addr', 'change', '2001:db8::/64', 'dev', 'eth0'), + ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on')] + self._test_initialize_gateway(existing, expected) -- cgit From 5ed0157ba4c8218680906e63277dfdf45e5719c3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 21 Sep 2011 05:56:48 -0700 Subject: pep8 --- nova/network/linux_net.py | 2 +- nova/tests/test_linux_net.py | 40 +++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0d150c78f..0459b4aeb 100755 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -474,7 +474,7 @@ def initialize_gateway_device(dev, network_ref): # bridge for it to respond to reqests properly full_ip = '%s/%s' % (network_ref['dhcp_server'], network_ref['cidr'].rpartition('/')[2]) - new_ip_params = [[ full_ip, 'brd', network_ref['broadcast'] ]] + new_ip_params = [[full_ip, 'brd', network_ref['broadcast']]] old_ip_params = [] out, err = _execute('ip', 'addr', 'show', 'dev', dev, 'scope', 'global', run_as_root=True) diff --git a/nova/tests/test_linux_net.py b/nova/tests/test_linux_net.py index e8f2d7c63..940af7b5f 100755 --- a/nova/tests/test_linux_net.py +++ b/nova/tests/test_linux_net.py @@ -349,6 +349,7 @@ class LinuxNetworkTestCase(test.TestCase): def _test_initialize_gateway(self, existing, expected): self.flags(fake_network=False) executes = [] + def fake_execute(*args, **kwargs): executes.append(args) if args[0] == 'ip' and args[1] == 'addr' and args[2] == 'show': @@ -368,12 +369,18 @@ class LinuxNetworkTestCase(test.TestCase): " inet 192.168.0.1/24 brd 192.168.0.255 scope global eth0\n" " inet6 dead::beef:dead:beef:dead/64 scope link\n" " valid_lft forever preferred_lft forever\n") - expected = [('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), - ('ip', 'addr', 'del', '192.168.0.1/24', 'brd', '192.168.0.255', 'scope', 'global', 'dev', 'eth0'), - ('ip', 'addr', 'add', '192.168.1.1/24', 'brd', '192.168.1.255', 'dev', 'eth0'), - ('ip', 'addr', 'add', '192.168.0.1/24', 'brd', '192.168.0.255', 'scope', 'global', 'dev', 'eth0'), - ('ip', '-f', 'inet6', 'addr', 'change', '2001:db8::/64', 'dev', 'eth0'), - ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on')] + expected = [ + ('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), + ('ip', 'addr', 'del', '192.168.0.1/24', + 'brd', '192.168.0.255', 'scope', 'global', 'dev', 'eth0'), + ('ip', 'addr', 'add', '192.168.1.1/24', + 'brd', '192.168.1.255', 'dev', 'eth0'), + ('ip', 'addr', 'add', '192.168.0.1/24', + 'brd', '192.168.0.255', 'scope', 'global', 'dev', 'eth0'), + ('ip', '-f', 'inet6', 'addr', 'change', + '2001:db8::/64', 'dev', 'eth0'), + ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on'), + ] self._test_initialize_gateway(existing, expected) def test_initialize_gateway_no_move_right_ip(self): @@ -384,9 +391,12 @@ class LinuxNetworkTestCase(test.TestCase): " inet 192.168.0.1/24 brd 192.168.0.255 scope global eth0\n" " inet6 dead::beef:dead:beef:dead/64 scope link\n" " valid_lft forever preferred_lft forever\n") - expected = [('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), - ('ip', '-f', 'inet6', 'addr', 'change', '2001:db8::/64', 'dev', 'eth0'), - ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on')] + expected = [ + ('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), + ('ip', '-f', 'inet6', 'addr', 'change', + '2001:db8::/64', 'dev', 'eth0'), + ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on'), + ] self._test_initialize_gateway(existing, expected) def test_initialize_gateway_add_if_blank(self): @@ -395,8 +405,12 @@ class LinuxNetworkTestCase(test.TestCase): " link/ether de:ad:be:ef:be:ef brd ff:ff:ff:ff:ff:ff\n" " inet6 dead::beef:dead:beef:dead/64 scope link\n" " valid_lft forever preferred_lft forever\n") - expected = [('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), - ('ip', 'addr', 'add', '192.168.1.1/24', 'brd', '192.168.1.255', 'dev', 'eth0'), - ('ip', '-f', 'inet6', 'addr', 'change', '2001:db8::/64', 'dev', 'eth0'), - ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on')] + expected = [ + ('ip', 'addr', 'show', 'dev', 'eth0', 'scope', 'global'), + ('ip', 'addr', 'add', '192.168.1.1/24', + 'brd', '192.168.1.255', 'dev', 'eth0'), + ('ip', '-f', 'inet6', 'addr', 'change', + '2001:db8::/64', 'dev', 'eth0'), + ('ip', 'link', 'set', 'dev', 'eth0', 'promisc', 'on'), + ] self._test_initialize_gateway(existing, expected) -- cgit