Instance deletions in Openstack are immediate. This can cause data to be lost accidentally.

This branch adds a new configuration flag, reclaim_instance_interval. The default of 0 results in the same behavior before this patch, immediate deletion of the instance. Any value greater than 0 will result in the instance being powered off immediately and then later the instance will be reclaimed.

New actions, restore and forceDelete allow a previously deleted instance to be restored, or reclaimed immediately.

3 files changed, 137 insertions, 18 deletions

diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py
index 44f4eb055..ca36523e4 100644
--- a/nova/tests/api/openstack/test_extensions.py
+++ b/nova/tests/api/openstack/test_extensions.py
@@ -86,6 +86,7 @@ class ExtensionControllerTest(test.TestCase):
         self.flags(osapi_extensions_path=ext_path)
         self.ext_list = [
             "Createserverext",
+            "DeferredDelete",
             "FlavorExtraSpecs",
             "FlavorExtraData",
             "Floating_ips",
diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py
index 49de9c854..b53e4cec6 100644
--- a/nova/tests/integrated/integrated_helpers.py
+++ b/nova/tests/integrated/integrated_helpers.py
@@ -70,10 +70,10 @@ class _IntegratedTestBase(test.TestCase):
         self.stubs.Set(nova.image, 'get_image_service', fake_get_image_service)
 
         # set up services
-        self.start_service('compute')
-        self.start_service('volume')
-        self.start_service('network')
-        self.start_service('scheduler')
+        self.compute = self.start_service('compute')
+        self.volume = self.start_service('volume')
+        self.network = self.start_service('network')
+        self.scheduler = self.start_service('scheduler')
 
         self._start_api_service()
 
diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py
index 2cf604d06..e9c79aa13 100644
--- a/nova/tests/integrated/test_servers.py
+++ b/nova/tests/integrated/test_servers.py
@@ -28,17 +28,25 @@ LOG = logging.getLogger('nova.tests.integrated')
 
 class ServersTest(integrated_helpers._IntegratedTestBase):
 
-    def _wait_for_creation(self, server):
-        retries = 0
-        while server['status'] == 'BUILD':
-            time.sleep(1)
+    def _wait_for_state_change(self, server, status):
+        for i in xrange(0, 50):
             server = self.api.get_server(server['id'])
             print server
-            retries = retries + 1
-            if retries > 5:
+            if server['status'] != status:
                 break
+            time.sleep(.1)
+
         return server
 
+    def _restart_compute_service(self, periodic_interval=None):
+        """restart compute service. NOTE: fake driver forgets all instances."""
+        self.compute.kill()
+        if periodic_interval:
+            self.compute = self.start_service(
+                'compute', periodic_interval=periodic_interval)
+        else:
+            self.compute = self.start_service('compute')
+
     def test_get_servers(self):
         """Simple check that listing servers works."""
         servers = self.api.get_servers()
@@ -102,7 +110,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase):
         server_ids = [server['id'] for server in servers]
         self.assertTrue(created_server_id in server_ids)
 
-        found_server = self._wait_for_creation(found_server)
+        found_server = self._wait_for_state_change(found_server, 'BUILD')
 
         # It should be available...
         # TODO(justinsb): Mock doesn't yet do this...
@@ -114,12 +122,117 @@ class ServersTest(integrated_helpers._IntegratedTestBase):
 
         self._delete_server(created_server_id)
 
-    def _delete_server(self, server_id):
+    def test_deferred_delete(self):
+        """Creates, deletes and waits for server to be reclaimed."""
+        self.flags(stub_network=True, reclaim_instance_interval=1)
+
+        # enforce periodic tasks run in short time to avoid wait for 60s.
+        self._restart_compute_service(periodic_interval=0.3)
+
+        # Create server
+        server = self._build_minimal_create_server_request()
+
+        created_server = self.api.post_server({'server': server})
+        LOG.debug("created_server: %s" % created_server)
+        self.assertTrue(created_server['id'])
+        created_server_id = created_server['id']
+
+        # Wait for it to finish being created
+        found_server = self._wait_for_state_change(created_server, 'BUILD')
+
+        # It should be available...
+        self.assertEqual('ACTIVE', found_server['status'])
+
+        # Cannot restore unless instance is deleted
+        self.api.post_server_action(created_server_id, {'restore': {}})
+
+        # Check it's still active
+        found_server = self.api.get_server(created_server_id)
+        self.assertEqual('ACTIVE', found_server['status'])
+
+        # Cannot forceDelete unless instance is deleted
+        self.api.post_server_action(created_server_id, {'forceDelete': {}})
+
+        # Check it's still active
+        found_server = self.api.get_server(created_server_id)
+        self.assertEqual('ACTIVE', found_server['status'])
+
         # Delete the server
-        self.api.delete_server(server_id)
+        self.api.delete_server(created_server_id)
+
+        # Wait for queued deletion
+        found_server = self._wait_for_state_change(found_server, 'ACTIVE')
+        self.assertEqual('DELETED', found_server['status'])
 
+        # Wait for real deletion
+        self._wait_for_deletion(created_server_id)
+
+    def test_deferred_delete_restore(self):
+        """Creates, deletes and restores a server."""
+        self.flags(stub_network=True, reclaim_instance_interval=1)
+
+        # Create server
+        server = self._build_minimal_create_server_request()
+
+        created_server = self.api.post_server({'server': server})
+        LOG.debug("created_server: %s" % created_server)
+        self.assertTrue(created_server['id'])
+        created_server_id = created_server['id']
+
+        # Wait for it to finish being created
+        found_server = self._wait_for_state_change(created_server, 'BUILD')
+
+        # It should be available...
+        self.assertEqual('ACTIVE', found_server['status'])
+
+        # Delete the server
+        self.api.delete_server(created_server_id)
+
+        # Wait for queued deletion
+        found_server = self._wait_for_state_change(found_server, 'ACTIVE')
+        self.assertEqual('DELETED', found_server['status'])
+
+        # Restore server
+        self.api.post_server_action(created_server_id, {'restore': {}})
+
+        # Wait for server to become active again
+        found_server = self._wait_for_state_change(found_server, 'DELETED')
+        self.assertEqual('ACTIVE', found_server['status'])
+
+    def test_deferred_delete_force(self):
+        """Creates, deletes and force deletes a server."""
+        self.flags(stub_network=True, reclaim_instance_interval=1)
+
+        # Create server
+        server = self._build_minimal_create_server_request()
+
+        created_server = self.api.post_server({'server': server})
+        LOG.debug("created_server: %s" % created_server)
+        self.assertTrue(created_server['id'])
+        created_server_id = created_server['id']
+
+        # Wait for it to finish being created
+        found_server = self._wait_for_state_change(created_server, 'BUILD')
+
+        # It should be available...
+        self.assertEqual('ACTIVE', found_server['status'])
+
+        # Delete the server
+        self.api.delete_server(created_server_id)
+
+        # Wait for queued deletion
+        found_server = self._wait_for_state_change(found_server, 'ACTIVE')
+        self.assertEqual('DELETED', found_server['status'])
+
+        # Force delete server
+        self.api.post_server_action(created_server_id, {'forceDelete': {}})
+
+        # Wait for real deletion
+        self._wait_for_deletion(created_server_id)
+
+    def _wait_for_deletion(self, server_id):
         # Wait (briefly) for deletion
-        for _retries in range(5):
+        for _retries in range(50):
             try:
                 found_server = self.api.get_server(server_id)
             except client.OpenStackApiNotFoundException:
@@ -132,11 +245,16 @@ class ServersTest(integrated_helpers._IntegratedTestBase):
             # TODO(justinsb): Mock doesn't yet do accurate state changes
             #if found_server['status'] != 'deleting':
             #    break
-            time.sleep(1)
+            time.sleep(.1)
 
         # Should be gone
         self.assertFalse(found_server)
 
+    def _delete_server(self, server_id):
+        # Delete the server
+        self.api.delete_server(server_id)
+        self._wait_for_deletion(server_id)
+
     def test_create_server_with_metadata(self):
         """Creates a server with metadata."""
 
@@ -194,7 +312,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase):
         self.assertTrue(created_server['id'])
         created_server_id = created_server['id']
 
-        created_server = self._wait_for_creation(created_server)
+        created_server = self._wait_for_state_change(created_server, 'BUILD')
 
         # rebuild the server with metadata
         post = {}
@@ -228,7 +346,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase):
         self.assertTrue(created_server['id'])
         created_server_id = created_server['id']
 
-        created_server = self._wait_for_creation(created_server)
+        created_server = self._wait_for_state_change(created_server, 'BUILD')
 
         # rebuild the server with metadata
         post = {}
@@ -274,7 +392,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase):
         self.assertTrue(created_server['id'])
         created_server_id = created_server['id']
 
-        created_server = self._wait_for_creation(created_server)
+        created_server = self._wait_for_state_change(created_server, 'BUILD')
 
         # rebuild the server with metadata
         post = {}