Fix poll_rescued_instances periodic task

The poll_rescued_instances periodic task now checks the amount of time
that an instance has been in the RESCUED stated before timing out the
rescue.  It also now performs the unrescue through the compute api in
order to make sure the database is left in a consistent state.

The poll_rescued_instances method is no longer necessary in the virt
driver interface and has been removed.  And also removed from the
different virt drivers, since it was just doing a 'pass' in each of
them.

bug 1088625
bug 1088627

Change-Id: I75f7dc188cc49e5f6e5c8a3cb256d1c42ff7d882

2 files changed, 34 insertions, 4 deletions

diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py
index ccc9614c5..079a25d27 100644
--- a/nova/tests/compute/test_compute.py
+++ b/nova/tests/compute/test_compute.py
@@ -2788,6 +2788,40 @@ class ComputeTestCase(BaseTestCase):
         self.assertEqual(call_info['get_by_uuid'], 3)
         self.assertEqual(call_info['get_nw_info'], 4)
 
+    def test_poll_rescued_instances(self):
+        timed_out_time = timeutils.utcnow() - datetime.timedelta(minutes=5)
+        not_timed_out_time = timeutils.utcnow()
+
+        instances = [{'uuid': 'fake_uuid1', 'vm_state': vm_states.RESCUED,
+                      'launched_at': timed_out_time},
+                     {'uuid': 'fake_uuid2', 'vm_state': vm_states.ACTIVE,
+                      'launched_at': timed_out_time},
+                     {'uuid': 'fake_uuid3', 'vm_state': vm_states.ACTIVE,
+                      'launched_at': not_timed_out_time},
+                     {'uuid': 'fake_uuid4', 'vm_state': vm_states.RESCUED,
+                      'launched_at': timed_out_time},
+                     {'uuid': 'fake_uuid5', 'vm_state': vm_states.RESCUED,
+                      'launched_at': not_timed_out_time}]
+        unrescued_instances = {'fake_uuid1': False, 'fake_uuid4': False}
+
+        def fake_instance_get_all_by_host(context, host):
+            return instances
+
+        def fake_unrescue(self, context, instance):
+            unrescued_instances[instance['uuid']] = True
+
+        self.stubs.Set(self.compute.conductor_api, 'instance_get_all_by_host',
+                       fake_instance_get_all_by_host)
+        self.stubs.Set(compute_api.API, 'unrescue', fake_unrescue)
+
+        self.flags(rescue_timeout=60)
+        ctxt = context.get_admin_context()
+
+        self.compute._poll_rescued_instances(ctxt)
+
+        for instance in unrescued_instances.values():
+            self.assertTrue(instance)
+
     def test_poll_unconfirmed_resizes(self):
         instances = [{'uuid': 'fake_uuid1', 'vm_state': vm_states.RESIZED,
                       'task_state': None},
diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py
index 834763540..563b3a44b 100644
--- a/nova/tests/test_virt_drivers.py
+++ b/nova/tests/test_virt_drivers.py
@@ -283,10 +283,6 @@ class _VirtDriverTestCase(_FakeDriverBackendTestCase):
         self.connection.poll_rebooting_instances(10, instances)
 
     @catch_notimplementederror
-    def test_poll_rescued_instances(self):
-        self.connection.poll_rescued_instances(10)
-
-    @catch_notimplementederror
     def test_migrate_disk_and_power_off(self):
         instance_ref, network_info = self._get_running_instance()
         instance_type_ref = test_utils.get_test_instance_type()