Look up stuck-in-rebooting instances in manager

The poll_rebooting_instances() method for a virt driver can be
called periodically from the manager to handle instances that
may be stuck in a rebooting state. The only information passed
to the virt driver is the configured timeout, which means any
driver implementing this will have to do the same database
query: db.instance_get_all_hung_un_rebooting (..., timeout).

This patch makes the manager do this before calling the virt
driver, passing the appropriate set of instances, which avoids
the need for the driver to make that call.

Related to bp/no-db-compute

Change-Id: I152cb8cd9a107f2d2b1df39401ab7bbe7ff366f1

7 files changed, 19 insertions, 10 deletions

diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 39c3faddf..74304a038 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -2667,7 +2667,10 @@ class ComputeManager(manager.SchedulerDependentManager):
     @manager.periodic_task
     def _poll_rebooting_instances(self, context):
         if FLAGS.reboot_timeout > 0:
-            self.driver.poll_rebooting_instances(FLAGS.reboot_timeout)
+            instances = self.db.instance_get_all_hung_in_rebooting(
+                context, FLAGS.reboot_timeout)
+            self.driver.poll_rebooting_instances(FLAGS.reboot_timeout,
+                                                 instances)
 
     @manager.periodic_task
     def _poll_rescued_instances(self, context):
diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py
index 1f30ee695..9d48cdf06 100644
--- a/nova/tests/test_virt_drivers.py
+++ b/nova/tests/test_virt_drivers.py
@@ -269,7 +269,8 @@ class _VirtDriverTestCase(_FakeDriverBackendTestCase):
 
     @catch_notimplementederror
     def test_poll_rebooting_instances(self):
-        self.connection.poll_rebooting_instances(10)
+        instances = [self._get_running_instance()]
+        self.connection.poll_rebooting_instances(10, instances)
 
     @catch_notimplementederror
     def test_poll_rescued_instances(self):
diff --git a/nova/virt/driver.py b/nova/virt/driver.py
index 9c8a6448d..a466fa180 100644
--- a/nova/virt/driver.py
+++ b/nova/virt/driver.py
@@ -604,8 +604,14 @@ class ComputeDriver(object):
         # TODO(Vek): Need to pass context in for access to auth_token
         pass
 
-    def poll_rebooting_instances(self, timeout):
-        """Poll for rebooting instances"""
+    def poll_rebooting_instances(self, timeout, instances):
+        """Poll for rebooting instances
+
+        :param timeout: the currently configured timeout for considering
+                        rebooting instances to be stuck
+        :param instances: instances that have been in rebooting state
+                          longer than the configured timeout
+        """
         # TODO(Vek): Need to pass context in for access to auth_token
         raise NotImplementedError()
 
diff --git a/nova/virt/fake.py b/nova/virt/fake.py
index 03711fe98..877fb7603 100644
--- a/nova/virt/fake.py
+++ b/nova/virt/fake.py
@@ -123,7 +123,7 @@ class FakeDriver(driver.ComputeDriver):
     def unrescue(self, instance, network_info):
         pass
 
-    def poll_rebooting_instances(self, timeout):
+    def poll_rebooting_instances(self, timeout, instances):
         pass
 
     def poll_rescued_instances(self, timeout):
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 97ce1710c..3104fafd3 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -1052,7 +1052,7 @@ class LibvirtDriver(driver.ComputeDriver):
             libvirt_utils.file_delete(rescue_file)
 
     @exception.wrap_exception()
-    def poll_rebooting_instances(self, timeout):
+    def poll_rebooting_instances(self, timeout, instances):
         pass
 
     @exception.wrap_exception()
diff --git a/nova/virt/xenapi/driver.py b/nova/virt/xenapi/driver.py
index e4c4150a8..a928bf861 100644
--- a/nova/virt/xenapi/driver.py
+++ b/nova/virt/xenapi/driver.py
@@ -290,9 +290,9 @@ class XenAPIDriver(driver.ComputeDriver):
         """Restore the specified instance"""
         self._vmops.restore(instance)
 
-    def poll_rebooting_instances(self, timeout):
+    def poll_rebooting_instances(self, timeout, instances):
         """Poll for rebooting instances"""
-        self._vmops.poll_rebooting_instances(timeout)
+        self._vmops.poll_rebooting_instances(timeout, instances)
 
     def poll_rescued_instances(self, timeout):
         """Poll for rescued instances"""
diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py
index 5a295d194..8bcefd876 100644
--- a/nova/virt/xenapi/vmops.py
+++ b/nova/virt/xenapi/vmops.py
@@ -1172,7 +1172,7 @@ class VMOps(object):
             if timeutils.is_older_than(task_created, timeout):
                 self._session.call_xenapi("task.cancel", task_ref)
 
-    def poll_rebooting_instances(self, timeout):
+    def poll_rebooting_instances(self, timeout, instances):
         """Look for expirable rebooting instances.
 
             - issue a "hard" reboot to any instance that has been stuck in a
@@ -1183,7 +1183,6 @@ class VMOps(object):
         self._cancel_stale_tasks(timeout, 'VM.clean_reboot')
 
         ctxt = nova_context.get_admin_context()
-        instances = db.instance_get_all_hung_in_rebooting(ctxt, timeout)
 
         instances_info = dict(instance_count=len(instances),
                 timeout=timeout)