Merge "Optional timeout for servers stuck in build"

1 files changed, 21 insertions, 0 deletions

diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 9cb080dbe..8025a46a5 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -96,6 +96,11 @@ compute_opts = [
                help="Automatically hard reboot an instance if it has been "
                     "stuck in a rebooting state longer than N seconds. "
                     "Set to 0 to disable."),
+    cfg.IntOpt("instance_build_timeout",
+               default=0,
+               help="Amount of time in seconds an instance can be in BUILD "
+                    "before going into ERROR status."
+                    "Set to 0 to disable."),
     cfg.IntOpt("rescue_timeout",
                default=0,
                help="Automatically unrescue an instance after N seconds. "
@@ -450,6 +455,22 @@ class ComputeManager(manager.SchedulerDependentManager):
             with excutils.save_and_reraise_exception():
                 self._set_instance_error_state(context, instance_uuid)
 
+    @manager.periodic_task
+    def _check_instance_build_time(self, context):
+        """Ensure that instances are not stuck in build."""
+        if FLAGS.instance_build_timeout == 0:
+            return
+
+        filters = {'vm_state': vm_states.BUILDING}
+        building_insts = self.db.instance_get_all_by_filters(context, filters)
+
+        for instance in building_insts:
+            if utils.is_older_than(instance['created_at'],
+                                   FLAGS.instance_build_timeout):
+                self._set_instance_error_state(context, instance['uuid'])
+                LOG.warn(_("Instance build timed out. Set to error state."),
+                         instance=instance)
+
     def _update_access_ip(self, context, instance, nw_info):
         """Update the access ip values for a given instance.