From 7b75fe7f571dd95287307f9d1138fb476a6bf721 Mon Sep 17 00:00:00 2001 From: Philip Knouff Date: Thu, 3 May 2012 14:55:03 -0400 Subject: Optional timeout for servers stuck in build Fixes bug 994786 Change-Id: Iae86c002073f45b48acde8eae07f9b0f62488f90 --- nova/compute/manager.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'nova/compute') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index e35ede5e4..3d42a36f0 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -96,6 +96,11 @@ compute_opts = [ help="Automatically hard reboot an instance if it has been " "stuck in a rebooting state longer than N seconds. " "Set to 0 to disable."), + cfg.IntOpt("instance_build_timeout", + default=0, + help="Amount of time in seconds an instance can be in BUILD " + "before going into ERROR status." + "Set to 0 to disable."), cfg.IntOpt("rescue_timeout", default=0, help="Automatically unrescue an instance after N seconds. " @@ -450,6 +455,22 @@ class ComputeManager(manager.SchedulerDependentManager): with excutils.save_and_reraise_exception(): self._set_instance_error_state(context, instance_uuid) + @manager.periodic_task + def _check_instance_build_time(self, context): + """Ensure that instances are not stuck in build.""" + if FLAGS.instance_build_timeout == 0: + return + + filters = {'vm_state': vm_states.BUILDING} + building_insts = self.db.instance_get_all_by_filters(context, filters) + + for instance in building_insts: + if utils.is_older_than(instance['created_at'], + FLAGS.instance_build_timeout): + self._set_instance_error_state(context, instance['uuid']) + LOG.warn(_("Instance build timed out. Set to error state."), + instance=instance) + def _update_access_ip(self, context, instance, nw_info): """Update the access ip values for a given instance. -- cgit