From 6fda12bb14ab91592910f9ebbde9ef9f8d588b08 Mon Sep 17 00:00:00 2001 From: Aaron Rosen Date: Fri, 31 May 2013 16:17:58 -0700 Subject: Fix nova-compute fails to start if quantum is down When nova-compute starts it needs to check the state of instances that were previously running and needs to query quantum. Previously, if it queried quantum and quantum raised an error or was unreachable nova-compute would terminate. This patch adds a retry loop to prevent this from happening. Fixes bug 1186357 Change-Id: I0d865e464f7bf60a0b6dc4ca8a42436d16e29790 --- nova/compute/manager.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index f9a1cc94d..14cde78e9 100755 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -516,7 +516,23 @@ class ComputeManager(manager.SchedulerDependentManager): legacy_net_info[0][1].get('vif_type') is None): # Call to network API to get instance info, this will # force an update to the instance's info_cache - net_info = self._get_instance_nw_info(context, instance) + retry_time = 0 + # Continue retrying until _get_instance_nw_info() succeeds. + while True: + try: + net_info = self._get_instance_nw_info(context, instance) + break + except Exception: + # Retry in an exponential backoff fashion + # capped at 60 seconds. + if retry_time < 60: + retry_time += 6 + LOG.exception(_("Error raised getting network info for " + "instance %(instance_uuid)s. Retrying " + "in %(retry_time)s seconds."), + {'instance_uuid': instance['uuid'], + 'retry_time': retry_time}) + time.sleep(retry_time) legacy_net_info = self._legacy_nw_info(net_info) self.driver.plug_vifs(instance, legacy_net_info) -- cgit