Merge "Adds retry on upload_vhd for xapi glance plugin"

2 files changed, 47 insertions, 14 deletions

diff --git a/nova/exception.py b/nova/exception.py
index 6d931588d..93dd2d205 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -1089,6 +1089,10 @@ class CouldNotFetchImage(NovaException):
     message = _("Could not fetch image %(image_id)s")
 
 
+class CouldNotUploadImage(NovaException):
+    message = _("Could not upload image %(image_id)s")
+
+
 class TaskAlreadyRunning(NovaException):
     message = _("Task %(task_name)s is already running on host %(host)s")
 
diff --git a/nova/virt/xenapi/imageupload/glance.py b/nova/virt/xenapi/imageupload/glance.py
index 664d242c0..d306e06b0 100644
--- a/nova/virt/xenapi/imageupload/glance.py
+++ b/nova/virt/xenapi/imageupload/glance.py
@@ -13,8 +13,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import time
+
 from oslo.config import cfg
 
+from nova import exception
 from nova.image import glance
 import nova.openstack.common.log as logging
 from nova.virt.xenapi import vm_utils
@@ -22,6 +25,7 @@ from nova.virt.xenapi import vm_utils
 LOG = logging.getLogger(__name__)
 
 CONF = cfg.CONF
+CONF.import_opt('glance_num_retries', 'nova.image.glance')
 
 
 class GlanceStore(object):
@@ -32,24 +36,49 @@ class GlanceStore(object):
         """
         # NOTE(sirp): Currently we only support uploading images as VHD, there
         # is no RAW equivalent (yet)
-        LOG.debug(_("Asking xapi to upload to glance %(vdi_uuids)s as"
-                    " ID %(image_id)s"), locals(), instance=instance)
-
+        max_attempts = CONF.glance_num_retries + 1
+        sleep_time = 0.5
         glance_api_servers = glance.get_api_servers()
-        glance_host, glance_port, glance_use_ssl = glance_api_servers.next()
-
         properties = {
             'auto_disk_config': instance['auto_disk_config'],
             'os_type': instance['os_type'] or CONF.default_os_type,
         }
 
-        params = {'vdi_uuids': vdi_uuids,
-                  'image_id': image_id,
-                  'glance_host': glance_host,
-                  'glance_port': glance_port,
-                  'glance_use_ssl': glance_use_ssl,
-                  'sr_path': vm_utils.get_sr_path(session),
-                  'auth_token': getattr(context, 'auth_token', None),
-                  'properties': properties}
+        for attempt_num in xrange(1, max_attempts + 1):
+
+            (glance_host,
+             glance_port,
+             glance_use_ssl) = glance_api_servers.next()
+
+            try:
+
+                params = {'vdi_uuids': vdi_uuids,
+                          'image_id': image_id,
+                          'glance_host': glance_host,
+                          'glance_port': glance_port,
+                          'glance_use_ssl': glance_use_ssl,
+                          'sr_path': vm_utils.get_sr_path(session),
+                          'auth_token': getattr(context, 'auth_token', None),
+                          'properties': properties}
+
+                LOG.debug(_("Asking xapi to upload to glance %(vdi_uuids)s as"
+                            " ID %(image_id)s"
+                            " glance server: %(glance_host)s:%(glance_port)d"
+                            " attempt %(attempt_num)d/%(max_attempts)d"),
+                            locals(), instance=instance)
+
+                return session.call_plugin_serialized('glance',
+                                                      'upload_vhd',
+                                                      **params)
+
+            except session.XenAPI.Failure as exc:
+                _type, _method, error = exc.details[:3]
+                if error == 'RetryableError':
+                    LOG.error(_('upload_vhd failed: %r') %
+                              (exc.details[3:],))
+                else:
+                    raise
+            time.sleep(sleep_time)
+            sleep_time = min(2 * sleep_time, 15)
 
-        session.call_plugin_serialized('glance', 'upload_vhd', **params)
+        raise exception.CouldNotUploadImage(image_id=image_id)