Adds retry on upload_vhd for xapi glance plugin

Add a retry for the xapi glance plugin to handle transient issues
when uploading the vhd.

An example issue we're seeing is a connection timeout:

['XENAPI_PLUGIN_FAILURE', 'upload_vhd', 'error',
		"(110, 'Connection timed out')"]

To work around transient issues such as a connection timeout, we
should retry based on glance_num_retries before outright failing.

Change-Id: Ice6fdd3dd39ef40e5997d69209aaafa66bff5d6e
Fixes: bug #1134493

1 files changed, 22 insertions, 10 deletions

diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
index 924bf10d7..47b12bd60 100755
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
@@ -126,14 +126,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
     url = '%s://%s:%s/v1/images/%s' % (scheme, glance_host, glance_port,
                                        image_id)
     logging.info("Writing image data to %s" % url)
-    if glance_use_ssl:
-        conn = httplib.HTTPSConnection(glance_host, glance_port)
-    else:
-        conn = httplib.HTTPConnection(glance_host, glance_port)
 
-    # NOTE(sirp): httplib under python2.4 won't accept a file-like object
-    # to request
-    conn.putrequest('PUT', '/v1/images/%s' % image_id)
+    try:
+        if glance_use_ssl:
+            conn = httplib.HTTPSConnection(glance_host, glance_port)
+        else:
+            conn = httplib.HTTPConnection(glance_host, glance_port)
+
+        # NOTE(sirp): httplib under python2.4 won't accept a file-like object
+        # to request
+        conn.putrequest('PUT', '/v1/images/%s' % image_id)
+    except Exception, error:
+        raise RetryableError(error)
 
     # NOTE(sirp): There is some confusion around OVF. Here's a summary of
     # where we currently stand:
@@ -172,12 +176,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
     def send_chunked_transfer_encoded(chunk):
         chunk_len = len(chunk)
         callback_data['bytes_written'] += chunk_len
-        conn.send("%x\r\n%s\r\n" % (chunk_len, chunk))
+        try:
+            conn.send("%x\r\n%s\r\n" % (chunk_len, chunk))
+        except Exception, error:
+            raise RetryableError(error)
 
     utils.create_tarball(
             None, staging_path, callback=send_chunked_transfer_encoded)
 
-    conn.send("0\r\n\r\n")  # Chunked-Transfer terminator
+    try:
+        conn.send("0\r\n\r\n")  # Chunked-Transfer terminator
+    except Exception, error:
+        raise RetryableError(error)
 
     bytes_written = callback_data['bytes_written']
     logging.info("Wrote %d bytes to %s" % (bytes_written, url))
@@ -187,9 +197,11 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port,
         logging.error("Unexpected response while writing image data to %s: "
                       "Response Status: %i, Response body: %s"
                       % (url, resp.status, resp.read()))
-        raise Exception("Unexpected response [%i] while uploading image [%s] "
+        raise RetryableError("Unexpected response [%i] while uploading "
+                        "image [%s] "
                         "to glance host [%s:%s]"
                         % (resp.status, image_id, glance_host, glance_port))
+
     conn.close()