From 2db1a0f62b797acdb8506e7e36fbced0828dd796 Mon Sep 17 00:00:00 2001 From: Nicholas Kuechler Date: Wed, 27 Feb 2013 10:02:33 -0600 Subject: Adds retry on upload_vhd for xapi glance plugin Add a retry for the xapi glance plugin to handle transient issues when uploading the vhd. An example issue we're seeing is a connection timeout: ['XENAPI_PLUGIN_FAILURE', 'upload_vhd', 'error', "(110, 'Connection timed out')"] To work around transient issues such as a connection timeout, we should retry based on glance_num_retries before outright failing. Change-Id: Ice6fdd3dd39ef40e5997d69209aaafa66bff5d6e Fixes: bug #1134493 --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 32 +++++++++++++++------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 924bf10d7..47b12bd60 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -126,14 +126,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, url = '%s://%s:%s/v1/images/%s' % (scheme, glance_host, glance_port, image_id) logging.info("Writing image data to %s" % url) - if glance_use_ssl: - conn = httplib.HTTPSConnection(glance_host, glance_port) - else: - conn = httplib.HTTPConnection(glance_host, glance_port) - # NOTE(sirp): httplib under python2.4 won't accept a file-like object - # to request - conn.putrequest('PUT', '/v1/images/%s' % image_id) + try: + if glance_use_ssl: + conn = httplib.HTTPSConnection(glance_host, glance_port) + else: + conn = httplib.HTTPConnection(glance_host, glance_port) + + # NOTE(sirp): httplib under python2.4 won't accept a file-like object + # to request + conn.putrequest('PUT', '/v1/images/%s' % image_id) + except Exception, error: + raise RetryableError(error) # NOTE(sirp): There is some confusion around OVF. Here's a summary of # where we currently stand: @@ -172,12 +176,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, def send_chunked_transfer_encoded(chunk): chunk_len = len(chunk) callback_data['bytes_written'] += chunk_len - conn.send("%x\r\n%s\r\n" % (chunk_len, chunk)) + try: + conn.send("%x\r\n%s\r\n" % (chunk_len, chunk)) + except Exception, error: + raise RetryableError(error) utils.create_tarball( None, staging_path, callback=send_chunked_transfer_encoded) - conn.send("0\r\n\r\n") # Chunked-Transfer terminator + try: + conn.send("0\r\n\r\n") # Chunked-Transfer terminator + except Exception, error: + raise RetryableError(error) bytes_written = callback_data['bytes_written'] logging.info("Wrote %d bytes to %s" % (bytes_written, url)) @@ -187,9 +197,11 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, logging.error("Unexpected response while writing image data to %s: " "Response Status: %i, Response body: %s" % (url, resp.status, resp.read())) - raise Exception("Unexpected response [%i] while uploading image [%s] " + raise RetryableError("Unexpected response [%i] while uploading " + "image [%s] " "to glance host [%s:%s]" % (resp.status, image_id, glance_host, glance_port)) + conn.close() -- cgit