diff options
| -rw-r--r-- | nova/exception.py | 4 | ||||
| -rw-r--r-- | nova/virt/xenapi/imageupload/glance.py | 57 | ||||
| -rwxr-xr-x | plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 32 |
3 files changed, 69 insertions, 24 deletions
diff --git a/nova/exception.py b/nova/exception.py index cad1c5c02..d1037aabc 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -1080,6 +1080,10 @@ class CouldNotFetchImage(NovaException): message = _("Could not fetch image %(image_id)s") +class CouldNotUploadImage(NovaException): + message = _("Could not upload image %(image_id)s") + + class TaskAlreadyRunning(NovaException): message = _("Task %(task_name)s is already running on host %(host)s") diff --git a/nova/virt/xenapi/imageupload/glance.py b/nova/virt/xenapi/imageupload/glance.py index 5ea75db16..1a4f7da59 100644 --- a/nova/virt/xenapi/imageupload/glance.py +++ b/nova/virt/xenapi/imageupload/glance.py @@ -13,8 +13,11 @@ # License for the specific language governing permissions and limitations # under the License. +import time + from oslo.config import cfg +from nova import exception from nova.image import glance import nova.openstack.common.log as logging from nova.virt.xenapi import vm_utils @@ -22,6 +25,7 @@ from nova.virt.xenapi import vm_utils LOG = logging.getLogger(__name__) CONF = cfg.CONF +CONF.import_opt('glance_num_retries', 'nova.image.glance') class GlanceStore(object): @@ -32,24 +36,49 @@ class GlanceStore(object): """ # NOTE(sirp): Currently we only support uploading images as VHD, there # is no RAW equivalent (yet) - LOG.debug(_("Asking xapi to upload to glance %(vdi_uuids)s as" - " ID %(image_id)s"), locals(), instance=instance) - + max_attempts = CONF.glance_num_retries + 1 + sleep_time = 0.5 glance_api_servers = glance.get_api_servers() - glance_host, glance_port, glance_use_ssl = glance_api_servers.next() - properties = { 'auto_disk_config': instance['auto_disk_config'], 'os_type': instance['os_type'] or CONF.default_os_type, } - params = {'vdi_uuids': vdi_uuids, - 'image_id': image_id, - 'glance_host': glance_host, - 'glance_port': glance_port, - 'glance_use_ssl': glance_use_ssl, - 'sr_path': vm_utils.get_sr_path(session), - 'auth_token': getattr(context, 'auth_token', None), - 'properties': properties} + for attempt_num in xrange(1, max_attempts + 1): + + (glance_host, + glance_port, + glance_use_ssl) = glance_api_servers.next() + + try: + + params = {'vdi_uuids': vdi_uuids, + 'image_id': image_id, + 'glance_host': glance_host, + 'glance_port': glance_port, + 'glance_use_ssl': glance_use_ssl, + 'sr_path': vm_utils.get_sr_path(session), + 'auth_token': getattr(context, 'auth_token', None), + 'properties': properties} + + LOG.debug(_("Asking xapi to upload to glance %(vdi_uuids)s as" + " ID %(image_id)s" + " glance server: %(glance_host)s:%(glance_port)d" + " attempt %(attempt_num)d/%(max_attempts)d"), + locals(), instance=instance) + + return session.call_plugin_serialized('glance', + 'upload_vhd', + **params) + + except session.XenAPI.Failure as exc: + _type, _method, error = exc.details[:3] + if error == 'RetryableError': + LOG.error(_('upload_vhd failed: %r') % + (exc.details[3:],)) + else: + raise + time.sleep(sleep_time) + sleep_time = min(2 * sleep_time, 15) - session.call_plugin_serialized('glance', 'upload_vhd', **params) + raise exception.CouldNotUploadImage(image_id=image_id) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 924bf10d7..47b12bd60 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -126,14 +126,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, url = '%s://%s:%s/v1/images/%s' % (scheme, glance_host, glance_port, image_id) logging.info("Writing image data to %s" % url) - if glance_use_ssl: - conn = httplib.HTTPSConnection(glance_host, glance_port) - else: - conn = httplib.HTTPConnection(glance_host, glance_port) - # NOTE(sirp): httplib under python2.4 won't accept a file-like object - # to request - conn.putrequest('PUT', '/v1/images/%s' % image_id) + try: + if glance_use_ssl: + conn = httplib.HTTPSConnection(glance_host, glance_port) + else: + conn = httplib.HTTPConnection(glance_host, glance_port) + + # NOTE(sirp): httplib under python2.4 won't accept a file-like object + # to request + conn.putrequest('PUT', '/v1/images/%s' % image_id) + except Exception, error: + raise RetryableError(error) # NOTE(sirp): There is some confusion around OVF. Here's a summary of # where we currently stand: @@ -172,12 +176,18 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, def send_chunked_transfer_encoded(chunk): chunk_len = len(chunk) callback_data['bytes_written'] += chunk_len - conn.send("%x\r\n%s\r\n" % (chunk_len, chunk)) + try: + conn.send("%x\r\n%s\r\n" % (chunk_len, chunk)) + except Exception, error: + raise RetryableError(error) utils.create_tarball( None, staging_path, callback=send_chunked_transfer_encoded) - conn.send("0\r\n\r\n") # Chunked-Transfer terminator + try: + conn.send("0\r\n\r\n") # Chunked-Transfer terminator + except Exception, error: + raise RetryableError(error) bytes_written = callback_data['bytes_written'] logging.info("Wrote %d bytes to %s" % (bytes_written, url)) @@ -187,9 +197,11 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, logging.error("Unexpected response while writing image data to %s: " "Response Status: %i, Response body: %s" % (url, resp.status, resp.read())) - raise Exception("Unexpected response [%i] while uploading image [%s] " + raise RetryableError("Unexpected response [%i] while uploading " + "image [%s] " "to glance host [%s:%s]" % (resp.status, image_id, glance_host, glance_port)) + conn.close() |