Adding support for retrying glance image downloads.

Change-Id: Ifff40d90f7dc61a6d41ae2d6908d6e1e6f0aea7e

5 files changed, 59 insertions, 12 deletions

diff --git a/nova/flags.py b/nova/flags.py
index e98c487aa..7253ad553 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -297,6 +297,8 @@ DEFINE_integer('glance_port', 9292, 'default glance port')
 DEFINE_list('glance_api_servers',
             ['%s:%d' % (FLAGS.glance_host, FLAGS.glance_port)],
             'list of glance api servers available to nova (host:port)')
+DEFINE_integer('glance_num_retries', 0,
+               'The number of times to retry downloading an image from glance')
 DEFINE_integer('s3_port', 3333, 's3 port')
 DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)')
 DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)')
diff --git a/nova/image/glance.py b/nova/image/glance.py
index c00e6bb5f..d9905297a 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -23,6 +23,7 @@ import copy
 import datetime
 import json
 import random
+import time
 from urlparse import urlparse
 
 from glance.common import exception as glance_exception
@@ -231,11 +232,18 @@ class GlanceImageService(object):
 
     def get(self, context, image_id, data):
         """Calls out to Glance for metadata and data and writes data."""
-        try:
+        num_retries = FLAGS.glance_num_retries
+        for count in xrange(1 + num_retries):
             client = self._get_client(context)
-            image_meta, image_chunks = client.get_image(image_id)
-        except glance_exception.NotFound:
-            raise exception.ImageNotFound(image_id=image_id)
+            try:
+                image_meta, image_chunks = client.get_image(image_id)
+                break
+            except glance_exception.NotFound:
+                raise exception.ImageNotFound(image_id=image_id)
+            except Exception:
+                if count == num_retries:
+                    raise
+            time.sleep(1)
 
         for chunk in image_chunks:
             data.write(chunk)
diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py
index 0934eb5eb..1ab40f5f6 100644
--- a/nova/tests/image/test_glance.py
+++ b/nova/tests/image/test_glance.py
@@ -508,6 +508,35 @@ class TestGlanceImageService(test.TestCase):
         self.assertEqual(image_meta['created_at'], self.NOW_DATETIME)
         self.assertEqual(image_meta['updated_at'], self.NOW_DATETIME)
 
+    def test_get_with_retries(self):
+        tries = [0]
+
+        class GlanceBusyException(Exception):
+            pass
+
+        class MyGlanceStubClient(glance_stubs.StubGlanceClient):
+            """A client that fails the first time, then succeeds."""
+            def get_image(self, image_id):
+                if tries[0] == 0:
+                    tries[0] = 1
+                    raise GlanceBusyException()
+                else:
+                    return {}, []
+
+        client = MyGlanceStubClient()
+        service = glance.GlanceImageService(client=client)
+        image_id = 1  # doesn't matter
+        writer = NullWriter()
+
+        # When retries are disabled, we should get an exception
+        self.flags(glance_num_retries=0)
+        self.assertRaises(GlanceBusyException, service.get, self.context,
+                          image_id, writer)
+
+        # Now lets enable retries. No exception should happen now.
+        self.flags(glance_num_retries=1)
+        service.get(self.context, image_id, writer)
+
     def test_glance_client_image_id(self):
         fixture = self._make_fixture(name='test image')
         image_id = self.service.create(self.context, fixture)['id']
diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py
index 640c74ab2..7142d0457 100644
--- a/nova/virt/xenapi/vm_utils.py
+++ b/nova/virt/xenapi/vm_utils.py
@@ -582,6 +582,7 @@ w
                   'glance_port': glance_port,
                   'uuid_stack': uuid_stack,
                   'sr_path': cls.get_sr_path(session),
+                  'num_retries': FLAGS.glance_num_retries,
                   'auth_token': getattr(context, 'auth_token', None)}
 
         kwargs = {'params': pickle.dumps(params)}
diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
index d45a0b1c4..47052905d 100755
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
@@ -33,6 +33,7 @@ import shlex
 import shutil
 import subprocess
 import tempfile
+import time
 
 import XenAPIPlugin
 
@@ -67,7 +68,7 @@ def _copy_kernel_vdi(dest, copy_args):
 
 
 def _download_tarball(sr_path, staging_path, image_id, glance_host,
-                      glance_port, auth_token):
+                      glance_port, auth_token, num_retries):
     """Download the tarball image from Glance and extract it into the staging
     area.
     """
@@ -77,12 +78,17 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host,
         headers['x-auth-token'] = auth_token
 
     conn = httplib.HTTPConnection(glance_host, glance_port)
-    conn.request('GET', '/v1/images/%s' % image_id, headers=headers)
-    resp = conn.getresponse()
-    if resp.status == httplib.NOT_FOUND:
-        raise Exception("Image '%s' not found in Glance" % image_id)
-    elif resp.status != httplib.OK:
-        raise Exception("Unexpected response from Glance %i" % resp.status)
+
+    for count in xrange(1 + num_retries):
+        conn.request('GET', '/v1/images/%s' % image_id, headers=headers)
+        resp = conn.getresponse()
+        if resp.status == httplib.OK:
+            break
+        elif resp.status == httplib.NOT_FOUND:
+            raise Exception("Image '%s' not found in Glance" % image_id)
+        elif count == num_retries:
+            raise Exception("Unexpected response from Glance %i" % resp.status)
+        time.sleep(1)
 
     tar_cmd = "tar -zx --directory=%(staging_path)s" % locals()
     tar_proc = _make_subprocess(tar_cmd, stderr=True, stdin=True)
@@ -404,11 +410,12 @@ def download_vhd(session, args):
     uuid_stack = params["uuid_stack"]
     sr_path = params["sr_path"]
     auth_token = params["auth_token"]
+    num_retries = params["num_retries"]
 
     staging_path = _make_staging_area(sr_path)
     try:
         _download_tarball(sr_path, staging_path, image_id, glance_host,
-                          glance_port, auth_token)
+                          glance_port, auth_token, num_retries)
         # Right now, it's easier to return a single string via XenAPI,
         # so we'll json encode the list of VHDs.
         return json.dumps(_import_vhds(sr_path, staging_path, uuid_stack))