Adding support for retrying glance image downloads.

Change-Id: Ifff40d90f7dc61a6d41ae2d6908d6e1e6f0aea7e

4 files changed, 44 insertions, 4 deletions

diff --git a/nova/flags.py b/nova/flags.py
index e98c487aa..7253ad553 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -297,6 +297,8 @@ DEFINE_integer('glance_port', 9292, 'default glance port')
 DEFINE_list('glance_api_servers',
             ['%s:%d' % (FLAGS.glance_host, FLAGS.glance_port)],
             'list of glance api servers available to nova (host:port)')
+DEFINE_integer('glance_num_retries', 0,
+               'The number of times to retry downloading an image from glance')
 DEFINE_integer('s3_port', 3333, 's3 port')
 DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)')
 DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)')
diff --git a/nova/image/glance.py b/nova/image/glance.py
index c00e6bb5f..d9905297a 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -23,6 +23,7 @@ import copy
 import datetime
 import json
 import random
+import time
 from urlparse import urlparse
 
 from glance.common import exception as glance_exception
@@ -231,11 +232,18 @@ class GlanceImageService(object):
 
     def get(self, context, image_id, data):
         """Calls out to Glance for metadata and data and writes data."""
-        try:
+        num_retries = FLAGS.glance_num_retries
+        for count in xrange(1 + num_retries):
             client = self._get_client(context)
-            image_meta, image_chunks = client.get_image(image_id)
-        except glance_exception.NotFound:
-            raise exception.ImageNotFound(image_id=image_id)
+            try:
+                image_meta, image_chunks = client.get_image(image_id)
+                break
+            except glance_exception.NotFound:
+                raise exception.ImageNotFound(image_id=image_id)
+            except Exception:
+                if count == num_retries:
+                    raise
+            time.sleep(1)
 
         for chunk in image_chunks:
             data.write(chunk)
diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py
index 0934eb5eb..1ab40f5f6 100644
--- a/nova/tests/image/test_glance.py
+++ b/nova/tests/image/test_glance.py
@@ -508,6 +508,35 @@ class TestGlanceImageService(test.TestCase):
         self.assertEqual(image_meta['created_at'], self.NOW_DATETIME)
         self.assertEqual(image_meta['updated_at'], self.NOW_DATETIME)
 
+    def test_get_with_retries(self):
+        tries = [0]
+
+        class GlanceBusyException(Exception):
+            pass
+
+        class MyGlanceStubClient(glance_stubs.StubGlanceClient):
+            """A client that fails the first time, then succeeds."""
+            def get_image(self, image_id):
+                if tries[0] == 0:
+                    tries[0] = 1
+                    raise GlanceBusyException()
+                else:
+                    return {}, []
+
+        client = MyGlanceStubClient()
+        service = glance.GlanceImageService(client=client)
+        image_id = 1  # doesn't matter
+        writer = NullWriter()
+
+        # When retries are disabled, we should get an exception
+        self.flags(glance_num_retries=0)
+        self.assertRaises(GlanceBusyException, service.get, self.context,
+                          image_id, writer)
+
+        # Now lets enable retries. No exception should happen now.
+        self.flags(glance_num_retries=1)
+        service.get(self.context, image_id, writer)
+
     def test_glance_client_image_id(self):
         fixture = self._make_fixture(name='test image')
         image_id = self.service.create(self.context, fixture)['id']
diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py
index 640c74ab2..7142d0457 100644
--- a/nova/virt/xenapi/vm_utils.py
+++ b/nova/virt/xenapi/vm_utils.py
@@ -582,6 +582,7 @@ w
                   'glance_port': glance_port,
                   'uuid_stack': uuid_stack,
                   'sr_path': cls.get_sr_path(session),
+                  'num_retries': FLAGS.glance_num_retries,
                   'auth_token': getattr(context, 'auth_token', None)}
 
         kwargs = {'params': pickle.dumps(params)}