Extend glance retries to show() as well

Fixes bug 930245

nova/image/glance.py would already retry get() requests if there was a
connection error, but didn't do so for show() requests.

Change-Id: Ifb1a17da18b7e10ddaaec46e124efcc963681c80

2 files changed, 27 insertions, 13 deletions

diff --git a/nova/exception.py b/nova/exception.py
index c93e2de11..5c4da0f41 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -196,6 +196,10 @@ class VirtualInterfaceMacAddressException(NovaException):
                 "with unique mac address failed")
 
 
+class GlanceConnectionFailed(NovaException):
+    message = _("Connection to glance failed") + ": %(reason)s"
+
+
 class NotAuthorized(NovaException):
     message = _("Not authorized.")
 
diff --git a/nova/image/glance.py b/nova/image/glance.py
index 092dd832a..e44f0aad8 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -135,6 +135,22 @@ class GlanceImageService(object):
         glance_host, glance_port = pick_glance_api_server()
         return _create_glance_client(context, glance_host, glance_port)
 
+    def _call_retry(self, context, name, *args, **kwargs):
+        """Retry call to glance server if there is a connection error.
+        Suitable only for idempotent calls."""
+        for i in xrange(FLAGS.glance_num_retries + 1):
+            client = self._get_client(context)
+            try:
+                return getattr(client, name)(*args, **kwargs)
+            except glance_exception.ClientConnectionError as e:
+                LOG.exception(_('Connection error contacting glance'
+                                ' server, retrying'))
+
+                time.sleep(1)
+
+        raise exception.GlanceConnectionFailed(
+                reason=_('Maximum attempts reached'))
+
     def index(self, context, **kwargs):
         """Calls out to Glance for a list of images available."""
         params = self._extract_query_params(kwargs)
@@ -215,7 +231,8 @@ class GlanceImageService(object):
     def show(self, context, image_id):
         """Returns a dict with image data for the given opaque image id."""
         try:
-            image_meta = self._get_client(context).get_image_meta(image_id)
+            image_meta = self._call_retry(context, 'get_image_meta',
+                                          image_id)
         except glance_exception.NotFound:
             raise exception.ImageNotFound(image_id=image_id)
 
@@ -235,18 +252,11 @@ class GlanceImageService(object):
 
     def get(self, context, image_id, data):
         """Calls out to Glance for metadata and data and writes data."""
-        num_retries = FLAGS.glance_num_retries
-        for count in xrange(1 + num_retries):
-            client = self._get_client(context)
-            try:
-                image_meta, image_chunks = client.get_image(image_id)
-                break
-            except glance_exception.NotFound:
-                raise exception.ImageNotFound(image_id=image_id)
-            except Exception:
-                if count == num_retries:
-                    raise
-            time.sleep(1)
+        try:
+            image_meta, image_chunks = self._call_retry(context, 'get_image',
+                                                        image_id)
+        except glance_exception.NotFound:
+            raise exception.ImageNotFound(image_id=image_id)
 
         for chunk in image_chunks:
             data.write(chunk)