Retry on network failure for melange GET requests

Fixes bug 929041

The melange service may need to periodically be restarted for various
operations reasons, so retry GET requests if we receive a socket error.

Change-Id: I010650dac7793d0041d41be067eb4b32e2171c8a

2 files changed, 28 insertions, 13 deletions

diff --git a/nova/exception.py b/nova/exception.py
index 5c4da0f41..c31f5e177 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -200,6 +200,10 @@ class GlanceConnectionFailed(NovaException):
     message = _("Connection to glance failed") + ": %(reason)s"
 
 
+class MelangeConnectionFailed(NovaException):
+    message = _("Connection to melange failed") + ": %(reason)s"
+
+
 class NotAuthorized(NovaException):
     message = _("Not authorized.")
 
diff --git a/nova/network/quantum/melange_connection.py b/nova/network/quantum/melange_connection.py
index d579b5f56..449b4f366 100644
--- a/nova/network/quantum/melange_connection.py
+++ b/nova/network/quantum/melange_connection.py
@@ -16,9 +16,10 @@
 #    under the License.
 
 import httplib
+import json
 import socket
+import time
 import urllib
-import json
 
 from nova import flags
 from nova import log as logging
@@ -32,6 +33,9 @@ melange_opts = [
     cfg.StrOpt('melange_port',
                default='9898',
                help='PORT for connecting to melange'),
+    cfg.IntOpt('melange_num_retries',
+               default=0,
+               help='Number retries when contacting melange'),
     ]
 
 FLAGS = flags.FLAGS
@@ -57,7 +61,8 @@ class MelangeConnection(object):
         self.version = "v0.1"
 
     def get(self, path, params=None, headers=None):
-        return self.do_request("GET", path, params=params, headers=headers)
+        return self.do_request("GET", path, params=params, headers=headers,
+                               retries=FLAGS.melange_num_retries)
 
     def post(self, path, body=None, headers=None):
         return self.do_request("POST", path, body=body, headers=headers)
@@ -72,24 +77,30 @@ class MelangeConnection(object):
             return httplib.HTTPConnection(self.host, self.port)
 
     def do_request(self, method, path, body=None, headers=None, params=None,
-                   content_type=".json"):
+                   content_type=".json", retries=0):
         headers = headers or {}
         params = params or {}
 
         url = "/%s/%s%s" % (self.version, path, content_type)
         if params:
             url += "?%s" % urllib.urlencode(params)
-        try:
+        for i in xrange(retries + 1):
             connection = self._get_connection()
-            connection.request(method, url, body, headers)
-            response = connection.getresponse()
-            response_str = response.read()
-            if response.status < 400:
-                return response_str
-            raise Exception(_("Server returned error: %s" % response_str))
-        except (socket.error, IOError), e:
-            raise Exception(_("Unable to connect to "
-                              "server. Got error: %s" % e))
+            try:
+                connection.request(method, url, body, headers)
+                response = connection.getresponse()
+                response_str = response.read()
+                if response.status < 400:
+                    return response_str
+                raise Exception(_("Server returned error: %s" % response_str))
+            except (socket.error, IOError), e:
+                LOG.exception(_('Connection error contacting melange'
+                                ' service, retrying'))
+
+                time.sleep(1)
+
+        raise exception.MelangeConnectionFailed(
+                reason=_("Maximum attempts reached"))
 
     def allocate_ip(self, network_id, network_tenant_id, vif_id,
                     project_id=None, mac_address=None):