Add support for middleware proxying to a ratelimiting.WSGIApp, for deployments that use more than one API Server and thus can't store ratelimiting counters in memory.

3 files changed, 165 insertions, 25 deletions

diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py
index 66d80a5b7..ac5365310 100644
--- a/nova/api/rackspace/__init__.py
+++ b/nova/api/rackspace/__init__.py
@@ -69,17 +69,26 @@ class AuthMiddleware(wsgi.Middleware):
 class RateLimitingMiddleware(wsgi.Middleware):
     """Rate limit incoming requests according to the OpenStack rate limits."""
 
-    def __init__(self, application):
+    def __init__(self, application, service_host=None):
+        """Create a rate limiting middleware that wraps the given application.
+
+        By default, rate counters are stored in memory.  If service_host is
+        specified, the middleware instead relies on the ratelimiting.WSGIApp
+        at the given host+port to keep rate counters.
+        """
         super(RateLimitingMiddleware, self).__init__(application)
-        #TODO(gundlach): These limits were based on limitations of Cloud 
-        #Servers.  We should revisit them in Nova.
-        self.limiter = ratelimiting.Limiter(limits={
-                'DELETE': (100, ratelimiting.PER_MINUTE),
-                'PUT': (10, ratelimiting.PER_MINUTE),
-                'POST': (10, ratelimiting.PER_MINUTE),
-                'POST servers': (50, ratelimiting.PER_DAY),
-                'GET changes-since': (3, ratelimiting.PER_MINUTE),
-            })
+        if not service_host:
+            #TODO(gundlach): These limits were based on limitations of Cloud 
+            #Servers.  We should revisit them in Nova.
+            self.limiter = ratelimiting.Limiter(limits={
+                    'DELETE': (100, ratelimiting.PER_MINUTE),
+                    'PUT': (10, ratelimiting.PER_MINUTE),
+                    'POST': (10, ratelimiting.PER_MINUTE),
+                    'POST servers': (50, ratelimiting.PER_DAY),
+                    'GET changes-since': (3, ratelimiting.PER_MINUTE),
+                })
+        else:
+            self.limiter = ratelimiting.WSGIAppProxy(service_host)
 
     @webob.dec.wsgify
     def __call__(self, req):
diff --git a/nova/api/rackspace/ratelimiting/__init__.py b/nova/api/rackspace/ratelimiting/__init__.py
index 64d5fff2c..f843bac0f 100644
--- a/nova/api/rackspace/ratelimiting/__init__.py
+++ b/nova/api/rackspace/ratelimiting/__init__.py
@@ -1,5 +1,6 @@
 """Rate limiting of arbitrary actions."""
 
+import httplib
 import time
 import urllib
 import webob.dec
@@ -98,6 +99,24 @@ class WSGIApp(object):
         delay = self.limiter.perform(action_name, username)
         if delay:
             return webob.exc.HTTPForbidden(
-                    headers={'X-Wait-Seconds': delay})
+                    headers={'X-Wait-Seconds': "%.2f" % delay})
         else:
             return '' # 200 OK
+
+
+class WSGIAppProxy(object):
+
+    """Limiter lookalike that proxies to a ratelimiting.WSGIApp."""
+
+    def __init__(self, service_host):
+        """Creates a proxy pointing to a ratelimiting.WSGIApp at the given 
+        host."""
+        self.service_host = service_host
+
+    def perform(self, action, username='nobody'):
+        conn = httplib.HTTPConnection(self.service_host)
+        conn.request('POST', '/limiter/%s/%s' % (username, action))
+        resp = conn.getresponse()
+        if resp.status == 200:
+            return None # no delay
+        return float(resp.getheader('X-Wait-Seconds'))
diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py
index f924e7805..13a47989b 100644
--- a/nova/api/rackspace/ratelimiting/tests.py
+++ b/nova/api/rackspace/ratelimiting/tests.py
@@ -1,3 +1,5 @@
+import httplib
+import StringIO
 import time
 import unittest
 import webob
@@ -62,22 +64,25 @@ class LimiterTest(unittest.TestCase):
         self.exhaust('c', 0, username='alice')
 
 
+class FakeLimiter(object):
+    """Fake Limiter class that you can tell how to behave."""
+    def __init__(self, test):
+        self._action = self._username = self._delay = None
+        self.test = test
+    def mock(self, action, username, delay):
+        self._action = action
+        self._username = username
+        self._delay = delay
+    def perform(self, action, username):
+        self.test.assertEqual(action, self._action)
+        self.test.assertEqual(username, self._username)
+        return self._delay
+
+
 class WSGIAppTest(unittest.TestCase):
 
     def setUp(self):
-        test = self
-        class FakeLimiter(object):
-            def __init__(self):
-                self._action = self._username = self._delay = None
-            def mock(self, action, username, delay):
-                self._action = action
-                self._username = username
-                self._delay = delay
-            def perform(self, action, username):
-                test.assertEqual(action, self._action)
-                test.assertEqual(username, self._username)
-                return self._delay
-        self.limiter = FakeLimiter()
+        self.limiter = FakeLimiter(self)
         self.app = ratelimiting.WSGIApp(self.limiter)
 
     def test_invalid_methods(self):
@@ -110,7 +115,7 @@ class WSGIAppTest(unittest.TestCase):
             self.assertEqual(resp.status_int, 200)
         else:
             self.assertEqual(resp.status_int, 403)
-            self.assertEqual(resp.headers['X-Wait-Seconds'], delay)
+            self.assertEqual(resp.headers['X-Wait-Seconds'], "%.2f" % delay)
 
     def test_good_urls(self):
         self.verify('/limiter/michael/hoot', 'michael', 'hoot')
@@ -124,5 +129,112 @@ class WSGIAppTest(unittest.TestCase):
         self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1000)
 
 
+class FakeHttplibSocket(object):
+    """a fake socket implementation for httplib.HTTPResponse, trivial"""
+
+    def __init__(self, response_string):
+        self._buffer = StringIO.StringIO(response_string)
+
+    def makefile(self, _mode, _other):
+        """Returns the socket's internal buffer"""
+        return self._buffer
+
+
+class FakeHttplibConnection(object):
+    """A fake httplib.HTTPConnection
+
+    Requests made via this connection actually get translated and routed into
+    our WSGI app, we then wait for the response and turn it back into
+    an httplib.HTTPResponse.
+    """
+    def __init__(self, app, host, is_secure=False):
+        self.app = app
+        self.host = host
+
+    def request(self, method, path, data='', headers={}):
+        req = webob.Request.blank(path)
+        req.method = method
+        req.body = data
+        req.headers = headers
+        req.host = self.host
+        # Call the WSGI app, get the HTTP response
+        resp = str(req.get_response(self.app))
+        # For some reason, the response doesn't have "HTTP/1.0 " prepended; I
+        # guess that's a function the web server usually provides.
+        resp = "HTTP/1.0 %s" % resp
+        sock = FakeHttplibSocket(resp)
+        self.http_response = httplib.HTTPResponse(sock)
+        self.http_response.begin()
+
+    def getresponse(self):
+        return self.http_response
+
+
+def wire_HTTPConnection_to_WSGI(host, app):
+    """Monkeypatches HTTPConnection so that if you try to connect to host, you
+    are instead routed straight to the given WSGI app.
+
+    After calling this method, when any code calls
+
+    httplib.HTTPConnection(host)
+
+    the connection object will be a fake.  Its requests will be sent directly
+    to the given WSGI app rather than through a socket.
+    
+    Code connecting to hosts other than host will not be affected.
+
+    This method may be called multiple times to map different hosts to
+    different apps.
+    """
+    class HTTPConnectionDecorator(object):
+        """Wraps the real HTTPConnection class so that when you instantiate
+        the class you might instead get a fake instance."""
+        def __init__(self, wrapped):
+            self.wrapped = wrapped
+        def __call__(self, connection_host, *args, **kwargs):
+            if connection_host == host:
+                return FakeHttplibConnection(app, host)
+            else:
+                return self.wrapped(connection_host, *args, **kwargs)
+    httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection)
+
+
+class WSGIAppProxyTest(unittest.TestCase):
+
+    def setUp(self):
+        """Our WSGIAppProxy is going to call across an HTTPConnection to a
+        WSGIApp running a limiter.  The proxy will send input, and the proxy
+        should receive that same input, pass it to the limiter who gives a
+        result, and send the expected result back.
+
+        The HTTPConnection isn't real -- it's monkeypatched to point straight
+        at the WSGIApp.  And the limiter isn't real -- it's a fake that
+        behaves the way we tell it to.
+        """
+        self.limiter = FakeLimiter(self)
+        app = ratelimiting.WSGIApp(self.limiter)
+        wire_HTTPConnection_to_WSGI('100.100.100.100:80', app)
+        self.proxy = ratelimiting.WSGIAppProxy('100.100.100.100:80')
+
+    def test_200(self):
+        self.limiter.mock('conquer', 'caesar', None)
+        when = self.proxy.perform('conquer', 'caesar')
+        self.assertEqual(when, None)
+
+    def test_403(self):
+        self.limiter.mock('grumble', 'proletariat', 1.5)
+        when = self.proxy.perform('grumble', 'proletariat')
+        self.assertEqual(when, 1.5)
+
+    def test_failure(self):
+        self.limiter.mock('murder', 'brutus', None)
+        try:
+            when = self.proxy.perform('stab', 'brutus')
+        except AssertionError:
+            pass
+        else:
+            self.fail("I didn't perform the action I expected")
+
+
 if __name__ == '__main__':
     unittest.main()