From f200587ce068482ab94e777154de3ac777269fa0 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 15 Sep 2010 13:54:38 -0400 Subject: Add support for middleware proxying to a ratelimiting.WSGIApp, for deployments that use more than one API Server and thus can't store ratelimiting counters in memory. --- nova/api/rackspace/__init__.py | 29 ++++-- nova/api/rackspace/ratelimiting/__init__.py | 21 ++++- nova/api/rackspace/ratelimiting/tests.py | 140 +++++++++++++++++++++++++--- 3 files changed, 165 insertions(+), 25 deletions(-) (limited to 'nova/api') diff --git a/nova/api/rackspace/__init__.py b/nova/api/rackspace/__init__.py index 66d80a5b7..ac5365310 100644 --- a/nova/api/rackspace/__init__.py +++ b/nova/api/rackspace/__init__.py @@ -69,17 +69,26 @@ class AuthMiddleware(wsgi.Middleware): class RateLimitingMiddleware(wsgi.Middleware): """Rate limit incoming requests according to the OpenStack rate limits.""" - def __init__(self, application): + def __init__(self, application, service_host=None): + """Create a rate limiting middleware that wraps the given application. + + By default, rate counters are stored in memory. If service_host is + specified, the middleware instead relies on the ratelimiting.WSGIApp + at the given host+port to keep rate counters. + """ super(RateLimitingMiddleware, self).__init__(application) - #TODO(gundlach): These limits were based on limitations of Cloud - #Servers. We should revisit them in Nova. - self.limiter = ratelimiting.Limiter(limits={ - 'DELETE': (100, ratelimiting.PER_MINUTE), - 'PUT': (10, ratelimiting.PER_MINUTE), - 'POST': (10, ratelimiting.PER_MINUTE), - 'POST servers': (50, ratelimiting.PER_DAY), - 'GET changes-since': (3, ratelimiting.PER_MINUTE), - }) + if not service_host: + #TODO(gundlach): These limits were based on limitations of Cloud + #Servers. We should revisit them in Nova. + self.limiter = ratelimiting.Limiter(limits={ + 'DELETE': (100, ratelimiting.PER_MINUTE), + 'PUT': (10, ratelimiting.PER_MINUTE), + 'POST': (10, ratelimiting.PER_MINUTE), + 'POST servers': (50, ratelimiting.PER_DAY), + 'GET changes-since': (3, ratelimiting.PER_MINUTE), + }) + else: + self.limiter = ratelimiting.WSGIAppProxy(service_host) @webob.dec.wsgify def __call__(self, req): diff --git a/nova/api/rackspace/ratelimiting/__init__.py b/nova/api/rackspace/ratelimiting/__init__.py index 64d5fff2c..f843bac0f 100644 --- a/nova/api/rackspace/ratelimiting/__init__.py +++ b/nova/api/rackspace/ratelimiting/__init__.py @@ -1,5 +1,6 @@ """Rate limiting of arbitrary actions.""" +import httplib import time import urllib import webob.dec @@ -98,6 +99,24 @@ class WSGIApp(object): delay = self.limiter.perform(action_name, username) if delay: return webob.exc.HTTPForbidden( - headers={'X-Wait-Seconds': delay}) + headers={'X-Wait-Seconds': "%.2f" % delay}) else: return '' # 200 OK + + +class WSGIAppProxy(object): + + """Limiter lookalike that proxies to a ratelimiting.WSGIApp.""" + + def __init__(self, service_host): + """Creates a proxy pointing to a ratelimiting.WSGIApp at the given + host.""" + self.service_host = service_host + + def perform(self, action, username='nobody'): + conn = httplib.HTTPConnection(self.service_host) + conn.request('POST', '/limiter/%s/%s' % (username, action)) + resp = conn.getresponse() + if resp.status == 200: + return None # no delay + return float(resp.getheader('X-Wait-Seconds')) diff --git a/nova/api/rackspace/ratelimiting/tests.py b/nova/api/rackspace/ratelimiting/tests.py index f924e7805..13a47989b 100644 --- a/nova/api/rackspace/ratelimiting/tests.py +++ b/nova/api/rackspace/ratelimiting/tests.py @@ -1,3 +1,5 @@ +import httplib +import StringIO import time import unittest import webob @@ -62,22 +64,25 @@ class LimiterTest(unittest.TestCase): self.exhaust('c', 0, username='alice') +class FakeLimiter(object): + """Fake Limiter class that you can tell how to behave.""" + def __init__(self, test): + self._action = self._username = self._delay = None + self.test = test + def mock(self, action, username, delay): + self._action = action + self._username = username + self._delay = delay + def perform(self, action, username): + self.test.assertEqual(action, self._action) + self.test.assertEqual(username, self._username) + return self._delay + + class WSGIAppTest(unittest.TestCase): def setUp(self): - test = self - class FakeLimiter(object): - def __init__(self): - self._action = self._username = self._delay = None - def mock(self, action, username, delay): - self._action = action - self._username = username - self._delay = delay - def perform(self, action, username): - test.assertEqual(action, self._action) - test.assertEqual(username, self._username) - return self._delay - self.limiter = FakeLimiter() + self.limiter = FakeLimiter(self) self.app = ratelimiting.WSGIApp(self.limiter) def test_invalid_methods(self): @@ -110,7 +115,7 @@ class WSGIAppTest(unittest.TestCase): self.assertEqual(resp.status_int, 200) else: self.assertEqual(resp.status_int, 403) - self.assertEqual(resp.headers['X-Wait-Seconds'], delay) + self.assertEqual(resp.headers['X-Wait-Seconds'], "%.2f" % delay) def test_good_urls(self): self.verify('/limiter/michael/hoot', 'michael', 'hoot') @@ -124,5 +129,112 @@ class WSGIAppTest(unittest.TestCase): self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1000) +class FakeHttplibSocket(object): + """a fake socket implementation for httplib.HTTPResponse, trivial""" + + def __init__(self, response_string): + self._buffer = StringIO.StringIO(response_string) + + def makefile(self, _mode, _other): + """Returns the socket's internal buffer""" + return self._buffer + + +class FakeHttplibConnection(object): + """A fake httplib.HTTPConnection + + Requests made via this connection actually get translated and routed into + our WSGI app, we then wait for the response and turn it back into + an httplib.HTTPResponse. + """ + def __init__(self, app, host, is_secure=False): + self.app = app + self.host = host + + def request(self, method, path, data='', headers={}): + req = webob.Request.blank(path) + req.method = method + req.body = data + req.headers = headers + req.host = self.host + # Call the WSGI app, get the HTTP response + resp = str(req.get_response(self.app)) + # For some reason, the response doesn't have "HTTP/1.0 " prepended; I + # guess that's a function the web server usually provides. + resp = "HTTP/1.0 %s" % resp + sock = FakeHttplibSocket(resp) + self.http_response = httplib.HTTPResponse(sock) + self.http_response.begin() + + def getresponse(self): + return self.http_response + + +def wire_HTTPConnection_to_WSGI(host, app): + """Monkeypatches HTTPConnection so that if you try to connect to host, you + are instead routed straight to the given WSGI app. + + After calling this method, when any code calls + + httplib.HTTPConnection(host) + + the connection object will be a fake. Its requests will be sent directly + to the given WSGI app rather than through a socket. + + Code connecting to hosts other than host will not be affected. + + This method may be called multiple times to map different hosts to + different apps. + """ + class HTTPConnectionDecorator(object): + """Wraps the real HTTPConnection class so that when you instantiate + the class you might instead get a fake instance.""" + def __init__(self, wrapped): + self.wrapped = wrapped + def __call__(self, connection_host, *args, **kwargs): + if connection_host == host: + return FakeHttplibConnection(app, host) + else: + return self.wrapped(connection_host, *args, **kwargs) + httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) + + +class WSGIAppProxyTest(unittest.TestCase): + + def setUp(self): + """Our WSGIAppProxy is going to call across an HTTPConnection to a + WSGIApp running a limiter. The proxy will send input, and the proxy + should receive that same input, pass it to the limiter who gives a + result, and send the expected result back. + + The HTTPConnection isn't real -- it's monkeypatched to point straight + at the WSGIApp. And the limiter isn't real -- it's a fake that + behaves the way we tell it to. + """ + self.limiter = FakeLimiter(self) + app = ratelimiting.WSGIApp(self.limiter) + wire_HTTPConnection_to_WSGI('100.100.100.100:80', app) + self.proxy = ratelimiting.WSGIAppProxy('100.100.100.100:80') + + def test_200(self): + self.limiter.mock('conquer', 'caesar', None) + when = self.proxy.perform('conquer', 'caesar') + self.assertEqual(when, None) + + def test_403(self): + self.limiter.mock('grumble', 'proletariat', 1.5) + when = self.proxy.perform('grumble', 'proletariat') + self.assertEqual(when, 1.5) + + def test_failure(self): + self.limiter.mock('murder', 'brutus', None) + try: + when = self.proxy.perform('stab', 'brutus') + except AssertionError: + pass + else: + self.fail("I didn't perform the action I expected") + + if __name__ == '__main__': unittest.main() -- cgit