From f3350750a7c68b191c3a85ac9caa5eff59b182a6 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 23 Jul 2010 08:03:26 -0500 Subject: Check signature for S3 requests. --- nova/auth/signer.py | 8 ++++++++ nova/auth/users.py | 13 +++++++++++-- nova/objectstore/handler.py | 14 +++++++------- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/nova/auth/signer.py b/nova/auth/signer.py index 83831bfac..7d7471575 100644 --- a/nova/auth/signer.py +++ b/nova/auth/signer.py @@ -48,6 +48,7 @@ import hashlib import hmac import logging import urllib +import boto.utils from nova.exception import Error @@ -59,6 +60,13 @@ class Signer(object): if hashlib.sha256: self.hmac_256 = hmac.new(secret_key, digestmod=hashlib.sha256) + def s3_authorization(self, headers, verb, path): + c_string = boto.utils.canonical_string(verb, path, headers) + hmac = self.hmac.copy() + hmac.update(c_string) + b64_hmac = base64.encodestring(hmac.digest()).strip() + return b64_hmac + def generate(self, params, verb, server_string, path): if params['SignatureVersion'] == '0': return self._calc_signature_0(params) diff --git a/nova/auth/users.py b/nova/auth/users.py index fc08dc34d..0e9ca4eeb 100644 --- a/nova/auth/users.py +++ b/nova/auth/users.py @@ -395,11 +395,13 @@ class UserManager(object): def authenticate(self, access, signature, params, verb='GET', server_string='127.0.0.1:8773', path='/', - verify_signature=True): + check_type='ec2', headers=None): # TODO: Check for valid timestamp (access_key, sep, project_name) = access.partition(':') + logging.info('Looking up user: %r', access_key) user = self.get_user_from_access_key(access_key) + logging.info('user: %r', user) if user == None: raise exception.NotFound('No user found for access key %s' % access_key) @@ -413,7 +415,14 @@ class UserManager(object): if not user.is_admin() and not project.has_member(user): raise exception.NotFound('User %s is not a member of project %s' % (user.id, project.id)) - if verify_signature: + if check_type == 's3': + expected_signature = signer.Signer(user.secret.encode()).s3_authorization(headers, verb, path) + logging.debug('user.secret: %s', user.secret) + logging.debug('expected_signature: %s', expected_signature) + logging.debug('signature: %s', signature) + if signature != expected_signature: + raise exception.NotAuthorized('Signature does not match') + elif check_type == 'ec2': # NOTE(vish): hmac can't handle unicode, so encode ensures that # secret isn't unicode expected_signature = signer.Signer(user.secret.encode()).generate( diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index b2ed3d482..655cab752 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -106,8 +106,8 @@ def get_context(request): access, sep, secret = request.getHeader('Authorization').split(' ')[1].rpartition(':') um = users.UserManager.instance() print 'um %s' % um - (user, project) = um.authenticate(access, secret, {}, request.method, request.host, request.uri, False) - # FIXME: check signature here! + + (user, project) = um.authenticate(access, secret, {}, request.method, request.getRequestHostname(), request.uri, headers=request.getAllHeaders(), check_type='s3') return api.APIRequestContext(None, user, project) except exception.Error, ex: logging.debug("Authentication Failure: %s" % ex) @@ -117,7 +117,6 @@ class S3(Resource): """Implementation of an S3-like storage server based on local files.""" def getChild(self, name, request): request.context = get_context(request) - if name == '': return self elif name == '_images': @@ -126,6 +125,7 @@ class S3(Resource): return BucketResource(name) def render_GET(self, request): + logging.debug('List of buckets requested') buckets = [b for b in bucket.Bucket.all() if b.is_authorized(request.context)] render_xml(request, {"ListAllMyBucketsResult": { @@ -169,7 +169,8 @@ class BucketResource(Resource): logging.exception(e) logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context)) bucket.Bucket.create(self.name, request.context) - return '' + request.finish() + return server.NOT_DONE_YET def render_DELETE(self, request): logging.debug("Deleting bucket %s" % (self.name)) @@ -225,8 +226,6 @@ class ObjectResource(Resource): return '' class ImageResource(Resource): - isLeaf = True - def getChild(self, name, request): if name == '': return self @@ -239,9 +238,10 @@ class ImageResource(Resource): """ returns a json listing of all images that a user has permissions to see """ - images = [i for i in image.Image.all() if i.is_authorized(self.context)] + images = [i for i in image.Image.all() if i.is_authorized(request.context)] request.write(json.dumps([i.metadata for i in images])) + request.finish() return server.NOT_DONE_YET def render_PUT(self, request): -- cgit From abac2033b4aeb399786735a94ce2ddf5c64e6e9e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 23 Jul 2010 23:55:39 +0200 Subject: Add (completely untested) code to include an Authorization header for the S3 request to fetch an image. --- nova/compute/node.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index 4683f1c8d..7c09d9583 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -25,6 +25,7 @@ Compute Node: """ import base64 +import boto.utils import json import logging import os @@ -45,11 +46,13 @@ from nova import fakevirt from nova import flags from nova import process from nova import utils +from nova.auth import signer from nova.compute import disk from nova.compute import model from nova.compute import network from nova.objectstore import image # for image_path flag from nova.volume import storage +from nova.users import UserManager FLAGS = flags.FLAGS @@ -446,8 +449,12 @@ class Instance(object): def _fetch_s3_image(self, image, path): url = _image_url('%s/image' % image) + user_id = self.datamodel['user_id'] + user = UserManager.instance().get_user(user_id) + auth = signer.Signer(user.secret.encode()).s3_authorization({}, 'GET', url) + auth_header = 'Authorization: %s:%s' % (user_id, auth) d = process.simple_execute( - 'curl --silent %s -o %s' % (url, path)) + 'curl --silent %s -o "%s"' % (url, auth_header, path)) return d def _fetch_local_image(self, image, path): -- cgit From 01dffdd79098429a448cf283119d9026ae1231c2 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sun, 25 Jul 2010 14:29:20 +0200 Subject: Always make sure to set a Date headers, since it's needed to calculate the S3 Auth header. --- nova/compute/node.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index 7c09d9583..a01cb3070 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -47,12 +47,12 @@ from nova import flags from nova import process from nova import utils from nova.auth import signer +from nova.auth.users import UserManager from nova.compute import disk from nova.compute import model from nova.compute import network from nova.objectstore import image # for image_path flag from nova.volume import storage -from nova.users import UserManager FLAGS = flags.FLAGS @@ -449,12 +449,18 @@ class Instance(object): def _fetch_s3_image(self, image, path): url = _image_url('%s/image' % image) + headers = {} + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) + user_id = self.datamodel['user_id'] user = UserManager.instance().get_user(user_id) - auth = signer.Signer(user.secret.encode()).s3_authorization({}, 'GET', url) - auth_header = 'Authorization: %s:%s' % (user_id, auth) - d = process.simple_execute( - 'curl --silent %s -o "%s"' % (url, auth_header, path)) + auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', url) + headers['Authorization'] = auth + + headers_opt_string = ' '.join(['-H %s:%s' % (k,v) for (k,v) in headers.iteritems()]) + d = process.simple_execute('curl --silent %s ' + '%s -o "%s"' % (url, headers_opt_string, + path)) return d def _fetch_local_image(self, image, path): -- cgit From dd17c810029d142abaa8f0c97e30a8ae1dc13b5c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sun, 25 Jul 2010 16:17:55 +0200 Subject: Return a 404 when attempting to access a bucket that does not exist. --- nova/objectstore/handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 558b04881..1a54f8a98 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -47,7 +47,7 @@ import urllib from twisted.application import internet, service from twisted.web.resource import Resource -from twisted.web import server, static +from twisted.web import server, static, error from nova import exception @@ -150,7 +150,10 @@ class BucketResource(Resource): def render_GET(self, request): logging.debug("List keys for bucket %s" % (self.name)) - bucket_object = bucket.Bucket(self.name) + try: + bucket_object = bucket.Bucket(self.name) + except exception.NotFound, e: + return error.NoResource(message="No such bucket").render(request) if not bucket_object.is_authorized(request.context): raise exception.NotAuthorized -- cgit From d5051bf90909f693a74ea0f2bdc3db341460a5a3 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 26 Jul 2010 16:03:23 +0200 Subject: Add a simple set of tests for S3 API (using boto). --- nova/objectstore/handler.py | 14 ++-- nova/tests/objectstore_unittest.py | 131 ++++++++++++++++++++++++++++--------- 2 files changed, 105 insertions(+), 40 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 1a54f8a98..45eeef163 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -108,8 +108,6 @@ def get_context(request): raise exception.NotAuthorized access, sep, secret = authorization_header.split(' ')[1].rpartition(':') um = users.UserManager.instance() - print 'um %s' % um - (user, project) = um.authenticate(access, secret, {}, request.method, request.getRequestHostname(), request.uri, headers=request.getAllHeaders(), check_type='s3') return api.APIRequestContext(None, user, project) except exception.Error as ex: @@ -169,10 +167,6 @@ class BucketResource(Resource): def render_PUT(self, request): logging.debug("Creating bucket %s" % (self.name)) - try: - print 'user is %s' % request.context - except Exception as e: - logging.exception(e) logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context)) bucket.Bucket.create(self.name, request.context) request.finish() @@ -300,9 +294,13 @@ class ImageResource(Resource): request.setResponseCode(204) return '' -def get_application(): +def get_site(): root = S3() - factory = server.Site(root) + site = server.Site(root) + return site + +def get_application(): + factory = get_site() application = service.Application("objectstore") objectStoreService = internet.TCPServer(FLAGS.s3_port, factory) objectStoreService.setServiceParent(application) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index f47ca7f00..ef1a477ff 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +import boto import glob import hashlib import logging @@ -27,7 +28,11 @@ from nova import flags from nova import objectstore from nova import test from nova.auth import users +from nova.objectstore.handler import S3 +from boto.s3.connection import S3Connection, OrdinaryCallingFormat +from twisted.internet import reactor, threads, defer +from twisted.web import http, server FLAGS = flags.FLAGS @@ -169,35 +174,97 @@ class ObjectStoreTestCase(test.BaseTestCase): self.context.project = self.um.get_project('proj2') self.assert_(my_img.is_authorized(self.context) == False) -# class ApiObjectStoreTestCase(test.BaseTestCase): -# def setUp(self): -# super(ApiObjectStoreTestCase, self).setUp() -# FLAGS.fake_users = True -# FLAGS.buckets_path = os.path.join(tempdir, 'buckets') -# FLAGS.images_path = os.path.join(tempdir, 'images') -# FLAGS.ca_path = os.path.join(os.path.dirname(__file__), 'CA') -# -# self.users = users.UserManager.instance() -# self.app = handler.Application(self.users) -# -# self.host = '127.0.0.1' -# -# self.conn = boto.s3.connection.S3Connection( -# aws_access_key_id=user.access, -# aws_secret_access_key=user.secret, -# is_secure=False, -# calling_format=boto.s3.connection.OrdinaryCallingFormat(), -# port=FLAGS.s3_port, -# host=FLAGS.s3_host) -# -# self.mox.StubOutWithMock(self.ec2, 'new_http_connection') -# -# def tearDown(self): -# FLAGS.Reset() -# super(ApiObjectStoreTestCase, self).tearDown() -# -# def test_describe_instances(self): -# self.expect_http() -# self.mox.ReplayAll() -# -# self.assertEqual(self.ec2.get_all_instances(), []) + +class TestHTTPChannel(http.HTTPChannel): + # Otherwise we end up with an unclean reactor + def checkPersistence(self, _, __): + return False + + +class TestSite(server.Site): + protocol = TestHTTPChannel + + +class S3APITestCase(test.TrialTestCase): + def setUp(self): + super(S3APITestCase, self).setUp() + FLAGS.fake_users = True + FLAGS.buckets_path = os.path.join(oss_tempdir, 'buckets') + + shutil.rmtree(FLAGS.buckets_path) + os.mkdir(FLAGS.buckets_path) + + root = S3() + self.site = TestSite(root) + self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1') + self.tcp_port = self.listening_port.getHost().port + + + boto.config.set('Boto', 'num_retries', '0') + self.conn = S3Connection(aws_access_key_id='admin', + aws_secret_access_key='admin', + host='127.0.0.1', + port=self.tcp_port, + is_secure=False, + calling_format=OrdinaryCallingFormat()) + + # Don't attempt to reuse connections + def get_http_connection(host, is_secure): + return self.conn.new_http_connection(host, is_secure) + self.conn.get_http_connection = get_http_connection + + def _ensure_empty_list(self, l): + self.assertEquals(len(l), 0, "List was not empty") + return True + + def _ensure_only_bucket(self, l, name): + self.assertEquals(len(l), 1, "List didn't have exactly one element in it") + self.assertEquals(l[0].name, name, "Wrong name") + + def test_000_list_buckets(self): + d = threads.deferToThread(self.conn.get_all_buckets) + d.addCallback(self._ensure_empty_list) + return d + + def test_001_create_and_delete_bucket(self): + bucket_name = 'testbucket' + + d = threads.deferToThread(self.conn.create_bucket, bucket_name) + d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets)) + + def ensure_only_bucket(l, name): + self.assertEquals(len(l), 1, "List didn't have exactly one element in it") + self.assertEquals(l[0].name, name, "Wrong name") + d.addCallback(ensure_only_bucket, bucket_name) + + d.addCallback(lambda _:threads.deferToThread(self.conn.delete_bucket, bucket_name)) + d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets)) + d.addCallback(self._ensure_empty_list) + return d + + def test_002_create_bucket_and_key_and_delete_key_again(self): + bucket_name = 'testbucket' + key_name = 'somekey' + key_contents = 'somekey' + + d = threads.deferToThread(self.conn.create_bucket, bucket_name) + d.addCallback(lambda b:threads.deferToThread(b.new_key, key_name)) + d.addCallback(lambda k:threads.deferToThread(k.set_contents_from_string, key_contents)) + def ensure_key_contents(bucket_name, key_name, contents): + bucket = self.conn.get_bucket(bucket_name) + key = bucket.get_key(key_name) + self.assertEquals(key.get_contents_as_string(), contents, "Bad contents") + d.addCallback(lambda _:threads.deferToThread(ensure_key_contents, bucket_name, key_name, key_contents)) + def delete_key(bucket_name, key_name): + bucket = self.conn.get_bucket(bucket_name) + key = bucket.get_key(key_name) + key.delete() + d.addCallback(lambda _:threads.deferToThread(delete_key, bucket_name, key_name)) + d.addCallback(lambda _:threads.deferToThread(self.conn.get_bucket, bucket_name)) + d.addCallback(lambda b:threads.deferToThread(b.get_all_keys)) + d.addCallback(self._ensure_empty_list) + return d + + def tearDown(self): + super(S3APITestCase, self).tearDown() + return defer.DeferredList([defer.maybeDeferred(self.listening_port.stopListening)]) -- cgit From bed760dcc4dbdf1927c41cfff325cc56102ef962 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 26 Jul 2010 23:26:54 +0200 Subject: Make image downloads work again in S3 handler. Listing worked, but fetching the images failed because I wasn't clever enough to use twisted.web.static.File correctly. --- nova/objectstore/handler.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 6e76abb5a..4074f005d 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -121,7 +121,7 @@ class S3(Resource): if name == '': return self elif name == '_images': - return ImageResource() + return ImagesResource() else: return BucketResource(name) @@ -226,13 +226,21 @@ class ObjectResource(Resource): return '' class ImageResource(Resource): + isLeaf = True + + def __init__(self, name): + Resource.__init__(self) + self.img = image.Image(name) + + def render_GET(self, request): + return static.File(self.img.image_path, defaultType='application/octet-stream').render_GET(request) + +class ImagesResource(Resource): def getChild(self, name, request): if name == '': return self else: - request.setHeader("Content-Type", "application/octet-stream") - img = image.Image(name) - return static.File(img.image_path) + return ImageResource(name) def render_GET(self, request): """ returns a json listing of all images -- cgit From 69cbb1ac3559c44f6640939cd2e1db64e82073fe Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 26 Jul 2010 23:27:42 +0200 Subject: Make _fetch_s3_image pass proper AWS Authorization headers so that image downloads work again. --- nova/compute/node.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/nova/compute/node.py b/nova/compute/node.py index 7b86ca749..a44583c06 100644 --- a/nova/compute/node.py +++ b/nova/compute/node.py @@ -31,6 +31,7 @@ import logging import os import shutil import sys +import time from twisted.internet import defer from twisted.internet import task from twisted.application import service @@ -453,19 +454,25 @@ class Instance(object): def _fetch_s3_image(self, image, path): url = _image_url('%s/image' % image) + + # This should probably move somewhere else, like e.g. a download_as + # method on User objects and at the same time get rewritten to use + # twisted web client. headers = {} headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) user_id = self.datamodel['user_id'] user = UserManager.instance().get_user(user_id) - auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', url) - headers['Authorization'] = auth + uri = '/' + url.partition('/')[2] + auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', uri) + headers['Authorization'] = 'AWS %s:%s' % (user.access, auth) - headers_opt_string = ' '.join(['-H %s:%s' % (k,v) for (k,v) in headers.iteritems()]) - d = process.simple_execute('curl --silent %s ' - '%s -o "%s"' % (url, headers_opt_string, - path)) - return d + cmd = ['/usr/bin/curl', '--silent', url] + for (k,v) in headers.iteritems(): + cmd += ['-H', '%s: %s' % (k,v)] + + cmd += ['-o', path] + return process.SharedPool().execute(executable=cmd[0], args=cmd[1:]) def _fetch_local_image(self, image, path): source = _image_path('%s/image' % image) -- cgit From 9641dac5d0cbfd8d1f3026e1bae5749b9d8e00fc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 27 Jul 2010 10:30:00 +0200 Subject: Ensure that boto's config has a "Boto" section before attempting to set a value in it. --- nova/tests/objectstore_unittest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index 0a2f54031..20053a258 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -189,6 +189,8 @@ class S3APITestCase(test.TrialTestCase): self.tcp_port = self.listening_port.getHost().port + if not boto.config.has_section('Boto'): + boto.config.add_section('Boto') boto.config.set('Boto', 'num_retries', '0') self.conn = S3Connection(aws_access_key_id='admin', aws_secret_access_key='admin', -- cgit