Makes the objectstore require authorization, checks it properly, and makes nova-compute provide it when fetching images.

5 files changed, 185 insertions, 24 deletions

diff --git a/nova/auth/manager.py b/nova/auth/manager.py
index bc373fd26..b3b5d14ca 100644
--- a/nova/auth/manager.py
+++ b/nova/auth/manager.py
@@ -342,7 +342,7 @@ class AuthManager(object):
 
     def authenticate(self, access, signature, params, verb='GET',
                      server_string='127.0.0.1:8773', path='/',
-                     verify_signature=True):
+                     check_type='ec2', headers=None):
         """Authenticates AWS request using access key and signature
 
         If the project is not specified, attempts to authenticate to
@@ -367,8 +367,14 @@ class AuthManager(object):
         @type path: str
         @param path: Web request path.
 
-        @type verify_signature: bool
-        @param verify_signature: Whether to verify the signature.
+        @type check_type: str
+        @param check_type: Type of signature to check. 'ec2' for EC2, 's3' for
+                           S3. Any other value will cause signature not to be
+                           checked.
+
+        @type headers: list
+        @param headers: HTTP headers passed with the request (only needed for
+                        s3 signature checks)
 
         @rtype: tuple (User, Project)
         @return: User and project that the request represents.
@@ -376,7 +382,9 @@ class AuthManager(object):
         # TODO(vish): check for valid timestamp
         (access_key, sep, project_id) = access.partition(':')
 
+        logging.info('Looking up user: %r', access_key)
         user = self.get_user_from_access_key(access_key)
+        logging.info('user: %r', user)
         if user == None:
             raise exception.NotFound('No user found for access key %s' %
                                      access_key)
@@ -394,7 +402,14 @@ class AuthManager(object):
                                                                   project):
             raise exception.NotFound('User %s is not a member of project %s' %
                                      (user.id, project.id))
-        if verify_signature:
+        if check_type == 's3':
+            expected_signature = signer.Signer(user.secret.encode()).s3_authorization(headers, verb, path)
+            logging.debug('user.secret: %s', user.secret)
+            logging.debug('expected_signature: %s', expected_signature)
+            logging.debug('signature: %s', signature)
+            if signature != expected_signature:
+                raise exception.NotAuthorized('Signature does not match')
+        elif check_type == 'ec2':
             # NOTE(vish): hmac can't handle unicode, so encode ensures that
             #             secret isn't unicode
             expected_signature = signer.Signer(user.secret.encode()).generate(
diff --git a/nova/auth/signer.py b/nova/auth/signer.py
index 83831bfac..7d7471575 100644
--- a/nova/auth/signer.py
+++ b/nova/auth/signer.py
@@ -48,6 +48,7 @@ import hashlib
 import hmac
 import logging
 import urllib
+import boto.utils
 
 from nova.exception import Error
 
@@ -59,6 +60,13 @@ class Signer(object):
         if hashlib.sha256:
             self.hmac_256 = hmac.new(secret_key, digestmod=hashlib.sha256)
 
+    def s3_authorization(self, headers, verb, path):
+        c_string = boto.utils.canonical_string(verb, path, headers)
+        hmac = self.hmac.copy()
+        hmac.update(c_string)
+        b64_hmac = base64.encodestring(hmac.digest()).strip()
+        return b64_hmac
+
     def generate(self, params, verb, server_string, path):
         if params['SignatureVersion'] == '0':
             return self._calc_signature_0(params)
diff --git a/nova/compute/node.py b/nova/compute/node.py
index 7cae86d02..772373061 100644
--- a/nova/compute/node.py
+++ b/nova/compute/node.py
@@ -25,11 +25,13 @@ Compute Node:
 """
 
 import base64
+import boto.utils
 import json
 import logging
 import os
 import shutil
 import sys
+import time
 from twisted.internet import defer
 from twisted.internet import task
 from twisted.application import service
@@ -45,6 +47,7 @@ from nova import fakevirt
 from nova import flags
 from nova import process
 from nova import utils
+from nova.auth import signer, manager
 from nova.compute import disk
 from nova.compute import model
 from nova.compute import network
@@ -450,9 +453,25 @@ class Instance(object):
 
     def _fetch_s3_image(self, image, path):
         url = _image_url('%s/image' % image)
-        d = process.simple_execute(
-                'curl --silent %s -o %s' % (url, path))
-        return d
+
+        # This should probably move somewhere else, like e.g. a download_as
+        # method on User objects and at the same time get rewritten to use
+        # twisted web client.
+        headers = {}
+        headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
+
+        user_id = self.datamodel['user_id']
+        user = manager.AuthManager().get_user(user_id)
+        uri = '/' + url.partition('/')[2]
+        auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', uri)
+        headers['Authorization'] = 'AWS %s:%s' % (user.access, auth)
+
+        cmd = ['/usr/bin/curl', '--silent', url]
+        for (k,v) in headers.iteritems():
+            cmd += ['-H', '%s: %s' % (k,v)]
+
+        cmd += ['-o', path]
+        return process.SharedPool().execute(executable=cmd[0], args=cmd[1:])
 
     def _fetch_local_image(self, image, path):
         source = _image_path('%s/image' % image)
diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py
index 7d997390b..999581c65 100644
--- a/nova/objectstore/handler.py
+++ b/nova/objectstore/handler.py
@@ -47,7 +47,7 @@ import urllib
 
 from twisted.application import internet, service
 from twisted.web.resource import Resource
-from twisted.web import server, static
+from twisted.web import server, static, error
 
 
 from nova import exception
@@ -111,10 +111,10 @@ def get_context(request):
                                                              secret,
                                                              {},
                                                              request.method,
-                                                             request.host,
+                                                             request.getRequestHostname(),
                                                              request.uri,
-                                                             False)
-        # FIXME: check signature here!
+                                                             headers=request.getAllHeaders(),
+                                                             check_type='s3')
         return api.APIRequestContext(None, user, project)
     except exception.Error as ex:
         logging.debug("Authentication Failure: %s" % ex)
@@ -124,15 +124,15 @@ class S3(Resource):
     """Implementation of an S3-like storage server based on local files."""
     def getChild(self, name, request):
         request.context = get_context(request)
-
         if name == '':
             return self
         elif name == '_images':
-            return ImageResource()
+            return ImagesResource()
         else:
             return BucketResource(name)
 
     def render_GET(self, request):
+        logging.debug('List of buckets requested')
         buckets = [b for b in bucket.Bucket.all() if b.is_authorized(request.context)]
 
         render_xml(request, {"ListAllMyBucketsResult": {
@@ -154,7 +154,10 @@ class BucketResource(Resource):
     def render_GET(self, request):
         logging.debug("List keys for bucket %s" % (self.name))
 
-        bucket_object = bucket.Bucket(self.name)
+        try:
+            bucket_object = bucket.Bucket(self.name)
+        except exception.NotFound, e:
+            return error.NoResource(message="No such bucket").render(request)
 
         if not bucket_object.is_authorized(request.context):
             raise exception.NotAuthorized
@@ -170,13 +173,10 @@ class BucketResource(Resource):
 
     def render_PUT(self, request):
         logging.debug("Creating bucket %s" % (self.name))
-        try:
-            print 'user is %s' % request.context
-        except Exception as e:
-            logging.exception(e)
         logging.debug("calling bucket.Bucket.create(%r, %r)" % (self.name, request.context))
         bucket.Bucket.create(self.name, request.context)
-        return ''
+        request.finish()
+        return server.NOT_DONE_YET
 
     def render_DELETE(self, request):
         logging.debug("Deleting bucket %s" % (self.name))
@@ -234,13 +234,19 @@ class ObjectResource(Resource):
 class ImageResource(Resource):
     isLeaf = True
 
+    def __init__(self, name):
+        Resource.__init__(self)
+        self.img = image.Image(name)
+
+    def render_GET(self, request):
+        return static.File(self.img.image_path, defaultType='application/octet-stream').render_GET(request)
+
+class ImagesResource(Resource):
     def getChild(self, name, request):
         if name == '':
             return self
         else:
-            request.setHeader("Content-Type", "application/octet-stream")
-            img = image.Image(name)
-            return static.File(img.image_path)
+            return ImageResource(name)
 
     def render_GET(self, request):
         """ returns a json listing of all images
@@ -302,9 +308,13 @@ class ImageResource(Resource):
         request.setResponseCode(204)
         return ''
 
-def get_application():
+def get_site():
     root = S3()
-    factory = server.Site(root)
+    site = server.Site(root)
+    return site
+
+def get_application():
+    factory = get_site()
     application = service.Application("objectstore")
     objectStoreService = internet.TCPServer(FLAGS.s3_port, factory)
     objectStoreService.setServiceParent(application)
diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py
index 8c6d866cd..c90120a6e 100644
--- a/nova/tests/objectstore_unittest.py
+++ b/nova/tests/objectstore_unittest.py
@@ -16,6 +16,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import boto
 import glob
 import hashlib
 import logging
@@ -27,8 +28,12 @@ from nova import flags
 from nova import objectstore
 from nova import test
 from nova.auth import manager
+from nova.objectstore.handler import S3
 from nova.exception import NotEmpty, NotFound, NotAuthorized
 
+from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+from twisted.internet import reactor, threads, defer
+from twisted.web import http, server
 
 FLAGS = flags.FLAGS
 
@@ -156,3 +161,107 @@ class ObjectStoreTestCase(test.BaseTestCase):
         self.context.user = self.um.get_user('user2')
         self.context.project = self.um.get_project('proj2')
         self.assertFalse(my_img.is_authorized(self.context))
+
+
+class TestHTTPChannel(http.HTTPChannel):
+    # Otherwise we end up with an unclean reactor
+    def checkPersistence(self, _, __):
+        return False
+
+
+class TestSite(server.Site):
+    protocol = TestHTTPChannel
+
+
+class S3APITestCase(test.TrialTestCase):
+    def setUp(self):
+        super(S3APITestCase, self).setUp()
+
+        FLAGS.auth_driver='nova.auth.ldapdriver.FakeLdapDriver',
+        FLAGS.buckets_path = os.path.join(oss_tempdir, 'buckets')
+
+        self.um = manager.AuthManager()
+        self.admin_user = self.um.create_user('admin', admin=True)
+        self.admin_project = self.um.create_project('admin', self.admin_user)
+
+        shutil.rmtree(FLAGS.buckets_path)
+        os.mkdir(FLAGS.buckets_path)
+
+        root = S3()
+        self.site = TestSite(root)
+        self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1')
+        self.tcp_port = self.listening_port.getHost().port
+
+
+        if not boto.config.has_section('Boto'):
+            boto.config.add_section('Boto')
+        boto.config.set('Boto', 'num_retries', '0')
+        self.conn = S3Connection(aws_access_key_id=self.admin_user.access,
+                                 aws_secret_access_key=self.admin_user.secret,
+                                 host='127.0.0.1',
+                                 port=self.tcp_port,
+                                 is_secure=False,
+                                 calling_format=OrdinaryCallingFormat())
+
+        # Don't attempt to reuse connections
+        def get_http_connection(host, is_secure):
+            return self.conn.new_http_connection(host, is_secure)
+        self.conn.get_http_connection = get_http_connection
+
+    def _ensure_empty_list(self, l):
+        self.assertEquals(len(l), 0, "List was not empty")
+        return True
+
+    def _ensure_only_bucket(self, l, name):
+        self.assertEquals(len(l), 1, "List didn't have exactly one element in it")
+        self.assertEquals(l[0].name, name, "Wrong name")
+
+    def test_000_list_buckets(self):
+        d = threads.deferToThread(self.conn.get_all_buckets)
+        d.addCallback(self._ensure_empty_list)
+        return d
+
+    def test_001_create_and_delete_bucket(self):
+        bucket_name = 'testbucket'
+
+        d = threads.deferToThread(self.conn.create_bucket, bucket_name)
+        d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets))
+
+        def ensure_only_bucket(l, name):
+            self.assertEquals(len(l), 1, "List didn't have exactly one element in it")
+            self.assertEquals(l[0].name, name, "Wrong name")
+        d.addCallback(ensure_only_bucket, bucket_name)
+
+        d.addCallback(lambda _:threads.deferToThread(self.conn.delete_bucket, bucket_name))
+        d.addCallback(lambda _:threads.deferToThread(self.conn.get_all_buckets))
+        d.addCallback(self._ensure_empty_list)
+        return d
+
+    def test_002_create_bucket_and_key_and_delete_key_again(self):
+        bucket_name = 'testbucket'
+        key_name = 'somekey'
+        key_contents = 'somekey'
+
+        d = threads.deferToThread(self.conn.create_bucket, bucket_name)
+        d.addCallback(lambda b:threads.deferToThread(b.new_key, key_name))
+        d.addCallback(lambda k:threads.deferToThread(k.set_contents_from_string, key_contents))
+        def ensure_key_contents(bucket_name, key_name, contents):
+            bucket = self.conn.get_bucket(bucket_name)
+            key = bucket.get_key(key_name)
+            self.assertEquals(key.get_contents_as_string(), contents,  "Bad contents")
+        d.addCallback(lambda _:threads.deferToThread(ensure_key_contents, bucket_name, key_name, key_contents))
+        def delete_key(bucket_name, key_name):
+            bucket = self.conn.get_bucket(bucket_name)
+            key = bucket.get_key(key_name)
+            key.delete()
+        d.addCallback(lambda _:threads.deferToThread(delete_key, bucket_name, key_name))
+        d.addCallback(lambda _:threads.deferToThread(self.conn.get_bucket, bucket_name))
+        d.addCallback(lambda b:threads.deferToThread(b.get_all_keys))
+        d.addCallback(self._ensure_empty_list)
+        return d
+
+    def tearDown(self):
+        self.um.delete_user('admin')
+        self.um.delete_project('admin')
+        return defer.DeferredList([defer.maybeDeferred(self.listening_port.stopListening)])
+        super(S3APITestCase, self).tearDown()