port the objectstore tests to the new tests

4 files changed, 64 insertions, 222 deletions

diff --git a/nova/objectstore/bucket.py b/nova/objectstore/bucket.py
index b213e18e8..017b933a9 100644
--- a/nova/objectstore/bucket.py
+++ b/nova/objectstore/bucket.py
@@ -33,8 +33,7 @@ from nova.objectstore import stored
 
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('buckets_path', '$state_path/buckets',
-                    'path to s3 buckets')
+flags.DECLARE('buckets_path', 'nova.objectstore.s3server')
 
 
 class Bucket(object):
diff --git a/nova/objectstore/s3server.py b/nova/objectstore/s3server.py
index 7a7317af2..83ca5bfa3 100644
--- a/nova/objectstore/s3server.py
+++ b/nova/objectstore/s3server.py
@@ -68,9 +68,9 @@ class S3Application(wsgi.Router):
         if mapper is None:
             mapper = routes.Mapper()
 
-        mapper.connect('/', controller=RootHandler(self))
-                #controller=lambda *a, **kw: RootHandler(self)(*a, **kw))
-        mapper.connect('/{bucket_name}/{object_name}',
+        mapper.connect('/',
+                controller=lambda *a, **kw: RootHandler(self)(*a, **kw))
+        mapper.connect('/{bucket}/{object_name}',
                 controller=lambda *a, **kw: ObjectHandler(self)(*a, **kw))
         mapper.connect('/{bucket_name}/',
                 controller=lambda *a, **kw: BucketHandler(self)(*a, **kw))
@@ -87,7 +87,6 @@ class BaseRequestHandler(wsgi.Controller):
 
     @webob.dec.wsgify
     def __call__(self, request):
-        logging.debug('GOT HERE')
         method = request.method.lower()
         f = getattr(self, method, self.invalid)
         self.request = request
@@ -109,7 +108,7 @@ class BaseRequestHandler(wsgi.Controller):
     def finish(self, body=''):
         self.response.body = utils.utf8(body)
 
-    def invalid(self, request, **kwargs):
+    def invalid(self, **kwargs):
         pass
 
     def render_xml(self, value):
@@ -181,7 +180,8 @@ class BucketHandler(BaseRequestHandler):
         terse = int(self.get_argument("terse", 0))
         if not path.startswith(self.application.directory) or \
            not os.path.isdir(path):
-            raise webob.exc.HTTPError(404)
+            self.set_status(404)
+            return
         object_names = []
         for root, dirs, files in os.walk(path):
             for file_name in files:
@@ -231,7 +231,8 @@ class BucketHandler(BaseRequestHandler):
             self.application.directory, bucket_name))
         if not path.startswith(self.application.directory) or \
            os.path.exists(path):
-            raise webob.exc.HTTPError(403)
+            self.set_status(403)
+            return
         os.makedirs(path)
         self.finish()
 
@@ -240,9 +241,11 @@ class BucketHandler(BaseRequestHandler):
             self.application.directory, bucket_name))
         if not path.startswith(self.application.directory) or \
            not os.path.isdir(path):
-            raise webob.exc.HTTPError(404)
+            self.set_status(404)
+            return
         if len(os.listdir(path)) > 0:
-            raise webob.exc.HTTPError(403)
+            self.set_status(403)
+            return
         os.rmdir(path)
         self.set_status(204)
         self.finish()
@@ -254,7 +257,8 @@ class ObjectHandler(BaseRequestHandler):
         path = self._object_path(bucket, object_name)
         if not path.startswith(self.application.directory) or \
            not os.path.isfile(path):
-            raise webob.exc.HTTPError(404)
+            self.set_status(404)
+            return
         info = os.stat(path)
         self.set_header("Content-Type", "application/unknown")
         self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp(
@@ -271,16 +275,20 @@ class ObjectHandler(BaseRequestHandler):
             self.application.directory, bucket))
         if not bucket_dir.startswith(self.application.directory) or \
            not os.path.isdir(bucket_dir):
-            raise webob.exc.HTTPError(404)
+            self.set_status(404)
+            return
         path = self._object_path(bucket, object_name)
         if not path.startswith(bucket_dir) or os.path.isdir(path):
-            raise webob.exc.HTTPError(403)
+            self.set_status(403)
+            return
         directory = os.path.dirname(path)
         if not os.path.exists(directory):
             os.makedirs(directory)
         object_file = open(path, "w")
         object_file.write(self.request.body)
         object_file.close()
+        self.set_header('ETag',
+                        '"%s"' % hashlib.md5(self.request.body).hexdigest())
         self.finish()
 
     def delete(self, bucket, object_name):
@@ -288,7 +296,8 @@ class ObjectHandler(BaseRequestHandler):
         path = self._object_path(bucket, object_name)
         if not path.startswith(self.application.directory) or \
            not os.path.isfile(path):
-            raise webob.exc.HTTPError(404)
+            self.set_status(404)
+            return
         os.unlink(path)
         self.set_status(204)
         self.finish()
diff --git a/nova/test.py b/nova/test.py
index 5170775d0..3b608520a 100644
--- a/nova/test.py
+++ b/nova/test.py
@@ -33,7 +33,6 @@ import unittest
 import mox
 import shutil
 import stubout
-from eventlet import greenpool
 from eventlet import greenthread
 
 from nova import context
diff --git a/nova/tests/test_objectstore.py b/nova/tests/test_objectstore.py
index 4e2ac205e..c4d344503 100644
--- a/nova/tests/test_objectstore.py
+++ b/nova/tests/test_objectstore.py
@@ -27,18 +27,18 @@ import os
 import shutil
 import tempfile
 
-from boto.s3.connection import S3Connection, OrdinaryCallingFormat
-from twisted.internet import reactor, threads, defer
-from twisted.web import http, server
+from boto import exception as boto_exception
+from boto.s3 import connection as s3
 
 from nova import context
+from nova import exception
 from nova import flags
 from nova import objectstore
+from nova import wsgi
 from nova import test
 from nova.auth import manager
-from nova.exception import NotEmpty, NotFound
-from nova.objectstore import image
-from nova.objectstore.handler import S3
+#from nova.exception import NotEmpty, NotFound
+from nova.objectstore import s3server
 
 
 FLAGS = flags.FLAGS
@@ -53,151 +53,15 @@ os.makedirs(os.path.join(OSS_TEMPDIR, 'images'))
 os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets'))
 
 
-class ObjectStoreTestCase(test.TestCase):
-    """Test objectstore API directly."""
-
-    def setUp(self):
-        """Setup users and projects."""
-        super(ObjectStoreTestCase, self).setUp()
-        self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'),
-                   images_path=os.path.join(OSS_TEMPDIR, 'images'),
-                   ca_path=os.path.join(os.path.dirname(__file__), 'CA'))
-
-        self.auth_manager = manager.AuthManager()
-        self.auth_manager.create_user('user1')
-        self.auth_manager.create_user('user2')
-        self.auth_manager.create_user('admin_user', admin=True)
-        self.auth_manager.create_project('proj1', 'user1', 'a proj', ['user1'])
-        self.auth_manager.create_project('proj2', 'user2', 'a proj', ['user2'])
-        self.context = context.RequestContext('user1', 'proj1')
-
-    def tearDown(self):
-        """Tear down users and projects."""
-        self.auth_manager.delete_project('proj1')
-        self.auth_manager.delete_project('proj2')
-        self.auth_manager.delete_user('user1')
-        self.auth_manager.delete_user('user2')
-        self.auth_manager.delete_user('admin_user')
-        super(ObjectStoreTestCase, self).tearDown()
-
-    def test_buckets(self):
-        """Test the bucket API."""
-        objectstore.bucket.Bucket.create('new_bucket', self.context)
-        bucket = objectstore.bucket.Bucket('new_bucket')
-
-        # creator is authorized to use bucket
-        self.assert_(bucket.is_authorized(self.context))
-
-        # another user is not authorized
-        context2 = context.RequestContext('user2', 'proj2')
-        self.assertFalse(bucket.is_authorized(context2))
-
-        # admin is authorized to use bucket
-        admin_context = context.RequestContext('admin_user', None)
-        self.assertTrue(bucket.is_authorized(admin_context))
-
-        # new buckets are empty
-        self.assertTrue(bucket.list_keys()['Contents'] == [])
-
-        # storing keys works
-        bucket['foo'] = "bar"
-
-        self.assertEquals(len(bucket.list_keys()['Contents']), 1)
-
-        self.assertEquals(bucket['foo'].read(), 'bar')
-
-        # md5 of key works
-        self.assertEquals(bucket['foo'].md5, hashlib.md5('bar').hexdigest())
-
-        # deleting non-empty bucket should throw a NotEmpty exception
-        self.assertRaises(NotEmpty, bucket.delete)
-
-        # deleting key
-        del bucket['foo']
-
-        # deleting empty bucket
-        bucket.delete()
-
-        # accessing deleted bucket throws exception
-        self.assertRaises(NotFound, objectstore.bucket.Bucket, 'new_bucket')
-
-    def test_images(self):
-        self.do_test_images('1mb.manifest.xml', True,
-                            'image_bucket1', 'i-testing1')
-
-    def test_images_no_kernel_or_ramdisk(self):
-        self.do_test_images('1mb.no_kernel_or_ramdisk.manifest.xml',
-                            False, 'image_bucket2', 'i-testing2')
-
-    def do_test_images(self, manifest_file, expect_kernel_and_ramdisk,
-                             image_bucket, image_name):
-        "Test the image API."
-
-        # create a bucket for our bundle
-        objectstore.bucket.Bucket.create(image_bucket, self.context)
-        bucket = objectstore.bucket.Bucket(image_bucket)
-
-        # upload an image manifest/parts
-        bundle_path = os.path.join(os.path.dirname(__file__), 'bundle')
-        for path in glob.glob(bundle_path + '/*'):
-            bucket[os.path.basename(path)] = open(path, 'rb').read()
-
-        # register an image
-        image.Image.register_aws_image(image_name,
-                                       '%s/%s' % (image_bucket, manifest_file),
-                                       self.context)
-
-        # verify image
-        my_img = image.Image(image_name)
-        result_image_file = os.path.join(my_img.path, 'image')
-        self.assertEqual(os.stat(result_image_file).st_size, 1048576)
-
-        sha = hashlib.sha1(open(result_image_file).read()).hexdigest()
-        self.assertEqual(sha, '3b71f43ff30f4b15b5cd85dd9e95ebc7e84eb5a3')
-
-        if expect_kernel_and_ramdisk:
-            # Verify the default kernel and ramdisk are set
-            self.assertEqual(my_img.metadata['kernelId'], 'aki-test')
-            self.assertEqual(my_img.metadata['ramdiskId'], 'ari-test')
-        else:
-            # Verify that the default kernel and ramdisk (the one from FLAGS)
-            # doesn't get embedded in the metadata
-            self.assertFalse('kernelId' in my_img.metadata)
-            self.assertFalse('ramdiskId' in my_img.metadata)
-
-        # verify image permissions
-        context2 = context.RequestContext('user2', 'proj2')
-        self.assertFalse(my_img.is_authorized(context2))
-
-        # change user-editable fields
-        my_img.update_user_editable_fields({'display_name': 'my cool image'})
-        self.assertEqual('my cool image', my_img.metadata['displayName'])
-        my_img.update_user_editable_fields({'display_name': ''})
-        self.assert_(not my_img.metadata['displayName'])
-
-
-class TestHTTPChannel(http.HTTPChannel):
-    """Dummy site required for twisted.web"""
-
-    def checkPersistence(self, _, __):  # pylint: disable=C0103
-        """Otherwise we end up with an unclean reactor."""
-        return False
-
-
-class TestSite(server.Site):
-    """Dummy site required for twisted.web"""
-    protocol = TestHTTPChannel
-
-
 class S3APITestCase(test.TestCase):
     """Test objectstore through S3 API."""
 
     def setUp(self):
         """Setup users, projects, and start a test server."""
         super(S3APITestCase, self).setUp()
-
-        FLAGS.auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver'
-        FLAGS.buckets_path = os.path.join(OSS_TEMPDIR, 'buckets')
+        self.flags(auth_driver='nova.auth.ldapdriver.FakeLdapDriver',
+                   buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'),
+                   s3_host='127.0.0.1')
 
         self.auth_manager = manager.AuthManager()
         self.admin_user = self.auth_manager.create_user('admin', admin=True)
@@ -207,23 +71,20 @@ class S3APITestCase(test.TestCase):
         shutil.rmtree(FLAGS.buckets_path)
         os.mkdir(FLAGS.buckets_path)
 
-        root = S3()
-        self.site = TestSite(root)
-        # pylint: disable=E1101
-        self.listening_port = reactor.listenTCP(0, self.site,
-                                                interface='127.0.0.1')
-        # pylint: enable=E1101
-        self.tcp_port = self.listening_port.getHost().port
+        router = s3server.S3Application(FLAGS.buckets_path)
+        server = wsgi.Server()
+        server.start(router, FLAGS.s3_port, host=FLAGS.s3_host)
 
         if not boto.config.has_section('Boto'):
             boto.config.add_section('Boto')
         boto.config.set('Boto', 'num_retries', '0')
-        self.conn = S3Connection(aws_access_key_id=self.admin_user.access,
-                                 aws_secret_access_key=self.admin_user.secret,
-                                 host='127.0.0.1',
-                                 port=self.tcp_port,
-                                 is_secure=False,
-                                 calling_format=OrdinaryCallingFormat())
+        conn = s3.S3Connection(aws_access_key_id=self.admin_user.access,
+                               aws_secret_access_key=self.admin_user.secret,
+                               host=FLAGS.s3_host,
+                               port=FLAGS.s3_port,
+                               is_secure=False,
+                               calling_format=s3.OrdinaryCallingFormat())
+        self.conn = conn
 
         def get_http_connection(host, is_secure):
             """Get a new S3 connection, don't attempt to reuse connections."""
@@ -243,27 +104,16 @@ class S3APITestCase(test.TestCase):
 
     def test_000_list_buckets(self):
         """Make sure we are starting with no buckets."""
-        deferred = threads.deferToThread(self.conn.get_all_buckets)
-        deferred.addCallback(self._ensure_no_buckets)
-        return deferred
+        self._ensure_no_buckets(self.conn.get_all_buckets())
 
     def test_001_create_and_delete_bucket(self):
         """Test bucket creation and deletion."""
         bucket_name = 'testbucket'
 
-        deferred = threads.deferToThread(self.conn.create_bucket, bucket_name)
-        deferred.addCallback(lambda _:
-                             threads.deferToThread(self.conn.get_all_buckets))
-
-        deferred.addCallback(self._ensure_one_bucket, bucket_name)
-
-        deferred.addCallback(lambda _:
-                             threads.deferToThread(self.conn.delete_bucket,
-                                                   bucket_name))
-        deferred.addCallback(lambda _:
-                             threads.deferToThread(self.conn.get_all_buckets))
-        deferred.addCallback(self._ensure_no_buckets)
-        return deferred
+        self.conn.create_bucket(bucket_name)
+        self._ensure_one_bucket(self.conn.get_all_buckets(), bucket_name)
+        self.conn.delete_bucket(bucket_name)
+        self._ensure_no_buckets(self.conn.get_all_buckets())
 
     def test_002_create_bucket_and_key_and_delete_key_again(self):
         """Test key operations on buckets."""
@@ -271,45 +121,30 @@ class S3APITestCase(test.TestCase):
         key_name = 'somekey'
         key_contents = 'somekey'
 
-        deferred = threads.deferToThread(self.conn.create_bucket, bucket_name)
-        deferred.addCallback(lambda b:
-                             threads.deferToThread(b.new_key, key_name))
-        deferred.addCallback(lambda k:
-                             threads.deferToThread(k.set_contents_from_string,
-                                                   key_contents))
+        b = self.conn.create_bucket(bucket_name)
+        k = b.new_key(key_name)
+        k.set_contents_from_string(key_contents)
+
+        bucket = self.conn.get_bucket(bucket_name)
 
-        def ensure_key_contents(bucket_name, key_name, contents):
-            """Verify contents for a key in the given bucket."""
-            bucket = self.conn.get_bucket(bucket_name)
-            key = bucket.get_key(key_name)
-            self.assertEquals(key.get_contents_as_string(), contents,
-                              "Bad contents")
+        # make sure the contents are correct
+        key = bucket.get_key(key_name)
+        self.assertEquals(key.get_contents_as_string(), key_contents,
+                          "Bad contents")
 
-        deferred.addCallback(lambda _:
-                             threads.deferToThread(ensure_key_contents,
-                                                   bucket_name, key_name,
-                                                   key_contents))
+        # delete the key
+        key.delete()
 
-        def delete_key(bucket_name, key_name):
-            """Delete a key for the given bucket."""
-            bucket = self.conn.get_bucket(bucket_name)
-            key = bucket.get_key(key_name)
-            key.delete()
+        self._ensure_no_buckets(bucket.get_all_keys())
 
-        deferred.addCallback(lambda _:
-                             threads.deferToThread(delete_key, bucket_name,
-                                                   key_name))
-        deferred.addCallback(lambda _:
-                             threads.deferToThread(self.conn.get_bucket,
-                                                   bucket_name))
-        deferred.addCallback(lambda b: threads.deferToThread(b.get_all_keys))
-        deferred.addCallback(self._ensure_no_buckets)
-        return deferred
+    def test_unknown_bucket(self):
+        bucket_name = 'falalala'
+        self.assertRaises(boto_exception.S3ResponseError,
+                          self.conn.get_bucket,
+                          bucket_name)
 
     def tearDown(self):
         """Tear down auth and test server."""
         self.auth_manager.delete_user('admin')
         self.auth_manager.delete_project('admin')
-        stop_listening = defer.maybeDeferred(self.listening_port.stopListening)
         super(S3APITestCase, self).tearDown()
-        return defer.DeferredList([stop_listening])