Replace objectstore images with S3 image service backending to glance or local

3 files changed, 232 insertions, 81 deletions

diff --git a/nova/image/glance.py b/nova/image/glance.py
index 593c4bce6..7db94c0d4 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -21,6 +21,8 @@ import httplib
 import json
 import urlparse
 
+from glance.common import exception as glance_exception
+
 from nova import exception
 from nova import flags
 from nova import log as logging
@@ -57,27 +59,32 @@ class GlanceImageService(service.BaseImageService):
         """
         Returns a dict containing image data for the given opaque image id.
         """
-        image = self.client.get_image_meta(id)
-        if image:
-            return image
-        raise exception.NotFound
+        try:
+            image = self.client.get_image_meta(id)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        return image
 
-    def create(self, context, data):
+    def create(self, context, metadata, data=None):
         """
         Store the image data and return the new image id.
 
         :raises AlreadyExists if the image already exist.
 
         """
-        return self.client.add_image(image_meta=data)
+        return self.client.add_image(metadata, data)
 
-    def update(self, context, image_id, data):
+    def update(self, context, image_id, metadata, data=None):
         """Replace the contents of the given image with the new data.
 
         :raises NotFound if the image does not exist.
 
         """
-        return self.client.update_image(image_id, data)
+        try:
+            result = self.client.update_image(image_id, metadata, data)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        return result
 
     def delete(self, context, image_id):
         """
@@ -86,7 +93,11 @@ class GlanceImageService(service.BaseImageService):
         :raises NotFound if the image does not exist.
 
         """
-        return self.client.delete_image(image_id)
+        try:
+            result = self.client.delete_image(image_id)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        return result
 
     def delete_all(self):
         """
diff --git a/nova/image/s3.py b/nova/image/s3.py
index 14135a1ee..a740b010c 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -21,8 +21,12 @@ Proxy AMI-related calls from the cloud controller, to the running
 objectstore service.
 """
 
-import json
-import urllib
+import binascii
+import os
+import shutil
+import tarfile
+import tempfile
+from xml.etree import ElementTree
 
 import boto.s3.connection
 
@@ -31,84 +35,89 @@ from nova import flags
 from nova import utils
 from nova.auth import manager
 from nova.image import service
+from nova.api.ec2 import ec2utils
 
 
 FLAGS = flags.FLAGS
+flags.DEFINE_string('image_decryption_dir', '/tmp',
+                    'parent dir for tempdir used for image decryption')
 
 
-def map_s3_to_base(image):
-    """Convert from S3 format to format defined by BaseImageService."""
-    i = {}
-    i['id'] = image.get('imageId')
-    i['name'] = image.get('imageId')
-    i['kernel_id'] = image.get('kernelId')
-    i['ramdisk_id'] = image.get('ramdiskId')
-    i['location'] = image.get('imageLocation')
-    i['owner_id'] = image.get('imageOwnerId')
-    i['status'] = image.get('imageState')
-    i['type'] = image.get('type')
-    i['is_public'] = image.get('isPublic')
-    i['architecture'] = image.get('architecture')
-    return i
+_type_prefix_map = {'machine': 'ami',
+                    'kernel': 'aki',
+                    'ramdisk': 'ari'}
+
+
+def image_ec2_id(image_id, image_type):
+    prefix = _type_prefix_map[image_type]
+    template = prefix + '-%08x'
+    return ec2utils.id_to_ec2_id(int(image_id), template=template)
 
 
 class S3ImageService(service.BaseImageService):
+    def __init__(self, service=None, *args, **kwargs):
+        if service == None:
+            service = utils.import_object(FLAGS.image_service)
+        self.service = service
+        self.service.__init__(*args, **kwargs)
+
+    def create(self, context, properties, data=None):
+        """image should contain image_location"""
+        image_id, metadata = self._s3_create(context, properties)
+        return image_ec2_id(image_id, metadata['type'])
+
+    def delete(self, context, image_id):
+        # FIXME(vish): call to show is to check filter
+        self.show(context, image_id)
+        image_id = ec2utils.ec2_id_to_id(image_id)
+        self.service.delete(context, image_id)
 
-    def modify(self, context, image_id, operation):
-        self._conn(context).make_request(
-            method='POST',
-            bucket='_images',
-            query_args=self._qs({'image_id': image_id,
-                                 'operation': operation}))
-        return True
-
-    def update(self, context, image_id, attributes):
-        """update an image's attributes / info.json"""
-        attributes.update({"image_id": image_id})
-        self._conn(context).make_request(
-            method='POST',
-            bucket='_images',
-            query_args=self._qs(attributes))
-        return True
-
-    def register(self, context, image_location):
-        """ rpc call to register a new image based from a manifest """
-        image_id = utils.generate_uid('ami')
-        self._conn(context).make_request(
-            method='PUT',
-            bucket='_images',
-            query_args=self._qs({'image_location': image_location,
-                                 'image_id': image_id}))
-        return image_id
+    def update(self, context, image_id, metadata, data=None):
+        # FIXME(vish): call to show is to check filter
+        self.show(context, image_id)
+        image_id = ec2utils.ec2_id_to_id(image_id)
+        image = self.service.update(context, image_id, metadata, data)
+        image['id'] = image_ec2_id(image['id'], image['type'])
+        return image
 
     def index(self, context):
-        """Return a list of all images that a user can see."""
-        response = self._conn(context).make_request(
-            method='GET',
-            bucket='_images')
-        images = json.loads(response.read())
-        return [map_s3_to_base(i) for i in images]
+        images = self.service.index(context)
+        # FIXME(vish): index doesn't filter so we do it manually
+        return self._filter(context, images)
+
+    def detail(self, context):
+        images = self.service.detail(context)
+        # FIXME(vish): detail doesn't filter so we do it manually
+        return self._filter(context, images)
+
+    @staticmethod
+    def _is_visible(context, image):
+        return (context.is_admin
+                or context.project_id == image['properties']['owner_id']
+                or image['properties']['is_public'] == 'True')
+
+    @staticmethod
+    def _filter(context, images):
+        filtered = []
+        for image in images:
+            if not S3ImageService._is_visible(context, image):
+                continue
+            image['id'] = image_ec2_id(image['id'], image['type'])
+            filtered.append(image)
+        return filtered
 
     def show(self, context, image_id):
-        """return a image object if the context has permissions"""
-        if FLAGS.connection_type == 'fake':
-            return {'imageId': 'bar'}
-        result = self.index(context)
-        result = [i for i in result if i['id'] == image_id]
-        if not result:
-            raise exception.NotFound(_('Image %s could not be found')
-                                     % image_id)
-        image = result[0]
+        image_id = ec2utils.ec2_id_to_id(image_id)
+        image = self.service.show(context, image_id)
+        if not self._is_visible(context, image):
+            raise exception.NotFound
+        image['id'] = image_ec2_id(image['id'], image['type'])
         return image
 
-    def deregister(self, context, image_id):
-        """ unregister an image """
-        self._conn(context).make_request(
-             method='DELETE',
-             bucket='_images',
-             query_args=self._qs({'image_id': image_id}))
-
-    def _conn(self, context):
+    @staticmethod
+    def _conn(context):
+        # TODO(vish): is there a better way to get creds to sign
+        #             for the user?
         access = manager.AuthManager().get_access_key(context.user,
                                                       context.project)
         secret = str(context.user.secret)
@@ -120,8 +129,139 @@ class S3ImageService(service.BaseImageService):
                                                port=FLAGS.s3_port,
                                                host=FLAGS.s3_host)
 
-    def _qs(self, params):
-        pairs = []
-        for key in params.keys():
-            pairs.append(key + '=' + urllib.quote(params[key]))
-        return '&'.join(pairs)
+    @staticmethod
+    def _download_file(bucket, filename, local_dir):
+        key = bucket.get_key(filename)
+        local_filename = os.path.join(local_dir, filename)
+        key.get_contents_to_filename(local_filename)
+        return local_filename
+
+    def _s3_create(self, context, properties):
+        image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir)
+
+        image_location = properties['image_location']
+        bucket_name = image_location.split("/")[0]
+        manifest_path = image_location[len(bucket_name) + 1:]
+        bucket = self._conn(context).get_bucket(bucket_name)
+        key = bucket.get_key(manifest_path)
+        manifest = key.get_contents_as_string()
+
+        manifest = ElementTree.fromstring(manifest)
+        image_type = 'machine'
+
+        try:
+            kernel_id = manifest.find("machine_configuration/kernel_id").text
+            if kernel_id == 'true':
+                image_type = 'kernel'
+                kernel_id = None
+        except:
+            kernel_id = None
+
+        try:
+            ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text
+            if ramdisk_id == 'true':
+                image_type = 'ramdisk'
+                ramdisk_id = None
+        except:
+            ramdisk_id = None
+
+        try:
+            arch = manifest.find("machine_configuration/architecture").text
+        except:
+            arch = 'x86_64'
+
+        properties.update({'owner_id': context.project_id,
+                           'architecture': arch})
+
+        if kernel_id:
+            properties['kernel_id'] = kernel_id
+
+        if ramdisk_id:
+            properties['ramdisk_id'] = ramdisk_id
+
+        properties['is_public'] = False
+        metadata = {'type': image_type,
+                    'status': 'queued',
+                    'is_public': True,
+                    'properties': properties}
+        metadata['properties']['image_state'] = 'pending'
+        image = self.service.create(context, metadata)
+        image_id = image['id']
+
+        parts = []
+        for fn_element in manifest.find("image").getiterator("filename"):
+            part = self._download_file(bucket, fn_element.text, image_path)
+            parts.append(part)
+
+        # NOTE(vish): this may be suboptimal, should we use cat?
+        encrypted_filename = os.path.join(image_path, 'image.encrypted')
+        with open(encrypted_filename, 'w') as combined:
+            for filename in parts:
+                with open(filename) as part:
+                    shutil.copyfileobj(part, combined)
+
+        metadata['properties']['image_state'] = 'decrypting'
+        self.service.update(context, image_id, metadata)
+
+        hex_key = manifest.find("image/ec2_encrypted_key").text
+        encrypted_key = binascii.a2b_hex(hex_key)
+        hex_iv = manifest.find("image/ec2_encrypted_iv").text
+        encrypted_iv = binascii.a2b_hex(hex_iv)
+
+        # FIXME(vish): grab key from common service so this can run on
+        #              any host.
+        cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem")
+
+        decrypted_filename = os.path.join(image_path, 'image.tar.gz')
+        self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
+                            cloud_private_key, decrypted_filename)
+
+        metadata['properties']['image_state'] = 'untarring'
+        self.service.update(context, image_id, metadata)
+
+        unz_filename = self._untarzip_image(image_path, decrypted_filename)
+
+        metadata['properties']['image_state'] = 'uploading'
+        with open(unz_filename) as image_file:
+            self.service.update(context, image_id, metadata, image_file)
+        metadata['properties']['image_state'] = 'available'
+        self.service.update(context, image_id, metadata)
+
+        shutil.rmtree(image_path)
+        return image_id, metadata
+
+    @staticmethod
+    def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
+                      cloud_private_key, decrypted_filename):
+        key, err = utils.execute(
+                'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
+                process_input=encrypted_key,
+                check_exit_code=False)
+        if err:
+            raise exception.Error(_("Failed to decrypt private key: %s")
+                                  % err)
+        iv, err = utils.execute(
+                'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
+                process_input=encrypted_iv,
+                check_exit_code=False)
+        if err:
+            raise exception.Error(_("Failed to decrypt initialization "
+                                    "vector: %s") % err)
+
+        _out, err = utils.execute(
+                'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s'
+                 % (encrypted_filename, key, iv, decrypted_filename),
+                 check_exit_code=False)
+        if err:
+            raise exception.Error(_("Failed to decrypt image file "
+                                    "%(image_file)s: %(err)s") %
+                                    {'image_file': encrypted_filename,
+                                     'err': err})
+
+    @staticmethod
+    def _untarzip_image(path, filename):
+        tar_file = tarfile.open(filename, "r|gz")
+        tar_file.extractall(path)
+        image_file = tar_file.getnames()[0]
+        tar_file.close()
+        return os.path.join(path, image_file)
diff --git a/nova/image/service.py b/nova/image/service.py
index ebee2228d..e429955f4 100644
--- a/nova/image/service.py
+++ b/nova/image/service.py
@@ -76,7 +76,7 @@ class BaseImageService(object):
         """
         raise NotImplementedError
 
-    def create(self, context, data):
+    def create(self, context, metadata, data=None):
         """
         Store the image data and return the new image id.
 
@@ -85,7 +85,7 @@ class BaseImageService(object):
         """
         raise NotImplementedError
 
-    def update(self, context, image_id, data):
+    def update(self, context, image_id, metadata, data=None):
         """Replace the contents of the given image with the new data.
 
         :raises NotFound if the image does not exist.