From a433ddeda77aaa4462694661ecdca71eed6db669 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Fri, 4 Mar 2011 02:36:55 +0000
Subject: Replace objectstore images with S3 image service backending to glance
 or local

---
 nova/image/glance.py  |  29 ++++--
 nova/image/s3.py      | 280 +++++++++++++++++++++++++++++++++++++-------------
 nova/image/service.py |   4 +-
 3 files changed, 232 insertions(+), 81 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/glance.py b/nova/image/glance.py
index 593c4bce6..7db94c0d4 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -21,6 +21,8 @@ import httplib
 import json
 import urlparse
 
+from glance.common import exception as glance_exception
+
 from nova import exception
 from nova import flags
 from nova import log as logging
@@ -57,27 +59,32 @@ class GlanceImageService(service.BaseImageService):
         """
         Returns a dict containing image data for the given opaque image id.
         """
-        image = self.client.get_image_meta(id)
-        if image:
-            return image
-        raise exception.NotFound
+        try:
+            image = self.client.get_image_meta(id)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        return image
 
-    def create(self, context, data):
+    def create(self, context, metadata, data=None):
         """
         Store the image data and return the new image id.
 
         :raises AlreadyExists if the image already exist.
 
         """
-        return self.client.add_image(image_meta=data)
+        return self.client.add_image(metadata, data)
 
-    def update(self, context, image_id, data):
+    def update(self, context, image_id, metadata, data=None):
         """Replace the contents of the given image with the new data.
 
         :raises NotFound if the image does not exist.
 
         """
-        return self.client.update_image(image_id, data)
+        try:
+            result = self.client.update_image(image_id, metadata, data)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        return result
 
     def delete(self, context, image_id):
         """
@@ -86,7 +93,11 @@ class GlanceImageService(service.BaseImageService):
         :raises NotFound if the image does not exist.
 
         """
-        return self.client.delete_image(image_id)
+        try:
+            result = self.client.delete_image(image_id)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        return result
 
     def delete_all(self):
         """
diff --git a/nova/image/s3.py b/nova/image/s3.py
index 14135a1ee..a740b010c 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -21,8 +21,12 @@ Proxy AMI-related calls from the cloud controller, to the running
 objectstore service.
 """
 
-import json
-import urllib
+import binascii
+import os
+import shutil
+import tarfile
+import tempfile
+from xml.etree import ElementTree
 
 import boto.s3.connection
 
@@ -31,84 +35,89 @@ from nova import flags
 from nova import utils
 from nova.auth import manager
 from nova.image import service
+from nova.api.ec2 import ec2utils
 
 
 FLAGS = flags.FLAGS
+flags.DEFINE_string('image_decryption_dir', '/tmp',
+                    'parent dir for tempdir used for image decryption')
 
 
-def map_s3_to_base(image):
-    """Convert from S3 format to format defined by BaseImageService."""
-    i = {}
-    i['id'] = image.get('imageId')
-    i['name'] = image.get('imageId')
-    i['kernel_id'] = image.get('kernelId')
-    i['ramdisk_id'] = image.get('ramdiskId')
-    i['location'] = image.get('imageLocation')
-    i['owner_id'] = image.get('imageOwnerId')
-    i['status'] = image.get('imageState')
-    i['type'] = image.get('type')
-    i['is_public'] = image.get('isPublic')
-    i['architecture'] = image.get('architecture')
-    return i
+_type_prefix_map = {'machine': 'ami',
+                    'kernel': 'aki',
+                    'ramdisk': 'ari'}
+
+
+def image_ec2_id(image_id, image_type):
+    prefix = _type_prefix_map[image_type]
+    template = prefix + '-%08x'
+    return ec2utils.id_to_ec2_id(int(image_id), template=template)
 
 
 class S3ImageService(service.BaseImageService):
+    def __init__(self, service=None, *args, **kwargs):
+        if service == None:
+            service = utils.import_object(FLAGS.image_service)
+        self.service = service
+        self.service.__init__(*args, **kwargs)
+
+    def create(self, context, properties, data=None):
+        """image should contain image_location"""
+        image_id, metadata = self._s3_create(context, properties)
+        return image_ec2_id(image_id, metadata['type'])
+
+    def delete(self, context, image_id):
+        # FIXME(vish): call to show is to check filter
+        self.show(context, image_id)
+        image_id = ec2utils.ec2_id_to_id(image_id)
+        self.service.delete(context, image_id)
 
-    def modify(self, context, image_id, operation):
-        self._conn(context).make_request(
-            method='POST',
-            bucket='_images',
-            query_args=self._qs({'image_id': image_id,
-                                 'operation': operation}))
-        return True
-
-    def update(self, context, image_id, attributes):
-        """update an image's attributes / info.json"""
-        attributes.update({"image_id": image_id})
-        self._conn(context).make_request(
-            method='POST',
-            bucket='_images',
-            query_args=self._qs(attributes))
-        return True
-
-    def register(self, context, image_location):
-        """ rpc call to register a new image based from a manifest """
-        image_id = utils.generate_uid('ami')
-        self._conn(context).make_request(
-            method='PUT',
-            bucket='_images',
-            query_args=self._qs({'image_location': image_location,
-                                 'image_id': image_id}))
-        return image_id
+    def update(self, context, image_id, metadata, data=None):
+        # FIXME(vish): call to show is to check filter
+        self.show(context, image_id)
+        image_id = ec2utils.ec2_id_to_id(image_id)
+        image = self.service.update(context, image_id, metadata, data)
+        image['id'] = image_ec2_id(image['id'], image['type'])
+        return image
 
     def index(self, context):
-        """Return a list of all images that a user can see."""
-        response = self._conn(context).make_request(
-            method='GET',
-            bucket='_images')
-        images = json.loads(response.read())
-        return [map_s3_to_base(i) for i in images]
+        images = self.service.index(context)
+        # FIXME(vish): index doesn't filter so we do it manually
+        return self._filter(context, images)
+
+    def detail(self, context):
+        images = self.service.detail(context)
+        # FIXME(vish): detail doesn't filter so we do it manually
+        return self._filter(context, images)
+
+    @staticmethod
+    def _is_visible(context, image):
+        return (context.is_admin
+                or context.project_id == image['properties']['owner_id']
+                or image['properties']['is_public'] == 'True')
+
+    @staticmethod
+    def _filter(context, images):
+        filtered = []
+        for image in images:
+            if not S3ImageService._is_visible(context, image):
+                continue
+            image['id'] = image_ec2_id(image['id'], image['type'])
+            filtered.append(image)
+        return filtered
 
     def show(self, context, image_id):
-        """return a image object if the context has permissions"""
-        if FLAGS.connection_type == 'fake':
-            return {'imageId': 'bar'}
-        result = self.index(context)
-        result = [i for i in result if i['id'] == image_id]
-        if not result:
-            raise exception.NotFound(_('Image %s could not be found')
-                                     % image_id)
-        image = result[0]
+        image_id = ec2utils.ec2_id_to_id(image_id)
+        image = self.service.show(context, image_id)
+        if not self._is_visible(context, image):
+            raise exception.NotFound
+        image['id'] = image_ec2_id(image['id'], image['type'])
         return image
 
-    def deregister(self, context, image_id):
-        """ unregister an image """
-        self._conn(context).make_request(
-             method='DELETE',
-             bucket='_images',
-             query_args=self._qs({'image_id': image_id}))
-
-    def _conn(self, context):
+    @staticmethod
+    def _conn(context):
+        # TODO(vish): is there a better way to get creds to sign
+        #             for the user?
         access = manager.AuthManager().get_access_key(context.user,
                                                       context.project)
         secret = str(context.user.secret)
@@ -120,8 +129,139 @@ class S3ImageService(service.BaseImageService):
                                                port=FLAGS.s3_port,
                                                host=FLAGS.s3_host)
 
-    def _qs(self, params):
-        pairs = []
-        for key in params.keys():
-            pairs.append(key + '=' + urllib.quote(params[key]))
-        return '&'.join(pairs)
+    @staticmethod
+    def _download_file(bucket, filename, local_dir):
+        key = bucket.get_key(filename)
+        local_filename = os.path.join(local_dir, filename)
+        key.get_contents_to_filename(local_filename)
+        return local_filename
+
+    def _s3_create(self, context, properties):
+        image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir)
+
+        image_location = properties['image_location']
+        bucket_name = image_location.split("/")[0]
+        manifest_path = image_location[len(bucket_name) + 1:]
+        bucket = self._conn(context).get_bucket(bucket_name)
+        key = bucket.get_key(manifest_path)
+        manifest = key.get_contents_as_string()
+
+        manifest = ElementTree.fromstring(manifest)
+        image_type = 'machine'
+
+        try:
+            kernel_id = manifest.find("machine_configuration/kernel_id").text
+            if kernel_id == 'true':
+                image_type = 'kernel'
+                kernel_id = None
+        except:
+            kernel_id = None
+
+        try:
+            ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text
+            if ramdisk_id == 'true':
+                image_type = 'ramdisk'
+                ramdisk_id = None
+        except:
+            ramdisk_id = None
+
+        try:
+            arch = manifest.find("machine_configuration/architecture").text
+        except:
+            arch = 'x86_64'
+
+        properties.update({'owner_id': context.project_id,
+                           'architecture': arch})
+
+        if kernel_id:
+            properties['kernel_id'] = kernel_id
+
+        if ramdisk_id:
+            properties['ramdisk_id'] = ramdisk_id
+
+        properties['is_public'] = False
+        metadata = {'type': image_type,
+                    'status': 'queued',
+                    'is_public': True,
+                    'properties': properties}
+        metadata['properties']['image_state'] = 'pending'
+        image = self.service.create(context, metadata)
+        image_id = image['id']
+
+        parts = []
+        for fn_element in manifest.find("image").getiterator("filename"):
+            part = self._download_file(bucket, fn_element.text, image_path)
+            parts.append(part)
+
+        # NOTE(vish): this may be suboptimal, should we use cat?
+        encrypted_filename = os.path.join(image_path, 'image.encrypted')
+        with open(encrypted_filename, 'w') as combined:
+            for filename in parts:
+                with open(filename) as part:
+                    shutil.copyfileobj(part, combined)
+
+        metadata['properties']['image_state'] = 'decrypting'
+        self.service.update(context, image_id, metadata)
+
+        hex_key = manifest.find("image/ec2_encrypted_key").text
+        encrypted_key = binascii.a2b_hex(hex_key)
+        hex_iv = manifest.find("image/ec2_encrypted_iv").text
+        encrypted_iv = binascii.a2b_hex(hex_iv)
+
+        # FIXME(vish): grab key from common service so this can run on
+        #              any host.
+        cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem")
+
+        decrypted_filename = os.path.join(image_path, 'image.tar.gz')
+        self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
+                            cloud_private_key, decrypted_filename)
+
+        metadata['properties']['image_state'] = 'untarring'
+        self.service.update(context, image_id, metadata)
+
+        unz_filename = self._untarzip_image(image_path, decrypted_filename)
+
+        metadata['properties']['image_state'] = 'uploading'
+        with open(unz_filename) as image_file:
+            self.service.update(context, image_id, metadata, image_file)
+        metadata['properties']['image_state'] = 'available'
+        self.service.update(context, image_id, metadata)
+
+        shutil.rmtree(image_path)
+        return image_id, metadata
+
+    @staticmethod
+    def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
+                      cloud_private_key, decrypted_filename):
+        key, err = utils.execute(
+                'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
+                process_input=encrypted_key,
+                check_exit_code=False)
+        if err:
+            raise exception.Error(_("Failed to decrypt private key: %s")
+                                  % err)
+        iv, err = utils.execute(
+                'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
+                process_input=encrypted_iv,
+                check_exit_code=False)
+        if err:
+            raise exception.Error(_("Failed to decrypt initialization "
+                                    "vector: %s") % err)
+
+        _out, err = utils.execute(
+                'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s'
+                 % (encrypted_filename, key, iv, decrypted_filename),
+                 check_exit_code=False)
+        if err:
+            raise exception.Error(_("Failed to decrypt image file "
+                                    "%(image_file)s: %(err)s") %
+                                    {'image_file': encrypted_filename,
+                                     'err': err})
+
+    @staticmethod
+    def _untarzip_image(path, filename):
+        tar_file = tarfile.open(filename, "r|gz")
+        tar_file.extractall(path)
+        image_file = tar_file.getnames()[0]
+        tar_file.close()
+        return os.path.join(path, image_file)
diff --git a/nova/image/service.py b/nova/image/service.py
index ebee2228d..e429955f4 100644
--- a/nova/image/service.py
+++ b/nova/image/service.py
@@ -76,7 +76,7 @@ class BaseImageService(object):
         """
         raise NotImplementedError
 
-    def create(self, context, data):
+    def create(self, context, metadata, data=None):
         """
         Store the image data and return the new image id.
 
@@ -85,7 +85,7 @@ class BaseImageService(object):
         """
         raise NotImplementedError
 
-    def update(self, context, image_id, data):
+    def update(self, context, image_id, metadata, data=None):
         """Replace the contents of the given image with the new data.
 
         :raises NotFound if the image does not exist.
-- 
cgit 


From 13307e02258a5a08bedb1ed933a107668aac6457 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Fri, 4 Mar 2011 05:04:49 +0000
Subject: make local image service work

---
 nova/image/local.py | 64 ++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 44 insertions(+), 20 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/local.py b/nova/image/local.py
index f78b9aa89..b4616729a 100644
--- a/nova/image/local.py
+++ b/nova/image/local.py
@@ -15,57 +15,81 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import cPickle as pickle
+import json
 import os.path
 import random
-import tempfile
+import shutil
 
+from nova import flags
 from nova import exception
 from nova.image import service
 
 
-class LocalImageService(service.BaseImageService):
+FLAGS = flags.FLAGS
+flags.DEFINE_string('images_path', '$state_path/images',
+                    'path to decrypted images')
 
+class LocalImageService(service.BaseImageService):
     """Image service storing images to local disk.
+
     It assumes that image_ids are integers.
 
     """
 
     def __init__(self):
-        self._path = tempfile.mkdtemp()
+        self._path = FLAGS.images_path
 
-    def _path_to(self, image_id):
+    def _path_to(self, image_id, fname='info.json'):
+        if fname:
+            return os.path.join(self._path, str(image_id), fname)
         return os.path.join(self._path, str(image_id))
 
     def _ids(self):
         """The list of all image ids."""
-        return [int(i) for i in os.listdir(self._path)]
+        return [int(i) for i in os.listdir(self._path)
+                if unicode(i).isnumeric()]
 
     def index(self, context):
-        return [dict(id=i['id'], name=i['name']) for i in self.detail(context)]
+        return [dict(image_id=i['id'], name=i.get('name'))
+                for i in self.detail(context)]
 
     def detail(self, context):
-        return [self.show(context, id) for id in self._ids()]
-
-    def show(self, context, id):
+        images = []
+        for image_id in self._ids():
+            try:
+                image = self.show(context, image_id)
+                images.append(image)
+            except exception.NotFound:
+                continue
+        return images
+
+    def show(self, context, image_id):
         try:
-            return pickle.load(open(self._path_to(id)))
+            with open(self._path_to(image_id)) as metadata_file:
+                return json.load(metadata_file)
         except IOError:
             raise exception.NotFound
 
-    def create(self, context, data):
+    def create(self, context, metadata, data=None):
         """Store the image data and return the new image id."""
-        id = random.randint(0, 2 ** 31 - 1)
-        data['id'] = id
-        self.update(context, id, data)
-        return id
+        image_id = random.randint(0, 2 ** 31 - 1)
+        image_path = self._path_to(image_id, None)
+        if not os.path.exists(image_path):
+            os.mkdir(image_path)
+        return  self.update(context, image_id, metadata, data)
 
-    def update(self, context, image_id, data):
+    def update(self, context, image_id, metadata, data=None):
         """Replace the contents of the given image with the new data."""
+        metadata['id'] = image_id
         try:
-            pickle.dump(data, open(self._path_to(image_id), 'w'))
+            with open(self._path_to(image_id), 'w') as metadata_file:
+                json.dump(metadata, metadata_file)
+            if data:
+                with open(self._path_to(image_id, 'image'), 'w') as image_file:
+                    shutil.copyfileobj(data, image_file)
         except IOError:
             raise exception.NotFound
+        return metadata
 
     def delete(self, context, image_id):
         """Delete the given image.
@@ -79,8 +103,8 @@ class LocalImageService(service.BaseImageService):
 
     def delete_all(self):
         """Clears out all images in local directory."""
-        for id in self._ids():
-            os.unlink(self._path_to(id))
+        for image_id in self._ids():
+            os.unlink(self._path_to(image_id))
 
     def delete_imagedir(self):
         """Deletes the local directory.
-- 
cgit 


From cf9bc248f0fc318c4a9fb5087f257216312e39d1 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Fri, 4 Mar 2011 05:21:28 +0000
Subject: fix a couple issues with local, update the glance fake to actually
 return the same types as the real client, fix the image tests

---
 nova/image/local.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/local.py b/nova/image/local.py
index b4616729a..6fa648b6b 100644
--- a/nova/image/local.py
+++ b/nova/image/local.py
@@ -29,6 +29,7 @@ FLAGS = flags.FLAGS
 flags.DEFINE_string('images_path', '$state_path/images',
                     'path to decrypted images')
 
+
 class LocalImageService(service.BaseImageService):
     """Image service storing images to local disk.
 
@@ -97,18 +98,11 @@ class LocalImageService(service.BaseImageService):
 
         """
         try:
-            os.unlink(self._path_to(image_id))
+            shutil.rmtree(self._path_to(image_id, None))
         except IOError:
             raise exception.NotFound
 
     def delete_all(self):
         """Clears out all images in local directory."""
         for image_id in self._ids():
-            os.unlink(self._path_to(image_id))
-
-    def delete_imagedir(self):
-        """Deletes the local directory.
-        Raises OSError if directory is not empty.
-
-        """
-        os.rmdir(self._path)
+            shutil.rmtree(self._path_to(image_id, None))
-- 
cgit 


From e2c95e198f1982bc50bc95bc61ef3211b17937a2 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Fri, 4 Mar 2011 05:55:41 +0000
Subject: spawn a greenthread for image registration because it is slow

---
 nova/image/s3.py | 67 ++++++++++++++++++++++++++++++--------------------------
 1 file changed, 36 insertions(+), 31 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/s3.py b/nova/image/s3.py
index a740b010c..e9542c7bd 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -22,6 +22,7 @@ objectstore service.
 """
 
 import binascii
+import eventlet
 import os
 import shutil
 import tarfile
@@ -188,46 +189,50 @@ class S3ImageService(service.BaseImageService):
         image = self.service.create(context, metadata)
         image_id = image['id']
 
-        parts = []
-        for fn_element in manifest.find("image").getiterator("filename"):
-            part = self._download_file(bucket, fn_element.text, image_path)
-            parts.append(part)
+        def delayed_create():
+            parts = []
+            for fn_element in manifest.find("image").getiterator("filename"):
+                part = self._download_file(bucket, fn_element.text, image_path)
+                parts.append(part)
 
-        # NOTE(vish): this may be suboptimal, should we use cat?
-        encrypted_filename = os.path.join(image_path, 'image.encrypted')
-        with open(encrypted_filename, 'w') as combined:
-            for filename in parts:
-                with open(filename) as part:
-                    shutil.copyfileobj(part, combined)
+            # NOTE(vish): this may be suboptimal, should we use cat?
+            encrypted_filename = os.path.join(image_path, 'image.encrypted')
+            with open(encrypted_filename, 'w') as combined:
+                for filename in parts:
+                    with open(filename) as part:
+                        shutil.copyfileobj(part, combined)
 
-        metadata['properties']['image_state'] = 'decrypting'
-        self.service.update(context, image_id, metadata)
+            metadata['properties']['image_state'] = 'decrypting'
+            self.service.update(context, image_id, metadata)
 
-        hex_key = manifest.find("image/ec2_encrypted_key").text
-        encrypted_key = binascii.a2b_hex(hex_key)
-        hex_iv = manifest.find("image/ec2_encrypted_iv").text
-        encrypted_iv = binascii.a2b_hex(hex_iv)
+            hex_key = manifest.find("image/ec2_encrypted_key").text
+            encrypted_key = binascii.a2b_hex(hex_key)
+            hex_iv = manifest.find("image/ec2_encrypted_iv").text
+            encrypted_iv = binascii.a2b_hex(hex_iv)
 
-        # FIXME(vish): grab key from common service so this can run on
-        #              any host.
-        cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem")
+            # FIXME(vish): grab key from common service so this can run on
+            #              any host.
+            cloud_pk = os.path.join(FLAGS.ca_path, "private/cakey.pem")
 
-        decrypted_filename = os.path.join(image_path, 'image.tar.gz')
-        self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
-                            cloud_private_key, decrypted_filename)
+            decrypted_filename = os.path.join(image_path, 'image.tar.gz')
+            self._decrypt_image(encrypted_filename, encrypted_key,
+                                encrypted_iv, cloud_pk, decrypted_filename)
 
-        metadata['properties']['image_state'] = 'untarring'
-        self.service.update(context, image_id, metadata)
+            metadata['properties']['image_state'] = 'untarring'
+            self.service.update(context, image_id, metadata)
 
-        unz_filename = self._untarzip_image(image_path, decrypted_filename)
+            unz_filename = self._untarzip_image(image_path, decrypted_filename)
 
-        metadata['properties']['image_state'] = 'uploading'
-        with open(unz_filename) as image_file:
-            self.service.update(context, image_id, metadata, image_file)
-        metadata['properties']['image_state'] = 'available'
-        self.service.update(context, image_id, metadata)
+            metadata['properties']['image_state'] = 'uploading'
+            with open(unz_filename) as image_file:
+                self.service.update(context, image_id, metadata, image_file)
+            metadata['properties']['image_state'] = 'available'
+            self.service.update(context, image_id, metadata)
+
+            shutil.rmtree(image_path)
+
+        eventlet.spawn_n(delayed_create)
 
-        shutil.rmtree(image_path)
         return image_id, metadata
 
     @staticmethod
-- 
cgit 


From f3c1c99ca0f6f3164430b33f46772ef8bdc87b70 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Fri, 4 Mar 2011 19:54:19 +0000
Subject: move the id wrapping into cloud layer instead of image_service

---
 nova/image/s3.py | 28 +++++-----------------------
 1 file changed, 5 insertions(+), 23 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/s3.py b/nova/image/s3.py
index e9542c7bd..c7446f4b0 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -36,7 +36,6 @@ from nova import flags
 from nova import utils
 from nova.auth import manager
 from nova.image import service
-from nova.api.ec2 import ec2utils
 
 
 FLAGS = flags.FLAGS
@@ -44,17 +43,6 @@ flags.DEFINE_string('image_decryption_dir', '/tmp',
                     'parent dir for tempdir used for image decryption')
 
 
-_type_prefix_map = {'machine': 'ami',
-                    'kernel': 'aki',
-                    'ramdisk': 'ari'}
-
-
-def image_ec2_id(image_id, image_type):
-    prefix = _type_prefix_map[image_type]
-    template = prefix + '-%08x'
-    return ec2utils.id_to_ec2_id(int(image_id), template=template)
-
-
 class S3ImageService(service.BaseImageService):
     def __init__(self, service=None, *args, **kwargs):
         if service == None:
@@ -62,23 +50,20 @@ class S3ImageService(service.BaseImageService):
         self.service = service
         self.service.__init__(*args, **kwargs)
 
-    def create(self, context, properties, data=None):
-        """image should contain image_location"""
-        image_id, metadata = self._s3_create(context, properties)
-        return image_ec2_id(image_id, metadata['type'])
+    def create(self, context, metadata, data=None):
+        """metadata should contain image_location"""
+        image = self._s3_create(context, metadata)
+        return image
 
     def delete(self, context, image_id):
         # FIXME(vish): call to show is to check filter
         self.show(context, image_id)
-        image_id = ec2utils.ec2_id_to_id(image_id)
         self.service.delete(context, image_id)
 
     def update(self, context, image_id, metadata, data=None):
         # FIXME(vish): call to show is to check filter
         self.show(context, image_id)
-        image_id = ec2utils.ec2_id_to_id(image_id)
         image = self.service.update(context, image_id, metadata, data)
-        image['id'] = image_ec2_id(image['id'], image['type'])
         return image
 
     def index(self, context):
@@ -103,16 +88,13 @@ class S3ImageService(service.BaseImageService):
         for image in images:
             if not S3ImageService._is_visible(context, image):
                 continue
-            image['id'] = image_ec2_id(image['id'], image['type'])
             filtered.append(image)
         return filtered
 
     def show(self, context, image_id):
-        image_id = ec2utils.ec2_id_to_id(image_id)
         image = self.service.show(context, image_id)
         if not self._is_visible(context, image):
             raise exception.NotFound
-        image['id'] = image_ec2_id(image['id'], image['type'])
         return image
 
     @staticmethod
@@ -233,7 +215,7 @@ class S3ImageService(service.BaseImageService):
 
         eventlet.spawn_n(delayed_create)
 
-        return image_id, metadata
+        return image
 
     @staticmethod
     def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
-- 
cgit 


From a5bee00af4d6ec3eed6ed0abd866948f4510f041 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Mon, 7 Mar 2011 01:25:01 +0000
Subject: make compute get the new images properly, fix a bunch of tests, and
 provide conversion commands

---
 nova/image/glance.py  | 35 +++++++++++++++++++++++++++++-----
 nova/image/local.py   | 52 +++++++++++++++++++++++++++++++++++++++------------
 nova/image/s3.py      | 33 +++++++++++++++++++++-----------
 nova/image/service.py | 18 ++++++++++++++----
 4 files changed, 106 insertions(+), 32 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/glance.py b/nova/image/glance.py
index 7db94c0d4..fb383f5e6 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -17,9 +17,6 @@
 """Implementation of an image service that uses Glance as the backend"""
 
 from __future__ import absolute_import
-import httplib
-import json
-import urlparse
 
 from glance.common import exception as glance_exception
 
@@ -55,16 +52,44 @@ class GlanceImageService(service.BaseImageService):
         """
         return self.client.get_images_detailed()
 
-    def show(self, context, id):
+    def show(self, context, image_id):
         """
         Returns a dict containing image data for the given opaque image id.
         """
         try:
-            image = self.client.get_image_meta(id)
+            image = self.client.get_image_meta(image_id)
         except glance_exception.NotFound:
             raise exception.NotFound
         return image
 
+    def show_by_name(self, context, name):
+        """
+        Returns a dict containing image data for the given name.
+        """
+        # TODO(vish): replace this with more efficient call when glance
+        #             supports it.
+        images = self.detail(context)
+        image = None
+        for cantidate in images:
+            if name == cantidate.get('name'):
+                image = cantidate
+                break
+        if image == None:
+            raise exception.NotFound
+        return image
+
+    def get(self, context, image_id, data):
+        """
+        Calls out to Glance for metadata and data and writes data.
+        """
+        try:
+            metadata, image_chunks = self.client.get_image(image_id)
+        except glance_exception.NotFound:
+            raise exception.NotFound
+        for chunk in image_chunks:
+            data.write(chunk)
+        return metadata
+
     def create(self, context, metadata, data=None):
         """
         Store the image data and return the new image id.
diff --git a/nova/image/local.py b/nova/image/local.py
index 6fa648b6b..c4ac3baaa 100644
--- a/nova/image/local.py
+++ b/nova/image/local.py
@@ -42,13 +42,12 @@ class LocalImageService(service.BaseImageService):
 
     def _path_to(self, image_id, fname='info.json'):
         if fname:
-            return os.path.join(self._path, str(image_id), fname)
-        return os.path.join(self._path, str(image_id))
+            return os.path.join(self._path, '%08x' % int(image_id), fname)
+        return os.path.join(self._path, '%08x' % int(image_id))
 
     def _ids(self):
         """The list of all image ids."""
-        return [int(i) for i in os.listdir(self._path)
-                if unicode(i).isnumeric()]
+        return [int(i, 16) for i in os.listdir(self._path)]
 
     def index(self, context):
         return [dict(image_id=i['id'], name=i.get('name'))
@@ -68,27 +67,56 @@ class LocalImageService(service.BaseImageService):
         try:
             with open(self._path_to(image_id)) as metadata_file:
                 return json.load(metadata_file)
-        except IOError:
+        except (IOError, ValueError):
             raise exception.NotFound
 
+    def show_by_name(self, context, name):
+        """Returns a dict containing image data for the given name."""
+        # NOTE(vish): Not very efficient, but the local image service
+        #             is for testing so it should be fine.
+        images = self.detail(context)
+        image = None
+        for cantidate in images:
+            if name == cantidate.get('name'):
+                image = cantidate
+                break
+        if image == None:
+            raise exception.NotFound
+        return image
+
+    def get(self, context, image_id, data):
+        """Get image and metadata."""
+        try:
+            with open(self._path_to(image_id)) as metadata_file:
+                metadata = json.load(metadata_file)
+            with open(self._path_to(image_id, 'image')) as image_file:
+                shutil.copyfileobj(image_file, data)
+        except (IOError, ValueError):
+            raise exception.NotFound
+        return metadata
+
     def create(self, context, metadata, data=None):
-        """Store the image data and return the new image id."""
+        """Store the image data and return the new image."""
         image_id = random.randint(0, 2 ** 31 - 1)
         image_path = self._path_to(image_id, None)
         if not os.path.exists(image_path):
             os.mkdir(image_path)
-        return  self.update(context, image_id, metadata, data)
+        return self.update(context, image_id, metadata, data)
 
     def update(self, context, image_id, metadata, data=None):
         """Replace the contents of the given image with the new data."""
         metadata['id'] = image_id
         try:
-            with open(self._path_to(image_id), 'w') as metadata_file:
-                json.dump(metadata, metadata_file)
             if data:
-                with open(self._path_to(image_id, 'image'), 'w') as image_file:
+                location = self._path_to(image_id, 'image')
+                with open(location, 'w') as image_file:
                     shutil.copyfileobj(data, image_file)
-        except IOError:
+                # NOTE(vish): update metadata similarly to glance
+                metadata['status'] = 'active'
+                metadata['location'] = location
+            with open(self._path_to(image_id), 'w') as metadata_file:
+                json.dump(metadata, metadata_file)
+        except (IOError, ValueError):
             raise exception.NotFound
         return metadata
 
@@ -99,7 +127,7 @@ class LocalImageService(service.BaseImageService):
         """
         try:
             shutil.rmtree(self._path_to(image_id, None))
-        except IOError:
+        except (IOError, ValueError):
             raise exception.NotFound
 
     def delete_all(self):
diff --git a/nova/image/s3.py b/nova/image/s3.py
index c7446f4b0..ab6eea8cf 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -36,6 +36,7 @@ from nova import flags
 from nova import utils
 from nova.auth import manager
 from nova.image import service
+from nova.api.ec2 import ec2utils
 
 
 FLAGS = flags.FLAGS
@@ -51,7 +52,7 @@ class S3ImageService(service.BaseImageService):
         self.service.__init__(*args, **kwargs)
 
     def create(self, context, metadata, data=None):
-        """metadata should contain image_location"""
+        """metadata['properties'] should contain image_location"""
         image = self._s3_create(context, metadata)
         return image
 
@@ -97,6 +98,12 @@ class S3ImageService(service.BaseImageService):
             raise exception.NotFound
         return image
 
+    def show_by_name(self, context, name):
+        image = self.service.show_by_name(context, name)
+        if not self._is_visible(context, image):
+            raise exception.NotFound
+        return image
+
     @staticmethod
     def _conn(context):
         # TODO(vish): is there a better way to get creds to sign
@@ -119,10 +126,12 @@ class S3ImageService(service.BaseImageService):
         key.get_contents_to_filename(local_filename)
         return local_filename
 
-    def _s3_create(self, context, properties):
+    def _s3_create(self, context, metadata):
+        """Gets a manifext from s3 and makes an image"""
+
         image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir)
 
-        image_location = properties['image_location']
+        image_location = metadata['properties']['image_location']
         bucket_name = image_location.split("/")[0]
         manifest_path = image_location[len(bucket_name) + 1:]
         bucket = self._conn(context).get_bucket(bucket_name)
@@ -153,25 +162,27 @@ class S3ImageService(service.BaseImageService):
         except:
             arch = 'x86_64'
 
-        properties.update({'owner_id': context.project_id,
-                           'architecture': arch})
+        properties = metadata['properties']
+        properties['owner_id'] = context.project_id
+        properties['architecture'] = arch
 
         if kernel_id:
-            properties['kernel_id'] = kernel_id
+            properties['kernel_id'] = ec2utils.ec2_id_to_id(kernel_id)
 
         if ramdisk_id:
-            properties['ramdisk_id'] = ramdisk_id
+            properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id)
 
         properties['is_public'] = False
-        metadata = {'type': image_type,
-                    'status': 'queued',
-                    'is_public': True,
-                    'properties': properties}
+        metadata.update({'type': image_type,
+                         'status': 'queued',
+                         'is_public': True,
+                         'properties': properties})
         metadata['properties']['image_state'] = 'pending'
         image = self.service.create(context, metadata)
         image_id = image['id']
 
         def delayed_create():
+            """This handles the fetching and decrypting of the part files."""
             parts = []
             for fn_element in manifest.find("image").getiterator("filename"):
                 part = self._download_file(bucket, fn_element.text, image_path)
diff --git a/nova/image/service.py b/nova/image/service.py
index e429955f4..c09052cab 100644
--- a/nova/image/service.py
+++ b/nova/image/service.py
@@ -56,9 +56,9 @@ class BaseImageService(object):
         """
         raise NotImplementedError
 
-    def show(self, context, id):
+    def show(self, context, image_id):
         """
-        Returns a dict containing image data for the given opaque image id.
+        Returns a dict containing image metadata for the given opaque image id.
 
         :retval a mapping with the following signature:
 
@@ -76,9 +76,19 @@ class BaseImageService(object):
         """
         raise NotImplementedError
 
+    def get(self, context, data):
+        """
+        Returns a dict containing image metadata and writes image data to data.
+
+        :param data: a file-like object to hold binary image data
+
+        :raises NotFound if the image does not exist
+        """
+        raise NotImplementedError
+
     def create(self, context, metadata, data=None):
         """
-        Store the image data and return the new image id.
+        Store the image metadata and data and return the new image id.
 
         :raises AlreadyExists if the image already exist.
 
@@ -86,7 +96,7 @@ class BaseImageService(object):
         raise NotImplementedError
 
     def update(self, context, image_id, metadata, data=None):
-        """Replace the contents of the given image with the new data.
+        """Update the given image with the new metadata and data.
 
         :raises NotFound if the image does not exist.
 
-- 
cgit 


From 23d3be4b6f28359211e29212867157daeac9e142 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Tue, 8 Mar 2011 20:25:05 +0000
Subject: update code to work with new container and disk formats from glance

---
 nova/image/s3.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'nova/image')

diff --git a/nova/image/s3.py b/nova/image/s3.py
index ab6eea8cf..bf104c29a 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -139,11 +139,13 @@ class S3ImageService(service.BaseImageService):
         manifest = key.get_contents_as_string()
 
         manifest = ElementTree.fromstring(manifest)
+        image_format = 'ami'
         image_type = 'machine'
 
         try:
             kernel_id = manifest.find("machine_configuration/kernel_id").text
             if kernel_id == 'true':
+                image_format = 'aki'
                 image_type = 'kernel'
                 kernel_id = None
         except:
@@ -152,6 +154,7 @@ class S3ImageService(service.BaseImageService):
         try:
             ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text
             if ramdisk_id == 'true':
+                image_format = 'ari'
                 image_type = 'ramdisk'
                 ramdisk_id = None
         except:
@@ -173,7 +176,9 @@ class S3ImageService(service.BaseImageService):
             properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id)
 
         properties['is_public'] = False
-        metadata.update({'type': image_type,
+        properties['type'] = image_type
+        metadata.update({'disk_format': image_format,
+                         'container_format': image_format,
                          'status': 'queued',
                          'is_public': True,
                          'properties': properties})
-- 
cgit 


From a83b4879f38d11634d405d0efe977d482abdc344 Mon Sep 17 00:00:00 2001
From: Vishvananda Ishaya <vishvananda@gmail.com>
Date: Thu, 10 Mar 2011 05:02:24 +0000
Subject: minor fixes from review

---
 nova/image/glance.py |  2 +-
 nova/image/s3.py     | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/glance.py b/nova/image/glance.py
index fb383f5e6..15fca69b8 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -74,7 +74,7 @@ class GlanceImageService(service.BaseImageService):
             if name == cantidate.get('name'):
                 image = cantidate
                 break
-        if image == None:
+        if image is None:
             raise exception.NotFound
         return image
 
diff --git a/nova/image/s3.py b/nova/image/s3.py
index bf104c29a..bbc54c263 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -77,17 +77,17 @@ class S3ImageService(service.BaseImageService):
         # FIXME(vish): detail doesn't filter so we do it manually
         return self._filter(context, images)
 
-    @staticmethod
-    def _is_visible(context, image):
+    @classmethod
+    def _is_visible(cls, context, image):
         return (context.is_admin
                 or context.project_id == image['properties']['owner_id']
                 or image['properties']['is_public'] == 'True')
 
-    @staticmethod
-    def _filter(context, images):
+    @classmethod
+    def _filter(cls, context, images):
         filtered = []
         for image in images:
-            if not S3ImageService._is_visible(context, image):
+            if not cls._is_visible(context, image):
                 continue
             filtered.append(image)
         return filtered
@@ -148,7 +148,7 @@ class S3ImageService(service.BaseImageService):
                 image_format = 'aki'
                 image_type = 'kernel'
                 kernel_id = None
-        except:
+        except Exception:
             kernel_id = None
 
         try:
@@ -157,12 +157,12 @@ class S3ImageService(service.BaseImageService):
                 image_format = 'ari'
                 image_type = 'ramdisk'
                 ramdisk_id = None
-        except:
+        except Exception:
             ramdisk_id = None
 
         try:
             arch = manifest.find("machine_configuration/architecture").text
-        except:
+        except Exception:
             arch = 'x86_64'
 
         properties = metadata['properties']
@@ -235,7 +235,7 @@ class S3ImageService(service.BaseImageService):
 
     @staticmethod
     def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
-                      cloud_private_key, decrypted_filename):
+                       cloud_private_key, decrypted_filename):
         key, err = utils.execute(
                 'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
                 process_input=encrypted_key,
-- 
cgit 


From 6601d52bfa501ac1ae266647be19fac2f6792efc Mon Sep 17 00:00:00 2001
From: Soren Hansen <soren@linux2go.dk>
Date: Thu, 10 Mar 2011 11:35:42 +0100
Subject: Make nova.image.s3 catch up with the new execute syntax.

---
 nova/image/s3.py | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

(limited to 'nova/image')

diff --git a/nova/image/s3.py b/nova/image/s3.py
index bbc54c263..85a2c651c 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -236,25 +236,32 @@ class S3ImageService(service.BaseImageService):
     @staticmethod
     def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv,
                        cloud_private_key, decrypted_filename):
-        key, err = utils.execute(
-                'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
-                process_input=encrypted_key,
-                check_exit_code=False)
+        key, err = utils.execute('openssl',
+                                 'rsautl',
+                                 '-decrypt',
+                                 '-inkey', '%s' % cloud_private_key,
+                                 process_input=encrypted_key,
+                                 check_exit_code=False)
         if err:
             raise exception.Error(_("Failed to decrypt private key: %s")
                                   % err)
-        iv, err = utils.execute(
-                'openssl rsautl -decrypt -inkey %s' % cloud_private_key,
-                process_input=encrypted_iv,
-                check_exit_code=False)
+        iv, err = utils.execute('openssl',
+                                'rsautl',
+                                '-decrypt',
+                                '-inkey', '%s' % cloud_private_key,
+                                process_input=encrypted_iv,
+                                check_exit_code=False)
         if err:
             raise exception.Error(_("Failed to decrypt initialization "
                                     "vector: %s") % err)
 
-        _out, err = utils.execute(
-                'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s'
-                 % (encrypted_filename, key, iv, decrypted_filename),
-                 check_exit_code=False)
+        _out, err = utils.execute('openssl', 'enc',
+                                  '-d', '-aes-128-cbc',
+                                  '-in', '%s' % (encrypted_filename,),
+                                  '-K', '%s' % (key,),
+                                  '-iv', '%s' % (iv,),
+                                  '-out', '%s' % (decrypted_filename,),
+                                  check_exit_code=False)
         if err:
             raise exception.Error(_("Failed to decrypt image file "
                                     "%(image_file)s: %(err)s") %
-- 
cgit