api/ec2/image: support block device mapping

This patch adds --block-device-mapping support for register image and
instance creation.

5 files changed, 282 insertions, 53 deletions

diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py
index d36eb7a04..1e594cd73 100644
--- a/nova/api/ec2/cloud.py
+++ b/nova/api/ec2/cloud.py
@@ -27,6 +27,7 @@ import IPy
 import os
 import urllib
 import tempfile
+import time
 import shutil
 
 from nova import compute
@@ -75,6 +76,73 @@ def _gen_key(context, user_id, key_name):
     return {'private_key': private_key, 'fingerprint': fingerprint}
 
 
+# TODO(yamahata): hypervisor dependent default device name
+_DEFAULT_ROOT_DEVICE_NAME = '/dev/sda1'
+
+
+def _parse_block_device_mapping(bdm):
+    """Parse BlockDeviceMappingItemType into flat hash
+    BlockDevicedMapping.<N>.DeviceName
+    BlockDevicedMapping.<N>.Ebs.SnapshotId
+    BlockDevicedMapping.<N>.Ebs.VolumeSize
+    BlockDevicedMapping.<N>.Ebs.DeleteOnTermination
+    BlockDevicedMapping.<N>.Ebs.NoDevice
+    BlockDevicedMapping.<N>.VirtualName
+    => remove .Ebs and allow volume id in SnapshotId
+    """
+    ebs = bdm.pop('ebs', None)
+    if ebs:
+        ec2_id = ebs.pop('snapshot_id')
+        id = ec2utils.ec2_id_to_id(ec2_id)
+        if ec2_id.startswith('snap-'):
+            bdm['snapshot_id'] = id
+        elif ec2_id.startswith('vol-'):
+            bdm['volume_id'] = id
+        ebs.setdefault('delete_on_termination', True)
+        bdm.update(ebs)
+    return bdm
+
+
+def _format_block_device_mapping(bdm):
+    """Contruct BlockDeviceMappingItemType
+    {'device_name': '...', 'Snapshot_Id': , ...}
+    => BlockDeviceMappingItemType
+    """
+    keys = (('deviceName', 'device_name'),
+             ('virtualName', 'virtual_name'))
+    item = {}
+    for name, k in keys:
+        if k in bdm:
+            item[name] = bdm[k]
+    if bdm.get('no_device'):
+        item['noDevice'] = True
+    if ('snapshot_id' in bdm) or ('volume_id' in bdm):
+        ebs_keys = (('snapshotId', 'snapshot_id'),
+                    ('snapshotId', 'volume_id'),        # snapshotId is abused
+                    ('volumeSize', 'volume_size'),
+                    ('deleteOnTermination', 'delete_on_termination'))
+        ebs = {}
+        for name, k in ebs_keys:
+            if k in bdm:
+                if k == 'snapshot_id':
+                    ebs[name] = ec2utils.id_to_ec2_snap_id(bdm[k])
+                elif k == 'volume_id':
+                    ebs[name] = ec2utils.id_to_ec2_vol_id(bdm[k])
+                else:
+                    ebs[name] = bdm[k]
+        assert 'snapshotId' in ebs
+        item['ebs'] = ebs
+    return item
+
+
+def _format_mappings(mappings, result):
+    """Format multiple BlockDeviceMappingItemType"""
+    block_device_mapping = [_format_block_device_mapping(bdm) for bdm in
+                            mappings]
+    if block_device_mapping:
+        result['blockDeviceMapping'] = block_device_mapping
+
+
 class CloudController(object):
     """ CloudController provides the critical dispatch between
  inbound API calls through the endpoint and messages
@@ -177,7 +245,7 @@ class CloudController(object):
                     # TODO(vish): replace with real data
                     'ami': 'sda1',
                     'ephemeral0': 'sda2',
-                    'root': '/dev/sda1',
+                    'root': _DEFAULT_ROOT_DEVICE_NAME,
                     'swap': 'sda3'},
                 'hostname': hostname,
                 'instance-action': 'none',
@@ -761,6 +829,37 @@ class CloudController(object):
         assert len(i) == 1
         return i[0]
 
+    def _format_instance_bdm(self, context, instance_id, root_device_name,
+                             result):
+        """Format InstanceBlockDeviceMappingResponseItemType"""
+        root_device_type = 'instance-store'
+        mapping = []
+        for bdm in db.block_device_mapping_get_all_by_instance(context,
+                                                               instance_id):
+            volume_id = bdm['volume_id']
+            if (volume_id is None or bdm['no_device']):
+                continue
+
+            if (bdm['device_name'] == root_device_name and
+                (bdm['snapshot_id'] or bdm['volume_id'])):
+                assert not bdm['virtual_name']
+                root_device_type = 'ebs'
+
+            vol = self.volume_api.get(context, volume_id=volume_id)
+            LOG.debug(_("vol = %s\n"), vol)
+            # TODO(yamahata): volume attach time
+            ebs = {'volumeId': volume_id,
+                   'deleteOnTermination': bdm['delete_on_termination'],
+                   'attachTime': vol['attach_time'] or '-',
+                   'status': vol['status'], }
+            res = {'deviceName': bdm['device_name'],
+                   'ebs': ebs, }
+            mapping.append(res)
+
+        if mapping:
+            result['blockDeviceMapping'] = mapping
+        result['rootDeviceType'] = root_device_type
+
     def _format_instances(self, context, instance_id=None, **kwargs):
         # TODO(termie): this method is poorly named as its name does not imply
         #               that it will be making a variety of database calls
@@ -822,6 +921,10 @@ class CloudController(object):
             i['amiLaunchIndex'] = instance['launch_index']
             i['displayName'] = instance['display_name']
             i['displayDescription'] = instance['display_description']
+            i['rootDeviceName'] = (instance['root_device_name'] or
+                                   _DEFAULT_ROOT_DEVICE_NAME)
+            self._format_instance_bdm(context, instance_id,
+                                      i['rootDeviceName'], i)
             host = instance['host']
             zone = self._get_availability_zone_by_host(context, host)
             i['placement'] = {'availabilityZone': zone}
@@ -907,24 +1010,7 @@ class CloudController(object):
         if kwargs.get('ramdisk_id'):
             ramdisk = self._get_image(context, kwargs['ramdisk_id'])
             kwargs['ramdisk_id'] = ramdisk['id']
-        for bdm in kwargs.get('block_device_mapping', []):
-            # NOTE(yamahata)
-            # BlockDevicedMapping.<N>.DeviceName
-            # BlockDevicedMapping.<N>.Ebs.SnapshotId
-            # BlockDevicedMapping.<N>.Ebs.VolumeSize
-            # BlockDevicedMapping.<N>.Ebs.DeleteOnTermination
-            # BlockDevicedMapping.<N>.VirtualName
-            # => remove .Ebs and allow volume id in SnapshotId
-            ebs = bdm.pop('ebs', None)
-            if ebs:
-                ec2_id = ebs.pop('snapshot_id')
-                id = ec2utils.ec2_id_to_id(ec2_id)
-                if ec2_id.startswith('snap-'):
-                    bdm['snapshot_id'] = id
-                elif ec2_id.startswith('vol-'):
-                    bdm['volume_id'] = id
-                ebs.setdefault('delete_on_termination', True)
-                bdm.update(ebs)
+        _parse_block_device_mapping(bdm)
 
         image = self._get_image(context, kwargs['image_id'])
 
@@ -1078,6 +1164,20 @@ class CloudController(object):
         i['imageType'] = display_mapping.get(image_type)
         i['isPublic'] = image.get('is_public') == True
         i['architecture'] = image['properties'].get('architecture')
+
+        properties = image['properties']
+        root_device_name = ec2utils.properties_root_device_name(properties)
+        root_device_type = 'instance-store'
+        for bdm in properties.get('block_device_mapping', []):
+            if (bdm.get('device_name') == root_device_name and
+                ('snapshot_id' in bdm or 'volume_id' in bdm) and
+                not bdm.get('no_device')):
+                root_device_type = 'ebs'
+        i['rootDeviceName'] = (root_device_name or _DEFAULT_ROOT_DEVICE_NAME)
+        i['rootDeviceType'] = root_device_type
+
+        _format_mappings(properties.get('block_device_mapping', []), i)
+
         return i
 
     def describe_images(self, context, image_id=None, **kwargs):
@@ -1102,30 +1202,65 @@ class CloudController(object):
         self.image_service.delete(context, internal_id)
         return {'imageId': image_id}
 
+    def _register_image(self, context, metadata):
+        image = self.image_service.create(context, metadata)
+        image_type = self._image_type(image.get('container_format'))
+        image_id = self.image_ec2_id(image['id'], image_type)
+        return image_id
+
     def register_image(self, context, image_location=None, **kwargs):
         if image_location is None and 'name' in kwargs:
             image_location = kwargs['name']
         metadata = {'properties': {'image_location': image_location}}
-        image = self.image_service.create(context, metadata)
-        image_type = self._image_type(image.get('container_format'))
-        image_id = self.image_ec2_id(image['id'],
-                                     image_type)
+
+        if 'root_device_name' in kwargs:
+            metadata['properties']['root_device_name'] = \
+            kwargs.get('root_device_name')
+
+        mappings = [_parse_block_device_mapping(bdm) for bdm in
+                    kwargs.get('block_device_mapping', [])]
+        if mappings:
+            metadata['properties']['block_device_mapping'] = mappings
+
+        image_id = self._register_image(context, metadata)
         msg = _("Registered image %(image_location)s with"
                 " id %(image_id)s") % locals()
         LOG.audit(msg, context=context)
         return {'imageId': image_id}
 
     def describe_image_attribute(self, context, image_id, attribute, **kwargs):
-        if attribute != 'launchPermission':
+        def _block_device_mapping_attribute(image, result):
+            _format_mappings(
+                image['properties'].get('block_device_mapping', []), result)
+
+        def _launch_permission_attribute(image, result):
+            result['launchPermission'] = []
+            if image['is_public']:
+                result['launchPermission'].append({'group': 'all'})
+
+        def _root_device_name_attribute(image, result):
+            result['rootDeviceName'] = \
+                    ec2utils.properties_root_device_name(image['properties'])
+            if result['rootDeviceName'] is None:
+                result['rootDeviceName'] = _DEFAULT_ROOT_DEVICE_NAME
+
+        supported_attributes = {
+            'blockDeviceMapping': _block_device_mapping_attribute,
+            'launchPermission': _launch_permission_attribute,
+            'rootDeviceName': _root_device_name_attribute,
+            }
+
+        fn = supported_attributes.get(attribute)
+        if fn is None:
             raise exception.ApiError(_('attribute not supported: %s')
                                      % attribute)
         try:
             image = self._get_image(context, image_id)
         except exception.NotFound:
             raise exception.ImageNotFound(image_id=image_id)
-        result = {'imageId': image_id, 'launchPermission': []}
-        if image['is_public']:
-            result['launchPermission'].append({'group': 'all'})
+
+        result = {'imageId': image_id}
+        fn(image, result)
         return result
 
     def modify_image_attribute(self, context, image_id, attribute,
diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py
index bcdf2ba78..9839f8604 100644
--- a/nova/api/ec2/ec2utils.py
+++ b/nova/api/ec2/ec2utils.py
@@ -135,3 +135,22 @@ def dict_from_dotted_str(items):
                 args[key] = value
 
     return args
+
+
+def properties_root_device_name(properties):
+    """get root device name from image meta data.
+    If it isn't specified, return None.
+    """
+    root_device_name = None
+
+    # NOTE(yamahata): see image_service.s3.s3create()
+    for bdm in properties.get('mappings', []):
+        if bdm['virtual'] == 'root':
+            root_device_name = bdm['device']
+
+    # NOTE(yamahata): register_image's command line can override
+    #                 <machine>.manifest.xml
+    if 'root_device_name' in properties:
+        root_device_name = properties['root_device_name']
+
+    return root_device_name
diff --git a/nova/compute/api.py b/nova/compute/api.py
index 18363ace0..d89dfb746 100644
--- a/nova/compute/api.py
+++ b/nova/compute/api.py
@@ -32,6 +32,7 @@ from nova import quota
 from nova import rpc
 from nova import utils
 from nova import volume
+from nova.api.ec2 import ec2utils
 from nova.compute import instance_types
 from nova.compute import power_state
 from nova.compute.utils import terminate_volumes
@@ -220,6 +221,9 @@ class API(base.Base):
         if reservation_id is None:
             reservation_id = utils.generate_uid('r')
 
+        root_device_name = ec2utils.properties_root_device_name(
+            image['properties'])
+
         base_options = {
             'reservation_id': reservation_id,
             'image_ref': image_href,
@@ -243,10 +247,62 @@ class API(base.Base):
             'metadata': metadata,
             'availability_zone': availability_zone,
             'os_type': os_type,
-            'vm_mode': vm_mode}
+            'vm_mode': vm_mode,
+            'root_device_name': root_device_name}
 
         return (num_instances, base_options, security_groups)
 
+    def _update_image_block_device_mapping(self, elevated_context, instance_id,
+                                           mappings):
+        """tell vm driver to create ephemeral/swap device at boot time by
+        updating BlockDeviceMapping
+        """
+        for bdm in mappings:
+            LOG.debug(_("bdm %s"), bdm)
+
+            virtual_name = bdm['virtualName']
+            if virtual_name == 'ami' or virtual_name == 'root':
+                continue
+
+            assert (virtual_name == 'swap' or
+                    virtual_name.startswith('ephemeral'))
+            values = {
+                'instance_id': instance_id,
+                'device_name': bdm['deviceName'],
+                'virtual_name': virtual_name, }
+            self.db.block_device_mapping_create(elevated_context, values)
+
+    def _update_block_device_mapping(self, elevated_context, instance_id,
+                                     block_device_mapping):
+        """tell vm driver to attach volume at boot time by updating
+        BlockDeviceMapping
+        """
+        for bdm in block_device_mapping:
+            LOG.debug(_('bdm %s'), bdm)
+            assert 'device_name' in bdm
+
+            values = {
+                'instance_id': instance_id,
+                'device_name': bdm['device_name'],
+                'delete_on_termination': bdm.get('delete_on_termination'),
+                'virtual_name': bdm.get('virtual_name'),
+                'snapshot_id': bdm.get('snapshot_id'),
+                'volume_id': bdm.get('volume_id'),
+                'volume_size': bdm.get('volume_size'),
+                'no_device': bdm.get('no_device')}
+
+            # NOTE(yamahata): NoDevice eliminates devices defined in image
+            #                 files by command line option.
+            #                 (--block-device-mapping)
+            if bdm.get('virtual_name') == 'NoDevice':
+                values['no_device'] = True
+                for k in ('delete_on_termination', 'volume_id',
+                          'snapshot_id', 'volume_id', 'volume_size',
+                          'virtual_name'):
+                    values[k] = None
+
+            self.db.block_device_mapping_create(elevated_context, values)
+
     def create_db_entry_for_new_instance(self, context, base_options,
              security_groups, block_device_mapping, num=1):
         """Create an entry in the DB for this new instance,
@@ -268,22 +324,14 @@ class API(base.Base):
                                                 instance_id,
                                                 security_group_id)
 
-        # NOTE(yamahata)
-        # tell vm driver to attach volume at boot time by updating
-        # BlockDeviceMapping
-        for bdm in block_device_mapping:
-            LOG.debug(_('bdm %s'), bdm)
-            assert 'device_name' in bdm
-            values = {
-                'instance_id': instance_id,
-                'device_name': bdm['device_name'],
-                'delete_on_termination': bdm.get('delete_on_termination'),
-                'virtual_name': bdm.get('virtual_name'),
-                'snapshot_id': bdm.get('snapshot_id'),
-                'volume_id': bdm.get('volume_id'),
-                'volume_size': bdm.get('volume_size'),
-                'no_device': bdm.get('no_device')}
-            self.db.block_device_mapping_create(elevated, values)
+        # BlockDeviceMapping table
+        self._update_image_block_device_mapping(elevated, instance_id,
+            image['properties'].get('mappings', []))
+        self._update_block_device_mapping(elevated, instance_id,
+            image['properties'].get('block_device_mapping', []))
+        # override via command line option
+        self._update_block_device_mapping(elevated, instance_id,
+                                          block_device_mapping)
 
         # Set sane defaults if not specified
         updates = dict(hostname=self.hostname_factory(instance_id))
diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 8ab744855..146cdbb9f 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -227,6 +227,17 @@ class ComputeManager(manager.SchedulerDependentManager):
         for bdm in self.db.block_device_mapping_get_all_by_instance(
             context, instance_id):
             LOG.debug(_("setting up bdm %s"), bdm)
+
+            if bdm['no_device']:
+                continue
+            if bdm['virtual_name']:
+                # TODO(yamahata):
+                # block devices for swap and ephemeralN will be
+                # created by virt driver locally in compute node.
+                assert (bdm['virtual_name'] == 'swap' or
+                        bdm['virtual_name'].startswith('ephemeral'))
+                continue
+
             if ((bdm['snapshot_id'] is not None) and
                 (bdm['volume_id'] is None)):
                 # TODO(yamahata): default name and description
@@ -259,15 +270,6 @@ class ComputeManager(manager.SchedulerDependentManager):
                 block_device_mapping.append({'device_path': dev_path,
                                              'mount_device':
                                              bdm['device_name']})
-            elif bdm['virtual_name'] is not None:
-                # TODO(yamahata): ephemeral/swap device support
-                LOG.debug(_('block_device_mapping: '
-                            'ephemeral device is not supported yet'))
-            else:
-                # TODO(yamahata): NoDevice support
-                assert bdm['no_device']
-                LOG.debug(_('block_device_mapping: '
-                            'no device is not supported yet'))
 
         return block_device_mapping
 
diff --git a/nova/image/s3.py b/nova/image/s3.py
index 9e95bd698..b1afc6a81 100644
--- a/nova/image/s3.py
+++ b/nova/image/s3.py
@@ -141,6 +141,28 @@ class S3ImageService(service.BaseImageService):
         except Exception:
             arch = 'x86_64'
 
+        # NOTE(yamahata):
+        # EC2 ec2-budlne-image --block-device-mapping accepts
+        # <virtual name>=<device name> where
+        # virtual name = {ami, root, swap, ephemeral<N>}
+        #                where N is no negative integer
+        # device name = the device name seen by guest kernel.
+        # They are converted into
+        # block_device_mapping/mapping/{virtual, device}
+        #
+        # Do NOT confuse this with ec2-register's block device mapping
+        # argument.
+        mappings = []
+        try:
+            block_device_mapping = manifest.findall('machine_configuration/'
+                                                    'block_device_mapping/'
+                                                    'mapping')
+            for bdm in block_device_mapping:
+                mappings.append({'virtual': bdm.find('virtual').text,
+                                 'device': bdm.find('device').text})
+        except Exception:
+            mappings = []
+
         properties = metadata['properties']
         properties['project_id'] = context.project_id
         properties['architecture'] = arch
@@ -151,6 +173,9 @@ class S3ImageService(service.BaseImageService):
         if ramdisk_id:
             properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id)
 
+        if mappings:
+            properties['mappings'] = mappings
+
         metadata.update({'disk_format': image_format,
                          'container_format': image_format,
                          'status': 'queued',