add ability to clone images

Given the backend location from Glance, drivers can determine
whether they can clone or otherwise efficiently create a volume
from the image without downloading all the data from Glance.

For now implement cloning for the RBD driver. There's already a
Glance backend that stores images as RBD snapshots, so they're
ready to be cloned into volumes. Fall back to copying all the
data if cloning is not possible.

Implements: blueprint efficient-volumes-from-images
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

Conflicts:

	nova/volume/api.py
	nova/volume/driver.py

This is based on a cherry-pick of cinder commit
edc11101cbc06bdce95b10cfd00a4849f6c01b33

Change-Id: I71a8172bd22a5bbf64d4c68631630125fcc7fd34

3 files changed, 260 insertions, 7 deletions

diff --git a/nova/tests/test_rbd.py b/nova/tests/test_rbd.py
new file mode 100644
index 000000000..8e90f3ae8
--- /dev/null
+++ b/nova/tests/test_rbd.py
@@ -0,0 +1,161 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 Josh Durgin
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova import db
+from nova import exception
+from nova.openstack.common import log as logging
+from nova.openstack.common import timeutils
+from nova import test
+from nova.tests.image import fake as fake_image
+from nova.tests.test_volume import DriverTestCase
+from nova.volume.driver import RBDDriver
+
+LOG = logging.getLogger(__name__)
+
+
+class RBDTestCase(test.TestCase):
+
+    def setUp(self):
+        super(RBDTestCase, self).setUp()
+
+        def fake_execute(*args):
+            pass
+        self.driver = RBDDriver(execute=fake_execute)
+
+    def test_good_locations(self):
+        locations = [
+            'rbd://fsid/pool/image/snap',
+            'rbd://%2F/%2F/%2F/%2F',
+            ]
+        map(self.driver._parse_location, locations)
+
+    def test_bad_locations(self):
+        locations = [
+            'rbd://image',
+            'http://path/to/somewhere/else',
+            'rbd://image/extra',
+            'rbd://image/',
+            'rbd://fsid/pool/image/',
+            'rbd://fsid/pool/image/snap/',
+            'rbd://///',
+            ]
+        for loc in locations:
+            self.assertRaises(exception.ImageUnacceptable,
+                              self.driver._parse_location,
+                              loc)
+            self.assertFalse(self.driver._is_cloneable(loc))
+
+    def test_cloneable(self):
+        self.stubs.Set(self.driver, '_get_fsid', lambda: 'abc')
+        location = 'rbd://abc/pool/image/snap'
+        self.assertTrue(self.driver._is_cloneable(location))
+
+    def test_uncloneable_different_fsid(self):
+        self.stubs.Set(self.driver, '_get_fsid', lambda: 'abc')
+        location = 'rbd://def/pool/image/snap'
+        self.assertFalse(self.driver._is_cloneable(location))
+
+    def test_uncloneable_unreadable(self):
+        def fake_exc(*args):
+            raise exception.ProcessExecutionError()
+        self.stubs.Set(self.driver, '_get_fsid', lambda: 'abc')
+        self.stubs.Set(self.driver, '_execute', fake_exc)
+        location = 'rbd://abc/pool/image/snap'
+        self.assertFalse(self.driver._is_cloneable(location))
+
+
+class FakeRBDDriver(RBDDriver):
+
+    def _clone(self):
+        pass
+
+    def _resize(self):
+        pass
+
+
+class ManagedRBDTestCase(DriverTestCase):
+    driver_name = "nova.tests.test_rbd.FakeRBDDriver"
+
+    def setUp(self):
+        super(ManagedRBDTestCase, self).setUp()
+        fake_image.stub_out_image_service(self.stubs)
+
+    def _clone_volume_from_image(self, expected_status,
+                                 clone_works=True):
+        """Try to clone a volume from an image, and check the status
+        afterwards"""
+        def fake_clone_image(volume, image_location):
+            pass
+
+        def fake_clone_error(volume, image_location):
+            raise exception.NovaException()
+
+        self.stubs.Set(self.volume.driver, '_is_cloneable', lambda x: True)
+        if clone_works:
+            self.stubs.Set(self.volume.driver, 'clone_image', fake_clone_image)
+        else:
+            self.stubs.Set(self.volume.driver, 'clone_image', fake_clone_error)
+
+        image_id = 'c905cedb-7281-47e4-8a62-f26bc5fc4c77'
+        volume_id = 1
+        # creating volume testdata
+        db.volume_create(self.context, {'id': volume_id,
+                            'updated_at': timeutils.utcnow(),
+                            'display_description': 'Test Desc',
+                            'size': 20,
+                            'status': 'creating',
+                            'instance_uuid': None,
+                            'host': 'dummy'})
+        try:
+            if clone_works:
+                self.volume.create_volume(self.context,
+                                          volume_id,
+                                          image_id=image_id)
+            else:
+                self.assertRaises(exception.NovaException,
+                                  self.volume.create_volume,
+                                  self.context,
+                                  volume_id,
+                                  image_id=image_id)
+
+            volume = db.volume_get(self.context, volume_id)
+            self.assertEqual(volume['status'], expected_status)
+        finally:
+            # cleanup
+            db.volume_destroy(self.context, volume_id)
+
+    def test_clone_image_status_available(self):
+        """Verify that before cloning, an image is in the available state."""
+        self._clone_volume_from_image('available', True)
+
+    def test_clone_image_status_error(self):
+        """Verify that before cloning, an image is in the available state."""
+        self._clone_volume_from_image('error', False)
+
+    def test_clone_success(self):
+        self.stubs.Set(self.volume.driver, '_is_cloneable', lambda x: True)
+        self.stubs.Set(self.volume.driver, 'clone_image', lambda a, b: True)
+        image_id = 'c905cedb-7281-47e4-8a62-f26bc5fc4c77'
+        self.assertTrue(self.volume.driver.clone_image({}, image_id))
+
+    def test_clone_bad_image_id(self):
+        self.stubs.Set(self.volume.driver, '_is_cloneable', lambda x: True)
+        self.assertFalse(self.volume.driver.clone_image({}, None))
+
+    def test_clone_uncloneable(self):
+        self.stubs.Set(self.volume.driver, '_is_cloneable', lambda x: False)
+        self.assertFalse(self.volume.driver.clone_image({}, 'dne'))
diff --git a/nova/volume/driver.py b/nova/volume/driver.py
index 0a124923b..498216a2c 100644
--- a/nova/volume/driver.py
+++ b/nova/volume/driver.py
@@ -21,7 +21,9 @@ Drivers for volumes.
 """
 
 import os
+import tempfile
 import time
+import urllib
 
 from nova import exception
 from nova import flags
@@ -65,6 +67,10 @@ volume_opts = [
                default=None,
                help='the libvirt uuid of the secret for the rbd_user'
                     'volumes'),
+    cfg.StrOpt('volume_tmp_dir',
+               default=None,
+               help='where to store temporary image files if the volume '
+                    'driver does not write them directly to the volume'),
     ]
 
 FLAGS = flags.FLAGS
@@ -269,6 +275,17 @@ class VolumeDriver(object):
         """Copy the volume to the specified image."""
         raise NotImplementedError()
 
+    def clone_image(self, volume, image_location):
+        """Create a volume efficiently from an existing image.
+
+        image_location is a string whose format depends on the
+        image service backend in use. The driver should use it
+        to determine whether cloning is possible.
+
+        Returns a boolean indicating whether cloning occurred
+        """
+        return False
+
 
 class ISCSIDriver(VolumeDriver):
     """Executes commands relating to ISCSI volumes.
@@ -717,6 +734,72 @@ class RBDDriver(VolumeDriver):
     def terminate_connection(self, volume, connector):
         pass
 
+    def _parse_location(self, location):
+        prefix = 'rbd://'
+        if not location.startswith(prefix):
+            reason = _('Image %s is not stored in rbd') % location
+            raise exception.ImageUnacceptable(reason)
+        pieces = map(urllib.unquote, location[len(prefix):].split('/'))
+        if any(map(lambda p: p == '', pieces)):
+            reason = _('Image %s has blank components') % location
+            raise exception.ImageUnacceptable(reason)
+        if len(pieces) != 4:
+            reason = _('Image %s is not an rbd snapshot') % location
+            raise exception.ImageUnacceptable(reason)
+        return pieces
+
+    def _get_fsid(self):
+        stdout, _ = self._execute('ceph', 'fsid')
+        return stdout.rstrip('\n')
+
+    def _is_cloneable(self, image_location):
+        try:
+            fsid, pool, image, snapshot = self._parse_location(image_location)
+        except exception.ImageUnacceptable:
+            return False
+
+        if self._get_fsid() != fsid:
+            reason = _('%s is in a different ceph cluster') % image_location
+            LOG.debug(reason)
+            return False
+
+        # check that we can read the image
+        try:
+            self._execute('rbd', 'info',
+                          '--pool', pool,
+                          '--image', image,
+                          '--snap', snapshot)
+        except exception.ProcessExecutionError:
+            LOG.debug(_('Unable to read image %s') % image_location)
+            return False
+
+        return True
+
+    def clone_image(self, volume, image_location):
+        if image_location is None or not self._is_cloneable(image_location):
+            return False
+        _, pool, image, snapshot = self._parse_location(image_location)
+        self._clone(volume, pool, image, snapshot)
+        self._resize(volume)
+        return True
+
+    def copy_image_to_volume(self, context, volume, image_service, image_id):
+        # TODO(jdurgin): replace with librbd
+        # this is a temporary hack, since rewriting this driver
+        # to use librbd would take too long
+        if FLAGS.volume_tmp_dir and not os.exists(FLAGS.volume_tmp_dir):
+            os.makedirs(FLAGS.volume_tmp_dir)
+
+        with tempfile.NamedTemporaryFile(dir=FLAGS.volume_tmp_dir) as tmp:
+            image_service.download(context, image_id, tmp)
+            # import creates the image, so we must remove it first
+            self._try_execute('rbd', 'rm',
+                              '--pool', FLAGS.rbd_pool,
+                              volume['name'])
+            self._try_execute('rbd', 'import',
+                              '--pool', FLAGS.rbd_pool,
+                              tmp.name, volume['name'])
+
 
 class SheepdogDriver(VolumeDriver):
     """Executes commands relating to Sheepdog Volumes"""
diff --git a/nova/volume/manager.py b/nova/volume/manager.py
index cd3471146..e25f3daf1 100644
--- a/nova/volume/manager.py
+++ b/nova/volume/manager.py
@@ -127,23 +127,32 @@ class VolumeManager(manager.SchedulerDependentManager):
         #             before passing it to the driver.
         volume_ref['host'] = self.host
 
-        if image_id:
-            status = 'downloading'
-        else:
-            status = 'available'
+        status = 'available'
+        model_update = False
 
         try:
             vol_name = volume_ref['name']
             vol_size = volume_ref['size']
             LOG.debug(_("volume %(vol_name)s: creating lv of"
                     " size %(vol_size)sG") % locals())
-            if snapshot_id is None:
+            if snapshot_id is None and image_id is None:
                 model_update = self.driver.create_volume(volume_ref)
-            else:
+            elif snapshot_id is not None:
                 snapshot_ref = self.db.snapshot_get(context, snapshot_id)
                 model_update = self.driver.create_volume_from_snapshot(
                     volume_ref,
                     snapshot_ref)
+            else:
+                # create the volume from an image
+                image_service, image_id = \
+                               glance.get_remote_image_service(context,
+                                                               image_id)
+                image_location = image_service.get_location(context, image_id)
+                cloned = self.driver.clone_image(volume_ref, image_location)
+                if not cloned:
+                    model_update = self.driver.create_volume(volume_ref)
+                    status = 'downloading'
+
             if model_update:
                 self.db.volume_update(context, volume_ref['id'], model_update)
 
@@ -170,7 +179,7 @@ class VolumeManager(manager.SchedulerDependentManager):
         self._reset_stats()
         self._notify_about_volume_usage(context, volume_ref, "create.end")
 
-        if image_id:
+        if image_id and not cloned:
             #copy the image onto the volume.
             self._copy_image_to_volume(context, volume_ref, image_id)
         return volume_id