improve efficiency of image transfer during migration

This reduces time to transfer a qcow2 image
with a virtual size of 10G, over GigE,
from about 7 minutes to about 30 seconds.

There are multiple inefficiencies in the existing process.
Taking an example of a qcow2 image with 10G virtual size,
the process was:

  qcow2 -> raw -> read -> send -> write -> qcow2

qcow2 to raw takes 20s,
transfer of the resultant 10G is another 4m9s, and
conversion back to qcow takes 2m33s.
I.E. a total of about 7 minutes.

So instead we try to avoid the initial qcow2 to raw
conversion completely, which results in the whole
process completing in about 30s, in the common
case where no conversion to raw is done on the destination.

We also optimize the case where the source qcow2
image doesn't have a backing file, and then directly
copy the source image without merging a backing file.

Note this will also improve the situation when
resizing/migrating within the same host as
needles conversions are avoided in that case too.

We also optimize the case where raw images are being used
by trying to use `rsync -Sz` rather than `scp`.
That compresses runs of zeros and create sparse destination files.
Testing a 10G raw image showed a saving of 30s in transfer time.
Also the network was greatly reduced (corresponding to holes
in the source), as was space usage at the destination.
This gain is limited though by rsync inefficiently reading
all the holes at the source:
https://bugzilla.samba.org/show_bug.cgi?id=8918

Thanks to David Naori <dnaori@redhat.com> for testing and ideas.

Change-Id: I9e87f912ef2717221c244241cda2f1027a4ca66a

4 files changed, 88 insertions, 24 deletions

diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py
index 16dcd338b..741ef566d 100644
--- a/nova/tests/test_libvirt.py
+++ b/nova/tests/test_libvirt.py
@@ -3485,6 +3485,9 @@ class LibvirtDriverTestCase(test.TestCase):
                       'local_gb': 10, 'backing_file': '/base/disk.local'}]
         disk_info_text = jsonutils.dumps(disk_info)
 
+        def fake_can_resize_fs(path, size, use_cow=False):
+            return False
+
         def fake_extend(path, size):
             pass
 
@@ -3513,6 +3516,8 @@ class LibvirtDriverTestCase(test.TestCase):
 
         self.flags(use_cow_images=True)
         self.stubs.Set(libvirt_driver.disk, 'extend', fake_extend)
+        self.stubs.Set(libvirt_driver.disk, 'can_resize_fs',
+                       fake_can_resize_fs)
         self.stubs.Set(self.libvirtconnection, 'to_xml', fake_to_xml)
         self.stubs.Set(self.libvirtconnection, 'plug_vifs', fake_plug_vifs)
         self.stubs.Set(self.libvirtconnection, '_create_image',
diff --git a/nova/virt/disk/api.py b/nova/virt/disk/api.py
index cf3b2f894..54d974534 100644
--- a/nova/virt/disk/api.py
+++ b/nova/virt/disk/api.py
@@ -124,6 +124,31 @@ def extend(image, size):
     resize2fs(image)
 
 
+def can_resize_fs(image, size, use_cow=False):
+    """Check whether we can resize contained file system."""
+
+    # Check that we're increasing the size
+    virt_size = get_image_virtual_size(image)
+    if virt_size >= size:
+        return False
+
+    # Check the image is unpartitioned
+    if use_cow:
+        # Try to mount an unpartitioned qcow2 image
+        try:
+            inject_data(image, use_cow=True)
+        except exception.NovaException:
+            return False
+    else:
+        # For raw, we can directly inspect the file system
+        try:
+            utils.execute('e2label', image)
+        except exception.ProcessExecutionError:
+            return False
+
+    return True
+
+
 def bind(src, target, instance_name):
     """Bind device to a filesytem"""
     if src:
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 6a68f5cd7..5da82fc65 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -2763,7 +2763,6 @@ class LibvirtDriver(driver.ComputeDriver):
         self.power_off(instance)
 
         # copy disks to destination
-        # if disk type is qcow2, convert to raw then send to dest.
         # rename instance dir to +_resize at first for using
         # shared storage for instance dir (eg. NFS).
         same_host = (dest == self.get_host_ip_addr())
@@ -2772,28 +2771,29 @@ class LibvirtDriver(driver.ComputeDriver):
         try:
             utils.execute('mv', inst_base, inst_base_resize)
             if same_host:
+                dest = None
                 utils.execute('mkdir', '-p', inst_base)
             else:
                 utils.execute('ssh', dest, 'mkdir', '-p', inst_base)
             for info in disk_info:
                 # assume inst_base == dirname(info['path'])
-                to_path = "%s:%s" % (dest, info['path'])
-                fname = os.path.basename(info['path'])
+                img_path = info['path']
+                fname = os.path.basename(img_path)
                 from_path = os.path.join(inst_base_resize, fname)
-                if info['type'] == 'qcow2':
+                if info['type'] == 'qcow2' and info['backing_file']:
                     tmp_path = from_path + "_rbase"
+                    # merge backing file
                     utils.execute('qemu-img', 'convert', '-f', 'qcow2',
-                                  '-O', 'raw', from_path, tmp_path)
+                                  '-O', 'qcow2', from_path, tmp_path)
+
                     if same_host:
-                        utils.execute('mv', tmp_path, info['path'])
+                        utils.execute('mv', tmp_path, img_path)
                     else:
-                        utils.execute('scp', tmp_path, to_path)
+                        libvirt_utils.copy_image(tmp_path, img_path, host=dest)
                         utils.execute('rm', '-f', tmp_path)
-                else:  # raw
-                    if same_host:
-                        utils.execute('cp', from_path, info['path'])
-                    else:
-                        utils.execute('scp', from_path, to_path)
+
+                else:  # raw or qcow2 with no backing file
+                    libvirt_utils.copy_image(from_path, img_path, host=dest)
         except Exception, e:
             try:
                 if os.path.exists(inst_base_resize):
@@ -2828,12 +2828,28 @@ class LibvirtDriver(driver.ComputeDriver):
         for info in disk_info:
             fname = os.path.basename(info['path'])
             if fname == 'disk':
-                disk.extend(info['path'],
-                            instance['root_gb'] * 1024 * 1024 * 1024)
+                size = instance['root_gb']
             elif fname == 'disk.local':
-                disk.extend(info['path'],
-                            instance['ephemeral_gb'] * 1024 * 1024 * 1024)
-            if FLAGS.use_cow_images:
+                size = instance['ephemeral_gb']
+            else:
+                size = 0
+            size *= 1024 * 1024 * 1024
+
+            # If we have a non partitioned image that we can extend
+            # then ensure we're in 'raw' format so we can extend file system.
+            fmt = info['type']
+            if (size and fmt == 'qcow2' and
+                disk.can_resize_fs(info['path'], size, use_cow=True)):
+                path_raw = info['path'] + '_raw'
+                utils.execute('qemu-img', 'convert', '-f', 'qcow2',
+                              '-O', 'raw', info['path'], path_raw)
+                utils.execute('mv', path_raw, info['path'])
+                fmt = 'raw'
+
+            if size:
+                disk.extend(info['path'], size)
+
+            if fmt == 'raw' and FLAGS.use_cow_images:
                 # back to qcow2 (no backing_file though) so that snapshot
                 # will be available
                 path_qcow = info['path'] + '_qcow'
diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py
index f08485ea3..7edc76122 100644
--- a/nova/virt/libvirt/utils.py
+++ b/nova/virt/libvirt/utils.py
@@ -180,17 +180,35 @@ def get_disk_backing_file(path):
     return backing_file
 
 
-def copy_image(src, dest):
-    """Copy a disk image
+def copy_image(src, dest, host=None):
+    """Copy a disk image to an existing directory
 
     :param src: Source image
     :param dest: Destination path
+    :param host: Remote host
     """
-    # We shell out to cp because that will intelligently copy
-    # sparse files.  I.E. holes will not be written to DEST,
-    # rather recreated efficiently.  In addition, since
-    # coreutils 8.11, holes can be read efficiently too.
-    execute('cp', src, dest)
+
+    if not host:
+        # We shell out to cp because that will intelligently copy
+        # sparse files.  I.E. holes will not be written to DEST,
+        # rather recreated efficiently.  In addition, since
+        # coreutils 8.11, holes can be read efficiently too.
+        execute('cp', src, dest)
+    else:
+        dest = "%s:%s" % (host, dest)
+        # Try rsync first as that can compress and create sparse dest files.
+        # Note however that rsync currently doesn't read sparse files
+        # efficiently: https://bugzilla.samba.org/show_bug.cgi?id=8918
+        # At least network traffic is mitigated with compression.
+        try:
+            # Do a relatively light weight test first, so that we
+            # can fall back to scp, without having run out of space
+            # on the destination for example.
+            execute('rsync', '--sparse', '--compress', '--dry-run', src, dest)
+        except exception.ProcessExecutionError:
+            execute('scp', src, dest)
+        else:
+            execute('rsync', '--sparse', '--compress', src, dest)
 
 
 def mkfs(fs, path, label=None):