Libvirt: Add support for live snapshots

blueprint libvirt-live-snapshots

Implements support for live snapshots when QEMU 1.3 and Libvirt
1.0.0 are available. This implementation may function with other
hypervisors, at which point _check_use_live_snapshots can be
adjusted accordingly.

At a high level, the process is as follows. First we create a
destination file for the snapshot. This file uses the same backing
file as the origin disk and is of the same virtual size. We do this
since the combination of blockRebase and "shallow" copies are the
only bug-free method available as of QEMU 1.3 (Vish has submitted
patches upstream). This process ultimately produces a CoW file,
representing only the current delta between the root disk and
backing file. The final step converts the CoW delta to a backless
image matching the format of the origin disk.

Flags: DocImpact
Change-Id: Icba53336556295722c3f8b7345e6cd09d8ccfed9

2 files changed, 144 insertions, 29 deletions

diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index c865e4b3a..9ed7a054c 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -47,6 +47,7 @@ import os
 import shutil
 import sys
 import tempfile
+import time
 import uuid
 
 from eventlet import greenthread
@@ -254,6 +255,10 @@ MIN_LIBVIRT_VERSION = (0, 9, 6)
 # When the above version matches/exceeds this version
 # delete it & corresponding code using it
 MIN_LIBVIRT_HOST_CPU_VERSION = (0, 9, 10)
+# Live snapshot requirements
+REQ_HYPERVISOR_LIVESNAPSHOT = "QEMU"
+MIN_LIBVIRT_LIVESNAPSHOT_VERSION = (1, 0, 0)
+MIN_QEMU_LIVESNAPSHOT_VERSION = (1, 3, 0)
 
 
 def _get_eph_disk(ephemeral):
@@ -325,16 +330,29 @@ class LibvirtDriver(driver.ComputeDriver):
             self._host_state = HostState(self.virtapi, self.read_only)
         return self._host_state
 
-    def has_min_version(self, ver):
-        libvirt_version = self._conn.getLibVersion()
-
+    def has_min_version(self, lv_ver=None, hv_ver=None, hv_type=None):
         def _munge_version(ver):
             return ver[0] * 1000000 + ver[1] * 1000 + ver[2]
 
-        if libvirt_version < _munge_version(ver):
-            return False
+        try:
+            if lv_ver is not None:
+                libvirt_version = self._conn.getLibVersion()
+                if libvirt_version < _munge_version(lv_ver):
+                    return False
 
-        return True
+            if hv_ver is not None:
+                hypervisor_version = self._conn.getVersion()
+                if hypervisor_version < _munge_version(hv_ver):
+                    return False
+
+            if hv_type is not None:
+                hypervisor_type = self._conn.getType()
+                if hypervisor_type != hv_type:
+                    return False
+
+            return True
+        except Exception:
+            return False
 
     def init_host(self, host):
         if not self.has_min_version(MIN_LIBVIRT_VERSION):
@@ -806,35 +824,64 @@ class LibvirtDriver(driver.ComputeDriver):
         (state, _max_mem, _mem, _cpus, _t) = virt_dom.info()
         state = LIBVIRT_POWER_STATE[state]
 
+        # NOTE(rmk): Live snapshots require QEMU 1.3 and Libvirt 1.0.0.
+        #            These restrictions can be relaxed as other configurations
+        #            can be validated.
+        if self.has_min_version(MIN_LIBVIRT_LIVESNAPSHOT_VERSION,
+                                MIN_QEMU_LIVESNAPSHOT_VERSION,
+                                REQ_HYPERVISOR_LIVESNAPSHOT):
+            live_snapshot = True
+        else:
+            live_snapshot = False
+
+        # NOTE(rmk): We cannot perform live snapshots when a managedSave
+        #            file is present, so we will use the cold/legacy method
+        #            for instances which are shutdown.
+        if state == power_state.SHUTDOWN:
+            live_snapshot = False
+
         # NOTE(dkang): managedSave does not work for LXC
-        if CONF.libvirt_type != 'lxc':
+        if CONF.libvirt_type != 'lxc' and not live_snapshot:
             if state == power_state.RUNNING or state == power_state.PAUSED:
                 virt_dom.managedSave(0)
 
-        # Make the snapshot
-        snapshot = self.image_backend.snapshot(disk_path, snapshot_name,
-                                               image_type=source_format)
+        if live_snapshot:
+            LOG.info(_("Beginning live snapshot process"),
+                     instance=instance)
+        else:
+            LOG.info(_("Beginning cold snapshot process"),
+                     instance=instance)
+            snapshot = self.image_backend.snapshot(disk_path, snapshot_name,
+                                                   image_type=source_format)
+            snapshot.create()
 
-        snapshot.create()
         update_task_state(task_state=task_states.IMAGE_PENDING_UPLOAD)
-
-        # Export the snapshot to a raw image
         snapshot_directory = CONF.libvirt_snapshots_directory
         fileutils.ensure_tree(snapshot_directory)
         with utils.tempdir(dir=snapshot_directory) as tmpdir:
             try:
                 out_path = os.path.join(tmpdir, snapshot_name)
-                snapshot.extract(out_path, image_format)
+                if live_snapshot:
+                    # NOTE (rmk): libvirt needs to be able to write to the
+                    #             temp directory, which is owned nova.
+                    utils.execute('chmod', '777', tmpdir, run_as_root=True)
+                    self._live_snapshot(virt_dom, disk_path, out_path,
+                                        image_format)
+                else:
+                    snapshot.extract(out_path, image_format)
             finally:
-                snapshot.delete()
+                if not live_snapshot:
+                    snapshot.delete()
                 # NOTE(dkang): because previous managedSave is not called
                 #              for LXC, _create_domain must not be called.
-                if CONF.libvirt_type != 'lxc':
+                if CONF.libvirt_type != 'lxc' and not live_snapshot:
                     if state == power_state.RUNNING:
                         self._create_domain(domain=virt_dom)
                     elif state == power_state.PAUSED:
                         self._create_domain(domain=virt_dom,
                                 launch_flags=libvirt.VIR_DOMAIN_START_PAUSED)
+                LOG.info(_("Snapshot extracted, beginning image upload"),
+                         instance=instance)
 
             # Upload that image to the image service
 
@@ -845,6 +892,72 @@ class LibvirtDriver(driver.ComputeDriver):
                                      image_href,
                                      metadata,
                                      image_file)
+                LOG.info(_("Snapshot image upload complete"),
+                         instance=instance)
+
+    def _live_snapshot(self, domain, disk_path, out_path, image_format):
+        """Snapshot an instance without downtime."""
+        # Save a copy of the domain's running XML file
+        xml = domain.XMLDesc(0)
+
+        # Abort is an idempotent operation, so make sure any block
+        # jobs which may have failed are ended.
+        try:
+            domain.blockJobAbort(disk_path, 0)
+        except Exception:
+            pass
+
+        def _wait_for_block_job(domain, disk_path):
+            status = domain.blockJobInfo(disk_path, 0)
+            try:
+                cur = status.get('cur', 0)
+                end = status.get('end', 0)
+            except Exception:
+                return False
+
+            if cur == end and cur != 0 and end != 0:
+                return False
+            else:
+                return True
+
+        # NOTE (rmk): We are using shallow rebases as a workaround to a bug
+        #             in QEMU 1.3. In order to do this, we need to create
+        #             a destination image with the original backing file
+        #             and matching size of the instance root disk.
+        src_disk_size = libvirt_utils.get_disk_size(disk_path)
+        src_back_path = libvirt_utils.get_disk_backing_file(disk_path,
+                                                            basename=False)
+        disk_delta = out_path + '.delta'
+        libvirt_utils.create_cow_image(src_back_path, disk_delta,
+                                       src_disk_size)
+
+        try:
+            # NOTE (rmk): blockRebase cannot be executed on persistent
+            #             domains, so we need to temporarily undefine it.
+            #             If any part of this block fails, the domain is
+            #             re-defined regardless.
+            if domain.isPersistent():
+                domain.undefine()
+
+            # NOTE (rmk): Establish a temporary mirror of our root disk and
+            #             issue an abort once we have a complete copy.
+            domain.blockRebase(disk_path, disk_delta, 0,
+                               libvirt.VIR_DOMAIN_BLOCK_REBASE_COPY |
+                               libvirt.VIR_DOMAIN_BLOCK_REBASE_REUSE_EXT |
+                               libvirt.VIR_DOMAIN_BLOCK_REBASE_SHALLOW)
+
+            while _wait_for_block_job(domain, disk_path):
+                time.sleep(0.5)
+
+            domain.blockJobAbort(disk_path, 0)
+            libvirt_utils.chown(disk_delta, os.getuid())
+        finally:
+            self._conn.defineXML(xml)
+
+        # Convert the delta (CoW) image with a backing file to a flat
+        # image with no backing file.
+        libvirt_utils.extract_snapshot(disk_delta, 'qcow2', None,
+                                       out_path, image_format)
 
     def reboot(self, instance, network_info, reboot_type='SOFT',
                block_device_info=None):
diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py
index 4b3517da7..bd4ec685c 100644
--- a/nova/virt/libvirt/utils.py
+++ b/nova/virt/libvirt/utils.py
@@ -63,7 +63,7 @@ def create_image(disk_format, path, size):
     execute('qemu-img', 'create', '-f', disk_format, path, size)
 
 
-def create_cow_image(backing_file, path):
+def create_cow_image(backing_file, path, size=None):
     """Create COW image
 
     Creates a COW image with the given backing file
@@ -89,6 +89,8 @@ def create_cow_image(backing_file, path):
     #     cow_opts += ['preallocation=%s' % base_details['preallocation']]
     if base_details and base_details.encryption:
         cow_opts += ['encryption=%s' % base_details.encryption]
+    if size is not None:
+        cow_opts += ['size=%s' % size]
     if cow_opts:
         # Format as a comma separated list
         csv_opts = ",".join(cow_opts)
@@ -292,14 +294,14 @@ def get_disk_size(path):
     return int(size)
 
 
-def get_disk_backing_file(path):
+def get_disk_backing_file(path, basename=True):
     """Get the backing file of a disk image
 
     :param path: Path to the disk image
     :returns: a path to the image's backing store
     """
     backing_file = images.qemu_img_info(path).backing_file
-    if backing_file:
+    if backing_file and basename:
         backing_file = os.path.basename(backing_file)
 
     return backing_file
@@ -403,16 +405,16 @@ def extract_snapshot(disk_path, source_fmt, snapshot_name, out_path, dest_fmt):
     # NOTE(markmc): ISO is just raw to qemu-img
     if dest_fmt == 'iso':
         dest_fmt = 'raw'
-    qemu_img_cmd = ('qemu-img',
-                    'convert',
-                    '-f',
-                    source_fmt,
-                    '-O',
-                    dest_fmt,
-                    '-s',
-                    snapshot_name,
-                    disk_path,
-                    out_path)
+
+    qemu_img_cmd = ('qemu-img', 'convert', '-f', source_fmt, '-O',
+                    dest_fmt, '-s', snapshot_name, disk_path, out_path)
+
+    # When snapshot name is omitted we do a basic convert, which
+    # is used by live snapshots.
+    if snapshot_name is None:
+        qemu_img_cmd = ('qemu-img', 'convert', '-f', source_fmt, '-O',
+                        dest_fmt, disk_path, out_path)
+
     execute(*qemu_img_cmd)