Remove VDI chain limit for migrations.

The strategy for removing the limit is to refactor migration so that
they work nearly identically to snapshots, meaning sequence-numbered
VHDs are rsynced over into a staging-area and then imported into the SR
using the `import_vhds` function.

Change-Id: Ibf5c82c52ae7d505ea9e54d64fcc8b8fdce4d05d

3 files changed, 34 insertions, 75 deletions

diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
index 6fbd14714..12616be40 100755
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
@@ -231,7 +231,7 @@ def upload_vhd(session, args):
 
     staging_path = utils.make_staging_area(sr_path)
     try:
-        utils.prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids)
+        utils.prepare_staging_area(sr_path, staging_path, vdi_uuids)
         _upload_tarball(staging_path, image_id, glance_host, glance_port,
                         auth_token, properties)
     finally:
diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration
index 42f8eaef8..85ad0bdbc 100755
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration
@@ -20,6 +20,10 @@ XenAPI Plugin for transfering data between host nodes
 """
 
 import cPickle as pickle
+try:
+    import json
+except ImportError:
+    import simplejson as json
 import os
 import os.path
 import shlex
@@ -28,71 +32,37 @@ import subprocess
 
 import XenAPIPlugin
 
+import utils
+
 from pluginlib_nova import *
 configure_logging('migration')
 
 
-def move_file(item, src, dst):
-    """Move file with logging."""
-    #NOTE(markwash): shutil.move can be less efficient than it should be if
-    #                dst is a directory. See http://bugs.python.org/issue1577.
-    if os.path.isdir(dst):
-        dst = os.path.join(dst, os.path.basename(src))
-    logging.debug('Moving %(item)s: %(src)s -> %(dst)s' % locals())
-    shutil.move(src, dst)
-
-
 def move_vhds_into_sr(session, args):
     """Moves the VHDs from their copied location to the SR"""
     params = pickle.loads(exists(args, 'params'))
     instance_uuid = params['instance_uuid']
-
     sr_path = params['sr_path']
-    sr_temp_path = "%s/tmp" % sr_path
-    temp_vhd_path = "%s/instance%s" % (sr_temp_path, instance_uuid)
-
-    logging.debug('Creating temporary SR path %s' % temp_vhd_path)
-    os.makedirs(temp_vhd_path)
-
-    # Discover the copied VHDs locally, and then set up paths to copy
-    # them to under the SR
-    source_image_path = "/images/instance%s" % instance_uuid
-
-    old_base_copy_uuid = params['old_base_copy_uuid']
-    new_base_copy_uuid = params['new_base_copy_uuid']
-    source_base_copy_path = "%s/%s.vhd" % (source_image_path,
-            old_base_copy_uuid)
-    new_base_copy_path = "%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid)
-
-    move_file('base', source_base_copy_path, new_base_copy_path)
-
-    if 'old_cow_uuid' in params:
-        old_cow_uuid = params['old_cow_uuid']
-        new_cow_uuid = params['new_cow_uuid']
+    uuid_stack = params['uuid_stack']
 
-        source_cow_path = "%s/%s.vhd" % (source_image_path, old_cow_uuid)
-        new_cow_path = "%s/%s.vhd" % (temp_vhd_path, new_cow_uuid)
+    staging_path = "/images/instance%s" % instance_uuid
+    imported_vhds = utils.import_vhds(sr_path, staging_path, uuid_stack)
+    utils.cleanup_staging_area(staging_path)
+    return json.dumps(imported_vhds)
 
-        move_file('COW', source_cow_path, new_cow_path)
 
-        # Link the COW to the base copy
-        logging.debug('Attaching COW to the base %s -> %s' %
-                      (new_cow_path, new_base_copy_path))
-        subprocess.call(['/usr/sbin/vhd-util', 'modify',
-                         '-n', new_cow_path, '-p', new_base_copy_path])
-
-        # NOTE(sirp): COW should be copied before base_copy to avoid
-        # snapwatchd GC'ing an unreferenced base copy VDI
-        move_file('COW', new_cow_path, sr_path)
+def _rsync_vhds(instance_uuid, host, staging_path, user="root"):
+    ssh_cmd = '\"ssh -o StrictHostKeyChecking=no\"'
 
-    move_file('base', new_base_copy_path, sr_path)
+    if not staging_path.endswith('/'):
+        staging_path += '/'
 
-    logging.debug('Cleaning up source path %s' % source_image_path)
-    os.rmdir(source_image_path)
+    dest_path = '%s@%s:/images/instance%s/' % (user, host, instance_uuid)
 
-    logging.debug('Cleaning up temporary SR path %s' % temp_vhd_path)
-    os.rmdir(temp_vhd_path)
-    return ""
+    rsync_cmd = "nohup /usr/bin/rsync -av -e %(ssh_cmd)s %(staging_path)s"\
+               " %(dest_path)s" % locals()
+    rsync_proc = utils.make_subprocess(rsync_cmd, stdout=True, stderr=True)
+    utils.finish_subprocess(rsync_proc, rsync_cmd)
 
 
 def transfer_vhd(session, args):
@@ -102,32 +72,19 @@ def transfer_vhd(session, args):
     host = params['host']
     vdi_uuid = params['vdi_uuid']
     sr_path = params['sr_path']
-    vhd_path = "%s.vhd" % vdi_uuid
-
-    source_path = "%s/%s" % (sr_path, vhd_path)
-    dest_user = 'root'
-    dest_path = '%s@%s:/images/instance%s/' % (dest_user, host, instance_uuid)
-
-    logging.debug("Preparing to transmit %s to %s" % (source_path,
-            dest_path))
-
-    ssh_cmd = '\"ssh -o StrictHostKeyChecking=no\"'
-
-    # NOTE(dprince): shlex python 2.4 doesn't like unicode so we
-    # explicitly convert to ascii
-    rsync_args = shlex.split(('nohup /usr/bin/rsync -av -e %s %s %s'
-            % (ssh_cmd, source_path, dest_path)).encode('ascii'))
+    seq_num = params['seq_num']
 
-    logging.debug('rsync %s' % (' '.join(rsync_args, )))
+    staging_path = utils.make_staging_area(sr_path)
+    try:
+        utils.prepare_staging_area(
+                sr_path, staging_path, [vdi_uuid], seq_num=seq_num)
+        _rsync_vhds(instance_uuid, host, staging_path)
+    finally:
+        utils.cleanup_staging_area(staging_path)
 
-    rsync_proc = subprocess.Popen(rsync_args, stdout=subprocess.PIPE)
-    logging.debug('Rsync output: \n %s' % rsync_proc.communicate()[0])
-    logging.debug('Rsync return: %d' % rsync_proc.returncode)
-    if rsync_proc.returncode != 0:
-        raise Exception("Unexpected VHD transfer failure")
     return ""
 
 
 if __name__ == '__main__':
     XenAPIPlugin.dispatch({'transfer_vhd': transfer_vhd,
-            'move_vhds_into_sr': move_vhds_into_sr, })
+                           'move_vhds_into_sr': move_vhds_into_sr})
diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/utils.py b/plugins/xenserver/xenapi/etc/xapi.d/plugins/utils.py
index 874d66fb5..20401b968 100644
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/utils.py
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/utils.py
@@ -28,6 +28,9 @@ CHUNK_SIZE = 8192
 def make_subprocess(cmdline, stdout=False, stderr=False, stdin=False):
     """Make a subprocess according to the given command-line string
     """
+    # NOTE(dprince): shlex python 2.4 doesn't like unicode so we
+    # explicitly convert to ascii
+    cmdline = cmdline.encode('ascii')
     logging.info("Running cmd '%s'" % cmdline)
     kwargs = {}
     kwargs['stdout'] = stdout and subprocess.PIPE or None
@@ -248,9 +251,8 @@ def import_vhds(sr_path, staging_path, uuid_stack):
     return imported_vhds
 
 
-def prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids):
+def prepare_staging_area(sr_path, staging_path, vdi_uuids, seq_num=0):
     """Hard-link VHDs into staging area."""
-    seq_num = 0
     for vdi_uuid in vdi_uuids:
         source = os.path.join(sr_path, "%s.vhd" % vdi_uuid)
         link_name = os.path.join(staging_path, "%d.vhd" % seq_num)