Fixes Hyper-V live migration with attached volumes

Fixes bug: 1153429

The previous Hyper-V driver live migration implementation expects to find iSCSI
devices mounted on the same path on source and destination, which is not an
option in this context.

In order to be able to live migrate instances with attached volumes, this fix
provides the following behavior, based on the creation of a staged VM on the
target, so called "planned" in the Hyper-V documentation and in this patch.

pre_live_migration

The target host logs into the storage targets.

live_migration

The source host:

1) checks and removes a previously created planned VM for the current
   instance if present on the target
2) creates a planned VM on the target by using the Hyper-V WMI API
3) maps the volume devices on the planned VM based on the target host
   devices
4) live migrates the source VM on the planned VM
5) logs off the storage targets on the source

This solution provides live migration of volumes without needing to pause
the VM and detach / attach the volumes, preserving also the atomicity of the
live operation.

In the case in which no volumes are attached to the VM, live migration is
performed without creating explicitly a planned VM, starting from step 4
on the source in the above list.

Unit tests have been added for this scenario.

Change-Id: Ib634b77894f492896d86dce65a7269ece8f3d55b

1 files changed, 145 insertions, 74 deletions

diff --git a/nova/tests/test_hypervapi.py b/nova/tests/test_hypervapi.py
index c6d75aea1..0ddfe080d 100644
--- a/nova/tests/test_hypervapi.py
+++ b/nova/tests/test_hypervapi.py
@@ -42,6 +42,7 @@ from nova.tests.image import fake as fake_image
 from nova.tests import matchers
 from nova import utils
 from nova.virt import configdrive
+from nova.virt import driver
 from nova.virt.hyperv import basevolumeutils
 from nova.virt.hyperv import constants
 from nova.virt.hyperv import driver as driver_hyperv
@@ -51,6 +52,7 @@ from nova.virt.hyperv import networkutils
 from nova.virt.hyperv import pathutils
 from nova.virt.hyperv import vhdutils
 from nova.virt.hyperv import vmutils
+from nova.virt.hyperv import volumeops
 from nova.virt.hyperv import volumeutils
 from nova.virt.hyperv import volumeutilsv2
 from nova.virt import images
@@ -88,10 +90,6 @@ class HyperVAPITestCase(test.TestCase):
         self.flags(instances_path=r'C:\Hyper-V\test\instances',
                    network_api_class='nova.network.quantumv2.api.API')
 
-        self.flags(vswitch_name='external',
-                   force_volumeutils_v1=True,
-                   group='hyperv')
-
         self._conn = driver_hyperv.HyperVDriver(None)
 
     def _setup_stubs(self):
@@ -118,6 +116,14 @@ class HyperVAPITestCase(test.TestCase):
             pass
         self.stubs.Set(time, 'sleep', fake_sleep)
 
+        def fake_vmutils__init__(self, host='.'):
+            pass
+        vmutils.VMUtils.__init__ = fake_vmutils__init__
+
+        def fake_get_volume_utils(self):
+            return volumeutils.VolumeUtils()
+        volumeops.VolumeOps._get_volume_utils = fake_get_volume_utils
+
         self.stubs.Set(pathutils, 'PathUtils', fake.PathUtils)
         self._mox.StubOutWithMock(fake.PathUtils, 'open')
         self._mox.StubOutWithMock(fake.PathUtils, 'copyfile')
@@ -141,7 +147,7 @@ class HyperVAPITestCase(test.TestCase):
         self._mox.StubOutWithMock(vmutils.VMUtils, 'take_vm_snapshot')
         self._mox.StubOutWithMock(vmutils.VMUtils, 'remove_vm_snapshot')
         self._mox.StubOutWithMock(vmutils.VMUtils, 'set_nic_connection')
-        self._mox.StubOutWithMock(vmutils.VMUtils, 'get_vm_iscsi_controller')
+        self._mox.StubOutWithMock(vmutils.VMUtils, 'get_vm_scsi_controller')
         self._mox.StubOutWithMock(vmutils.VMUtils, 'get_vm_ide_controller')
         self._mox.StubOutWithMock(vmutils.VMUtils, 'get_attached_disks_count')
         self._mox.StubOutWithMock(vmutils.VMUtils,
@@ -150,6 +156,8 @@ class HyperVAPITestCase(test.TestCase):
                                   'get_mounted_disk_by_drive_number')
         self._mox.StubOutWithMock(vmutils.VMUtils, 'detach_vm_disk')
         self._mox.StubOutWithMock(vmutils.VMUtils, 'get_vm_storage_paths')
+        self._mox.StubOutWithMock(vmutils.VMUtils,
+                                  'get_controller_volume_paths')
 
         self._mox.StubOutWithMock(vhdutils.VHDUtils, 'create_differencing_vhd')
         self._mox.StubOutWithMock(vhdutils.VHDUtils, 'reconnect_parent_vhd')
@@ -183,6 +191,8 @@ class HyperVAPITestCase(test.TestCase):
                                   'get_session_id_from_mounted_disk')
         self._mox.StubOutWithMock(basevolumeutils.BaseVolumeUtils,
                                   'get_device_number_for_target')
+        self._mox.StubOutWithMock(basevolumeutils.BaseVolumeUtils,
+                                  'get_target_from_disk_path')
 
         self._mox.StubOutWithMock(volumeutils.VolumeUtils,
                                   'login_storage_target')
@@ -523,16 +533,21 @@ class HyperVAPITestCase(test.TestCase):
         self._conn.destroy(self._instance_data, None)
         self._mox.VerifyAll()
 
-    def test_live_migration(self):
-        self._test_live_migration(False)
+    def test_live_migration_without_volumes(self):
+        self._test_live_migration()
+
+    def test_live_migration_with_volumes(self):
+        self._test_live_migration(with_volumes=True)
 
     def test_live_migration_with_target_failure(self):
-        self._test_live_migration(True)
+        self._test_live_migration(test_failure=True)
 
-    def _test_live_migration(self, test_failure):
+    def _test_live_migration(self, test_failure=False,
+                             with_volumes=False):
         dest_server = 'fake_server'
 
         instance_data = self._get_instance_data()
+        instance_name = instance_data['name']
 
         fake_post_method = self._mox.CreateMockAnything()
         if not test_failure:
@@ -544,10 +559,27 @@ class HyperVAPITestCase(test.TestCase):
             fake_recover_method(self._context, instance_data, dest_server,
                                 False)
 
+        fake_ide_controller_path = 'fakeide'
+        fake_scsi_controller_path = 'fakescsi'
+
+        if with_volumes:
+            fake_scsi_disk_path = 'fake_scsi_disk_path'
+            fake_target_iqn = 'fake_target_iqn'
+            fake_target_lun = 1
+            fake_scsi_paths = {0: fake_scsi_disk_path}
+        else:
+            fake_scsi_paths = {}
+
         m = livemigrationutils.LiveMigrationUtils.live_migrate_vm(
             instance_data['name'], dest_server)
         if test_failure:
-            m.AndRaise(Exception('Simulated failure'))
+            m.AndRaise(vmutils.HyperVException('Simulated failure'))
+
+        if with_volumes:
+            m.AndReturn([(fake_target_iqn, fake_target_lun)])
+            volumeutils.VolumeUtils.logout_storage_target(fake_target_iqn)
+        else:
+            m.AndReturn([])
 
         self._mox.ReplayAll()
         try:
@@ -555,19 +587,22 @@ class HyperVAPITestCase(test.TestCase):
                                       dest_server, fake_post_method,
                                       fake_recover_method)
             exception_raised = False
-        except Exception:
+        except vmutils.HyperVException:
             exception_raised = True
 
         self.assertTrue(not test_failure ^ exception_raised)
         self._mox.VerifyAll()
 
     def test_pre_live_migration_cow_image(self):
-        self._test_pre_live_migration(True)
+        self._test_pre_live_migration(True, False)
 
     def test_pre_live_migration_no_cow_image(self):
-        self._test_pre_live_migration(False)
+        self._test_pre_live_migration(False, False)
 
-    def _test_pre_live_migration(self, cow):
+    def test_pre_live_migration_with_volumes(self):
+        self._test_pre_live_migration(False, True)
+
+    def _test_pre_live_migration(self, cow, with_volumes):
         self.flags(use_cow_images=cow)
 
         instance_data = self._get_instance_data()
@@ -591,9 +626,29 @@ class HyperVAPITestCase(test.TestCase):
             fake.PathUtils.copyfile(mox.IsA(str), mox.IsA(str))
             vhdutils.VHDUtils.resize_vhd(mox.IsA(str), mox.IsA(object))
 
+        if with_volumes:
+            block_device_info = db_fakes.get_fake_block_device_info(
+                self._volume_target_portal, self._volume_id)
+
+            mapping = driver.block_device_info_get_mapping(block_device_info)
+            data = mapping[0]['connection_info']['data']
+            target_lun = data['target_lun']
+            target_iqn = data['target_iqn']
+            target_portal = data['target_portal']
+
+            fake_mounted_disk = "fake_mounted_disk"
+            fake_device_number = 0
+
+            self._mock_login_storage_target(target_iqn, target_lun,
+                                            target_portal,
+                                            fake_mounted_disk,
+                                            fake_device_number)
+        else:
+            block_device_info = None
+
         self._mox.ReplayAll()
         self._conn.pre_live_migration(self._context, instance,
-                                      None, network_info)
+                                      block_device_info, network_info)
         self._mox.VerifyAll()
 
         if cow:
@@ -734,7 +789,8 @@ class HyperVAPITestCase(test.TestCase):
         return image_path == self._fetched_image
 
     def _setup_create_instance_mocks(self, setup_vif_mocks_func=None,
-                                     boot_from_volume=False):
+                                     boot_from_volume=False,
+                                     block_device_info=None):
         vmutils.VMUtils.create_vm(mox.Func(self._check_vm_name), mox.IsA(int),
                                   mox.IsA(int), mox.IsA(bool))
 
@@ -750,6 +806,16 @@ class HyperVAPITestCase(test.TestCase):
         m = vmutils.VMUtils.create_scsi_controller(func)
         m.InAnyOrder()
 
+        if boot_from_volume:
+            mapping = driver.block_device_info_get_mapping(block_device_info)
+            data = mapping[0]['connection_info']['data']
+            target_lun = data['target_lun']
+            target_iqn = data['target_iqn']
+            target_portal = data['target_portal']
+
+            self._mock_attach_volume(mox.Func(self._check_vm_name), target_iqn,
+                                     target_lun, target_portal, True)
+
         vmutils.VMUtils.create_nic(mox.Func(self._check_vm_name), mox.IsA(str),
                                    mox.IsA(str)).InAnyOrder()
 
@@ -787,7 +853,8 @@ class HyperVAPITestCase(test.TestCase):
                 fake.PathUtils.copyfile(mox.IsA(str), mox.IsA(str))
 
         self._setup_create_instance_mocks(setup_vif_mocks_func,
-                                          boot_from_volume)
+                                          boot_from_volume,
+                                          block_device_info)
 
         # TODO(alexpilotti) Based on where the exception is thrown
         # some of the above mock calls need to be skipped
@@ -818,41 +885,57 @@ class HyperVAPITestCase(test.TestCase):
         vhd_path = pathutils.PathUtils().get_vhd_path(self._test_vm_name)
         self.assertEquals(vhd_path, self._instance_ide_disks[0])
 
-    def test_attach_volume(self):
-        instance_data = self._get_instance_data()
-        instance_name = instance_data['name']
+    def _mock_get_mounted_disk_from_lun(self, target_iqn, target_lun,
+                                        fake_mounted_disk,
+                                        fake_device_number):
+        m = volumeutils.VolumeUtils.get_device_number_for_target(target_iqn,
+                                                                 target_lun)
+        m.AndReturn(fake_device_number)
 
-        connection_info = db_fakes.get_fake_volume_info_data(
-            self._volume_target_portal, self._volume_id)
-        data = connection_info['data']
-        target_lun = data['target_lun']
-        target_iqn = data['target_iqn']
-        target_portal = data['target_portal']
+        m = vmutils.VMUtils.get_mounted_disk_by_drive_number(
+            fake_device_number)
+        m.AndReturn(fake_mounted_disk)
 
-        mount_point = '/dev/sdc'
+    def _mock_login_storage_target(self, target_iqn, target_lun, target_portal,
+                                   fake_mounted_disk, fake_device_number):
+        m = volumeutils.VolumeUtils.get_device_number_for_target(target_iqn,
+                                                                 target_lun)
+        m.AndReturn(fake_device_number)
 
         volumeutils.VolumeUtils.login_storage_target(target_lun,
                                                      target_iqn,
                                                      target_portal)
 
+        self._mock_get_mounted_disk_from_lun(target_iqn, target_lun,
+                                             fake_mounted_disk,
+                                             fake_device_number)
+
+    def _mock_attach_volume(self, instance_name, target_iqn, target_lun,
+                            target_portal=None, boot_from_volume=False):
         fake_mounted_disk = "fake_mounted_disk"
         fake_device_number = 0
         fake_controller_path = 'fake_scsi_controller_path'
-        fake_free_slot = 1
 
-        m = volumeutils.VolumeUtils.get_device_number_for_target(target_iqn,
-                                                                 target_lun)
-        m.AndReturn(fake_device_number)
+        self._mock_login_storage_target(target_iqn, target_lun,
+                                        target_portal,
+                                        fake_mounted_disk,
+                                        fake_device_number)
 
-        m = vmutils.VMUtils.get_mounted_disk_by_drive_number(
-            fake_device_number)
-        m.AndReturn(fake_mounted_disk)
+        self._mock_get_mounted_disk_from_lun(target_iqn, target_lun,
+                                             fake_mounted_disk,
+                                             fake_device_number)
 
-        m = vmutils.VMUtils.get_vm_iscsi_controller(instance_name)
-        m.AndReturn(fake_controller_path)
+        if boot_from_volume:
+            m = vmutils.VMUtils.get_vm_ide_controller(instance_name, 0)
+            m.AndReturn(fake_controller_path)
+            fake_free_slot = 0
+        else:
+            m = vmutils.VMUtils.get_vm_scsi_controller(instance_name)
+            m.AndReturn(fake_controller_path)
 
-        m = vmutils.VMUtils.get_attached_disks_count(fake_controller_path)
-        m.AndReturn(fake_free_slot)
+            fake_free_slot = 1
+            m = vmutils.VMUtils.get_attached_disks_count(fake_controller_path)
+            m.AndReturn(fake_free_slot)
 
         m = vmutils.VMUtils.attach_volume_to_controller(instance_name,
                                                         fake_controller_path,
@@ -860,15 +943,8 @@ class HyperVAPITestCase(test.TestCase):
                                                         fake_mounted_disk)
         m.WithSideEffects(self._add_volume_disk)
 
-        self._mox.ReplayAll()
-        self._conn.attach_volume(connection_info, instance_data, mount_point)
-        self._mox.VerifyAll()
-
-        self.assertEquals(len(self._instance_volume_disks), 1)
-
-    def test_detach_volume(self):
+    def test_attach_volume(self):
         instance_data = self._get_instance_data()
-        instance_name = instance_data['name']
 
         connection_info = db_fakes.get_fake_volume_info_data(
             self._volume_target_portal, self._volume_id)
@@ -878,6 +954,18 @@ class HyperVAPITestCase(test.TestCase):
         target_portal = data['target_portal']
         mount_point = '/dev/sdc'
 
+        self._mock_attach_volume(instance_data['name'], target_iqn, target_lun,
+                                 target_portal)
+
+        self._mox.ReplayAll()
+        self._conn.attach_volume(connection_info, instance_data, mount_point)
+        self._mox.VerifyAll()
+
+        self.assertEquals(len(self._instance_volume_disks), 1)
+
+    def _mock_detach_volume(self, target_iqn, target_lun):
+        mount_point = '/dev/sdc'
+
         fake_mounted_disk = "fake_mounted_disk"
         fake_device_number = 0
         fake_free_slot = 1
@@ -893,11 +981,10 @@ class HyperVAPITestCase(test.TestCase):
 
         volumeutils.VolumeUtils.logout_storage_target(mox.IsA(str))
 
-        self._mox.ReplayAll()
-        self._conn.detach_volume(connection_info, instance_data, mount_point)
-        self._mox.VerifyAll()
+    def test_detach_volume(self):
+        instance_data = self._get_instance_data()
+        instance_name = instance_data['name']
 
-    def test_boot_from_volume(self):
         connection_info = db_fakes.get_fake_volume_info_data(
             self._volume_target_portal, self._volume_id)
         data = connection_info['data']
@@ -905,33 +992,17 @@ class HyperVAPITestCase(test.TestCase):
         target_iqn = data['target_iqn']
         target_portal = data['target_portal']
 
-        block_device_info = db_fakes.get_fake_block_device_info(
-            self._volume_target_portal, self._volume_id)
-
-        fake_mounted_disk = "fake_mounted_disk"
-        fake_device_number = 0
-        fake_controller_path = 'fake_scsi_controller_path'
-
-        volumeutils.VolumeUtils.login_storage_target(target_lun,
-                                                     target_iqn,
-                                                     target_portal)
+        mount_point = '/dev/sdc'
 
-        m = volumeutils.VolumeUtils.get_device_number_for_target(target_iqn,
-                                                                 target_lun)
-        m.AndReturn(fake_device_number)
+        self._mock_detach_volume(target_iqn, target_lun)
 
-        m = vmutils.VMUtils.get_mounted_disk_by_drive_number(
-            fake_device_number)
-        m.AndReturn(fake_mounted_disk)
-
-        m = vmutils.VMUtils.get_vm_ide_controller(mox.IsA(str), mox.IsA(int))
-        m.AndReturn(fake_controller_path)
+        self._mox.ReplayAll()
+        self._conn.detach_volume(connection_info, instance_data, mount_point)
+        self._mox.VerifyAll()
 
-        m = vmutils.VMUtils.attach_volume_to_controller(mox.IsA(str),
-                                                        fake_controller_path,
-                                                        0,
-                                                        fake_mounted_disk)
-        m.WithSideEffects(self._add_volume_disk)
+    def test_boot_from_volume(self):
+        block_device_info = db_fakes.get_fake_block_device_info(
+            self._volume_target_portal, self._volume_id)
 
         self._setup_spawn_instance_mocks(cow=False,
                                          block_device_info=block_device_info,