This patch adds instance progress which is used by the OpenStack API to indicate how far along the current executing action is (BUILD/REBUILD, MIGRATION/RESIZE).

For the first cut, we decided to keep it simple and compute progress by counting discrete steps. This is not ideal since some steps, in particular, steps which involve transferring large amounts of data over the network, take *much* longer than others. A better approximation would account for the data-transferred to the destination host, since in most cases, this dominates the time spent.

In addition to adding progress, this patch:

- Allows resizes to use same host for source and destination which is useful for dev environments without a second host. This is enabled by the --allow_resize_to_same_host flag.

- Fixes a bug in the glance and migration XenAPI plugins where the VHDs were being copied into the SR in the wrong order. Before the base-copy was copied first meaning it was possible for snapwatchd to see the base-copy before the dependent cow was present. It was treat the base_copy as an unreferenced parent, and GC it.

- Additional refactoring and cleanups.

8 files changed, 70 insertions, 43 deletions

diff --git a/nova/tests/api/openstack/contrib/test_createserverext.py b/nova/tests/api/openstack/contrib/test_createserverext.py
index 03c7d1ec5..dfd6142ca 100644
--- a/nova/tests/api/openstack/contrib/test_createserverext.py
+++ b/nova/tests/api/openstack/contrib/test_createserverext.py
@@ -1,4 +1,4 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# vim: tabstop=5 shiftwidth=4 softtabstop=4
 
 # Copyright 2010-2011 OpenStack LLC.
 # All Rights Reserved.
@@ -54,6 +54,7 @@ INSTANCE = {
              "created_at": datetime.datetime(2010, 10, 10, 12, 0, 0),
              "updated_at": datetime.datetime(2010, 11, 11, 11, 0, 0),
              "security_groups": [{"id": 1, "name": "test"}],
+             "progress": 0,
              "image_ref": 'http://foo.com/123',
              "instance_type": {"flavorid": '124'},
         }
@@ -119,7 +120,8 @@ class CreateserverextTest(test.TestCase):
                          'user_id': 'fake',
                          'project_id': 'fake',
                          'created_at': "",
-                         'updated_at': ""}]
+                         'updated_at': "",
+                         'progress': 0}]
 
             def set_admin_password(self, *args, **kwargs):
                 pass
diff --git a/nova/tests/api/openstack/contrib/test_volumes.py b/nova/tests/api/openstack/contrib/test_volumes.py
index 52b65f5e1..ec69c1fd1 100644
--- a/nova/tests/api/openstack/contrib/test_volumes.py
+++ b/nova/tests/api/openstack/contrib/test_volumes.py
@@ -1,4 +1,4 @@
-# Copyright 2011 Josh Durgin
+# Copyright 2013 Josh Durgin
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -43,6 +43,7 @@ def fake_compute_api_create(cls, context, instance_type, image_href, **kwargs):
              'project_id': 'fake',
              'created_at': datetime.datetime(2010, 10, 10, 12, 0, 0),
              'updated_at': datetime.datetime(2010, 11, 11, 11, 0, 0),
+             'progress': 0
              }]
 
 
diff --git a/nova/tests/api/openstack/test_server_actions.py b/nova/tests/api/openstack/test_server_actions.py
index 4a215dd74..251b5d126 100644
--- a/nova/tests/api/openstack/test_server_actions.py
+++ b/nova/tests/api/openstack/test_server_actions.py
@@ -91,6 +91,7 @@ def stub_instance(id, metadata=None, image_ref="10", flavor_id="1",
         "access_ip_v6": "",
         "uuid": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
         "virtual_interfaces": [],
+        "progress": 0,
     }
 
     instance["fixed_ips"] = {
diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py
index c21fb4a62..b83aad49f 100644
--- a/nova/tests/api/openstack/test_servers.py
+++ b/nova/tests/api/openstack/test_servers.py
@@ -162,7 +162,7 @@ def stub_instance(id, user_id='fake', project_id='fake', private_address=None,
                   vm_state=None, task_state=None,
                   reservation_id="", uuid=FAKE_UUID, image_ref="10",
                   flavor_id="1", interfaces=None, name=None, key_name='',
-                  access_ipv4=None, access_ipv6=None):
+                  access_ipv4=None, access_ipv6=None, progress=0):
     metadata = []
     metadata.append(InstanceMetadata(key='seq', value=id))
 
@@ -222,7 +222,8 @@ def stub_instance(id, user_id='fake', project_id='fake', private_address=None,
         "access_ip_v4": access_ipv4,
         "access_ip_v6": access_ipv6,
         "uuid": uuid,
-        "virtual_interfaces": interfaces}
+        "virtual_interfaces": interfaces,
+        "progress": progress}
 
     instance["fixed_ips"] = {
         "address": private_address,
@@ -548,7 +549,8 @@ class ServersTest(test.TestCase):
             },
         ]
         new_return_server = return_server_with_attributes(
-            interfaces=interfaces, vm_state=vm_states.ACTIVE)
+            interfaces=interfaces, vm_state=vm_states.ACTIVE,
+            progress=100)
         self.stubs.Set(nova.db.api, 'instance_get', new_return_server)
 
         req = webob.Request.blank('/v1.1/fake/servers/1')
@@ -645,7 +647,7 @@ class ServersTest(test.TestCase):
         ]
         new_return_server = return_server_with_attributes(
             interfaces=interfaces, vm_state=vm_states.ACTIVE,
-            image_ref=image_ref, flavor_id=flavor_id)
+            image_ref=image_ref, flavor_id=flavor_id, progress=100)
         self.stubs.Set(nova.db.api, 'instance_get', new_return_server)
 
         req = webob.Request.blank('/v1.1/fake/servers/1')
@@ -1527,6 +1529,7 @@ class ServersTest(test.TestCase):
                     "created_at": datetime.datetime(2010, 10, 10, 12, 0, 0),
                     "updated_at": datetime.datetime(2010, 11, 11, 11, 0, 0),
                     "config_drive": self.config_drive,
+                    "progress": 0
                    }
 
         def server_update(context, id, params):
@@ -3819,7 +3822,8 @@ class ServersViewBuilderV11Test(test.TestCase):
             "accessIPv6": "fead::1234",
             #"address": ,
             #"floating_ips": [{"address":ip} for ip in public_addresses]}
-            "uuid": "deadbeef-feed-edee-beef-d0ea7beefedd"}
+            "uuid": "deadbeef-feed-edee-beef-d0ea7beefedd",
+            "progress": 0}
 
         return instance
 
@@ -3942,6 +3946,7 @@ class ServersViewBuilderV11Test(test.TestCase):
     def test_build_server_detail_active_status(self):
         #set the power state of the instance to running
         self.instance['vm_state'] = vm_states.ACTIVE
+        self.instance['progress'] = 100
         image_bookmark = "http://localhost/images/5"
         flavor_bookmark = "http://localhost/flavors/1"
         expected_server = {
diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 948c7ad40..7ce9e740a 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -584,7 +584,7 @@ class ComputeTestCase(test.TestCase):
             pass
 
         self.stubs.Set(self.compute.driver, 'finish_migration', fake)
-        self.stubs.Set(self.compute.driver, 'revert_migration', fake)
+        self.stubs.Set(self.compute.driver, 'finish_revert_migration', fake)
         self.stubs.Set(self.compute.network_api, 'get_instance_nw_info', fake)
 
         self.compute.run_instance(self.context, instance_id)
diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py
index 8e20e999f..d4180b6f7 100644
--- a/nova/tests/test_virt_drivers.py
+++ b/nova/tests/test_virt_drivers.py
@@ -185,7 +185,8 @@ class _VirtDriverTestCase(test.TestCase):
         instance_ref = test_utils.get_test_instance()
         network_info = test_utils.get_test_network_info()
         self.connection.spawn(self.ctxt, instance_ref, network_info)
-        self.connection.migrate_disk_and_power_off(instance_ref, 'dest_host')
+        self.connection.migrate_disk_and_power_off(
+            self.ctxt, instance_ref, 'dest_host')
 
     @catch_notimplementederror
     def test_pause(self):
diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py
index 47c6a3c95..2cacd2364 100644
--- a/nova/tests/test_xenapi.py
+++ b/nova/tests/test_xenapi.py
@@ -76,7 +76,7 @@ class XenAPIVolumeTestCase(test.TestCase):
         db_fakes.stub_out_db_instance_api(self.stubs)
         stubs.stub_out_get_target(self.stubs)
         xenapi_fake.reset()
-        self.values = {'id': 1,
+        self.instance_values = {'id': 1,
                   'project_id': self.user_id,
                   'user_id': 'fake',
                   'image_ref': 1,
@@ -132,7 +132,7 @@ class XenAPIVolumeTestCase(test.TestCase):
         stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests)
         conn = xenapi_conn.get_connection(False)
         volume = self._create_volume()
-        instance = db.instance_create(self.context, self.values)
+        instance = db.instance_create(self.context, self.instance_values)
         vm = xenapi_fake.create_vm(instance.name, 'Running')
         result = conn.attach_volume(instance.name, volume['id'], '/dev/sdc')
 
@@ -152,7 +152,7 @@ class XenAPIVolumeTestCase(test.TestCase):
                               stubs.FakeSessionForVolumeFailedTests)
         conn = xenapi_conn.get_connection(False)
         volume = self._create_volume()
-        instance = db.instance_create(self.context, self.values)
+        instance = db.instance_create(self.context, self.instance_values)
         xenapi_fake.create_vm(instance.name, 'Running')
         self.assertRaises(Exception,
                           conn.attach_volume,
@@ -369,7 +369,7 @@ class XenAPIVMTestCase(test.TestCase):
                     create_record=True, empty_dns=False):
         stubs.stubout_loopingcall_start(self.stubs)
         if create_record:
-            values = {'id': instance_id,
+            instance_values = {'id': instance_id,
                       'project_id': self.project_id,
                       'user_id': self.user_id,
                       'image_ref': image_ref,
@@ -379,7 +379,7 @@ class XenAPIVMTestCase(test.TestCase):
                       'os_type': os_type,
                       'hostname': hostname,
                       'architecture': architecture}
-            instance = db.instance_create(self.context, values)
+            instance = db.instance_create(self.context, instance_values)
         else:
             instance = db.instance_get(self.context, instance_id)
         network_info = [({'bridge': 'fa0', 'id': 0, 'injected': True},
@@ -623,28 +623,28 @@ class XenAPIVMTestCase(test.TestCase):
         # Ensure that it will not unrescue a non-rescued instance.
         self.assertRaises(Exception, conn.unrescue, instance, None)
 
-    def test_revert_migration(self):
+    def test_finish_revert_migration(self):
         instance = self._create_instance()
 
         class VMOpsMock():
 
             def __init__(self):
-                self.revert_migration_called = False
+                self.finish_revert_migration_called = False
 
-            def revert_migration(self, instance):
-                self.revert_migration_called = True
+            def finish_revert_migration(self, instance):
+                self.finish_revert_migration_called = True
 
         stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
 
         conn = xenapi_conn.get_connection(False)
         conn._vmops = VMOpsMock()
-        conn.revert_migration(instance)
-        self.assertTrue(conn._vmops.revert_migration_called)
+        conn.finish_revert_migration(instance)
+        self.assertTrue(conn._vmops.finish_revert_migration_called)
 
     def _create_instance(self, instance_id=1, spawn=True):
         """Creates and spawns a test instance."""
         stubs.stubout_loopingcall_start(self.stubs)
-        values = {
+        instance_values = {
             'id': instance_id,
             'project_id': self.project_id,
             'user_id': self.user_id,
@@ -654,7 +654,7 @@ class XenAPIVMTestCase(test.TestCase):
             'instance_type_id': '3',  # m1.large
             'os_type': 'linux',
             'architecture': 'x86-64'}
-        instance = db.instance_create(self.context, values)
+        instance = db.instance_create(self.context, instance_values)
         network_info = [({'bridge': 'fa0', 'id': 0, 'injected': False},
                           {'broadcast': '192.168.0.255',
                            'dns': ['192.168.0.1'],
@@ -732,7 +732,7 @@ class XenAPIMigrateInstance(test.TestCase):
         self.user_id = 'fake'
         self.project_id = 'fake'
         self.context = context.RequestContext(self.user_id, self.project_id)
-        self.values = {'id': 1,
+        self.instance_values = {'id': 1,
                   'project_id': self.project_id,
                   'user_id': self.user_id,
                   'image_ref': 1,
@@ -743,22 +743,34 @@ class XenAPIMigrateInstance(test.TestCase):
                   'os_type': 'linux',
                   'architecture': 'x86-64'}
 
+        migration_values = {
+            'source_compute': 'nova-compute',
+            'dest_compute': 'nova-compute',
+            'dest_host': '10.127.5.114',
+            'status': 'post-migrating',
+            'instance_uuid': '15f23e6a-cc6e-4d22-b651-d9bdaac316f7',
+            'old_instance_type_id': 5,
+            'new_instance_type_id': 1
+        }
+        self.migration = db.migration_create(
+            context.get_admin_context(), migration_values)
+
         fake_utils.stub_out_utils_execute(self.stubs)
         stubs.stub_out_migration_methods(self.stubs)
         stubs.stubout_get_this_vm_uuid(self.stubs)
         glance_stubs.stubout_glance_client(self.stubs)
 
     def test_migrate_disk_and_power_off(self):
-        instance = db.instance_create(self.context, self.values)
+        instance = db.instance_create(self.context, self.instance_values)
         stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
         conn = xenapi_conn.get_connection(False)
-        conn.migrate_disk_and_power_off(instance, '127.0.0.1')
+        conn.migrate_disk_and_power_off(self.context, instance, '127.0.0.1')
 
     def test_revert_migrate(self):
-        instance = db.instance_create(self.context, self.values)
+        instance = db.instance_create(self.context, self.instance_values)
         self.called = False
         self.fake_vm_start_called = False
-        self.fake_revert_migration_called = False
+        self.fake_finish_revert_migration_called = False
 
         def fake_vm_start(*args, **kwargs):
             self.fake_vm_start_called = True
@@ -766,13 +778,14 @@ class XenAPIMigrateInstance(test.TestCase):
         def fake_vdi_resize(*args, **kwargs):
             self.called = True
 
-        def fake_revert_migration(*args, **kwargs):
-            self.fake_revert_migration_called = True
+        def fake_finish_revert_migration(*args, **kwargs):
+            self.fake_finish_revert_migration_called = True
 
         self.stubs.Set(stubs.FakeSessionForMigrationTests,
                 "VDI_resize_online", fake_vdi_resize)
         self.stubs.Set(vmops.VMOps, '_start', fake_vm_start)
-        self.stubs.Set(vmops.VMOps, 'revert_migration', fake_revert_migration)
+        self.stubs.Set(vmops.VMOps, 'finish_revert_migration',
+                       fake_finish_revert_migration)
 
         stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
         stubs.stubout_loopingcall_start(self.stubs)
@@ -791,17 +804,17 @@ class XenAPIMigrateInstance(test.TestCase):
                            'label': 'fake',
                            'mac': 'DE:AD:BE:EF:00:00',
                            'rxtx_cap': 3})]
-        conn.finish_migration(self.context, instance,
+        conn.finish_migration(self.context, self.migration, instance,
                               dict(base_copy='hurr', cow='durr'),
                               network_info, resize_instance=True)
         self.assertEqual(self.called, True)
         self.assertEqual(self.fake_vm_start_called, True)
 
-        conn.revert_migration(instance)
-        self.assertEqual(self.fake_revert_migration_called, True)
+        conn.finish_revert_migration(instance)
+        self.assertEqual(self.fake_finish_revert_migration_called, True)
 
     def test_finish_migrate(self):
-        instance = db.instance_create(self.context, self.values)
+        instance = db.instance_create(self.context, self.instance_values)
         self.called = False
         self.fake_vm_start_called = False
 
@@ -832,7 +845,7 @@ class XenAPIMigrateInstance(test.TestCase):
                            'label': 'fake',
                            'mac': 'DE:AD:BE:EF:00:00',
                            'rxtx_cap': 3})]
-        conn.finish_migration(self.context, instance,
+        conn.finish_migration(self.context, self.migration, instance,
                               dict(base_copy='hurr', cow='durr'),
                               network_info, resize_instance=True)
         self.assertEqual(self.called, True)
@@ -841,8 +854,9 @@ class XenAPIMigrateInstance(test.TestCase):
     def test_finish_migrate_no_local_storage(self):
         tiny_type_id = \
                 instance_types.get_instance_type_by_name('m1.tiny')['id']
-        self.values.update({'instance_type_id': tiny_type_id, 'local_gb': 0})
-        instance = db.instance_create(self.context, self.values)
+        self.instance_values.update({'instance_type_id': tiny_type_id,
+                                     'local_gb': 0})
+        instance = db.instance_create(self.context, self.instance_values)
 
         def fake_vdi_resize(*args, **kwargs):
             raise Exception("This shouldn't be called")
@@ -866,12 +880,12 @@ class XenAPIMigrateInstance(test.TestCase):
                            'label': 'fake',
                            'mac': 'DE:AD:BE:EF:00:00',
                            'rxtx_cap': 3})]
-        conn.finish_migration(self.context, instance,
+        conn.finish_migration(self.context, self.migration, instance,
                               dict(base_copy='hurr', cow='durr'),
                               network_info, resize_instance=True)
 
     def test_finish_migrate_no_resize_vdi(self):
-        instance = db.instance_create(self.context, self.values)
+        instance = db.instance_create(self.context, self.instance_values)
 
         def fake_vdi_resize(*args, **kwargs):
             raise Exception("This shouldn't be called")
@@ -897,7 +911,7 @@ class XenAPIMigrateInstance(test.TestCase):
                            'rxtx_cap': 3})]
 
         # Resize instance would be determined by the compute call
-        conn.finish_migration(self.context, instance,
+        conn.finish_migration(self.context, self.migration, instance,
                               dict(base_copy='hurr', cow='durr'),
                               network_info, resize_instance=False)
 
diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py
index 647a4c1df..aee279920 100644
--- a/nova/tests/xenapi/stubs.py
+++ b/nova/tests/xenapi/stubs.py
@@ -295,9 +295,12 @@ class FakeSessionForMigrationTests(fake.SessionBase):
         vm['is_control_domain'] = False
         vm['domid'] = random.randrange(1, 1 << 16)
 
+    def VM_set_name_label(self, *args):
+        pass
+
 
 def stub_out_migration_methods(stubs):
-    def fake_get_snapshot(self, instance):
+    def fake_create_snapshot(self, instance):
         return 'vm_ref', dict(image='foo', snap='bar')
 
     @classmethod
@@ -327,7 +330,7 @@ def stub_out_migration_methods(stubs):
     stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
     stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr)
     stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr)
-    stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot)
+    stubs.Set(vmops.VMOps, '_create_snapshot', fake_create_snapshot)
     stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi)
     stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None)
     stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path)