summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRick Harris <rconradharris@gmail.com>2011-09-21 22:14:15 +0000
committerTarmac <>2011-09-21 22:14:15 +0000
commitd865a2a97c013a811c2c6baad00fa1eb95406c8d (patch)
treee8a8881f91717d2f16b4de995f774e92f168e7b5
parent221509abcdef0077e8c592c5e0a6ae3add78b007 (diff)
parent4fb602fcbc2a7b0681e79454fe7c3f01110b1f0e (diff)
downloadnova-d865a2a97c013a811c2c6baad00fa1eb95406c8d.tar.gz
nova-d865a2a97c013a811c2c6baad00fa1eb95406c8d.tar.xz
nova-d865a2a97c013a811c2c6baad00fa1eb95406c8d.zip
This patch adds instance progress which is used by the OpenStack API to indicate how far along the current executing action is (BUILD/REBUILD, MIGRATION/RESIZE).
For the first cut, we decided to keep it simple and compute progress by counting discrete steps. This is not ideal since some steps, in particular, steps which involve transferring large amounts of data over the network, take *much* longer than others. A better approximation would account for the data-transferred to the destination host, since in most cases, this dominates the time spent. In addition to adding progress, this patch: - Allows resizes to use same host for source and destination which is useful for dev environments without a second host. This is enabled by the --allow_resize_to_same_host flag. - Fixes a bug in the glance and migration XenAPI plugins where the VHDs were being copied into the SR in the wrong order. Before the base-copy was copied first meaning it was possible for snapwatchd to see the base-copy before the dependent cow was present. It was treat the base_copy as an unreferenced parent, and GC it. - Additional refactoring and cleanups.
-rw-r--r--nova/api/openstack/views/servers.py10
-rw-r--r--nova/compute/api.py9
-rw-r--r--nova/compute/manager.py15
-rw-r--r--nova/db/sqlalchemy/migrate_repo/versions/049_add_instances_progress.py43
-rw-r--r--nova/db/sqlalchemy/models.py2
-rw-r--r--nova/flags.py4
-rw-r--r--nova/tests/api/openstack/contrib/test_createserverext.py6
-rw-r--r--nova/tests/api/openstack/contrib/test_volumes.py3
-rw-r--r--nova/tests/api/openstack/test_server_actions.py1
-rw-r--r--nova/tests/api/openstack/test_servers.py15
-rw-r--r--nova/tests/test_compute.py2
-rw-r--r--nova/tests/test_virt_drivers.py3
-rw-r--r--nova/tests/test_xenapi.py76
-rw-r--r--nova/tests/xenapi/stubs.py7
-rw-r--r--nova/utils.py14
-rw-r--r--nova/virt/driver.py16
-rw-r--r--nova/virt/fake.py4
-rw-r--r--nova/virt/xenapi/vm_utils.py4
-rw-r--r--nova/virt/xenapi/vmops.py152
-rw-r--r--nova/virt/xenapi_conn.py21
-rwxr-xr-xplugins/xenserver/xenapi/etc/xapi.d/plugins/glance6
-rwxr-xr-xplugins/xenserver/xenapi/etc/xapi.d/plugins/migration11
22 files changed, 318 insertions, 106 deletions
diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py
index 473dc9e7e..925668e56 100644
--- a/nova/api/openstack/views/servers.py
+++ b/nova/api/openstack/views/servers.py
@@ -137,11 +137,11 @@ class ViewBuilderV11(ViewBuilder):
response = super(ViewBuilderV11, self)._build_detail(inst)
response['server']['created'] = utils.isotime(inst['created_at'])
response['server']['updated'] = utils.isotime(inst['updated_at'])
- if 'status' in response['server']:
- if response['server']['status'] == "ACTIVE":
- response['server']['progress'] = 100
- elif response['server']['status'] == "BUILD":
- response['server']['progress'] = 0
+
+ status = response['server'].get('status')
+ if status in ('ACTIVE', 'BUILD', 'REBUILD', 'RESIZE',
+ 'VERIFY_RESIZE'):
+ response['server']['progress'] = inst['progress'] or 0
response['server']['accessIPv4'] = inst.get('access_ip_v4') or ""
response['server']['accessIPv6'] = inst.get('access_ip_v6') or ""
diff --git a/nova/compute/api.py b/nova/compute/api.py
index 130828d7b..cedbd747a 100644
--- a/nova/compute/api.py
+++ b/nova/compute/api.py
@@ -846,7 +846,8 @@ class API(base.Base):
self.update(context,
instance_id,
- task_state=task_states.DELETING)
+ task_state=task_states.DELETING,
+ progress=0)
host = instance['host']
if host:
@@ -869,7 +870,8 @@ class API(base.Base):
instance_id,
vm_state=vm_states.ACTIVE,
task_state=task_states.STOPPING,
- terminated_at=utils.utcnow())
+ terminated_at=utils.utcnow(),
+ progress=0)
host = instance['host']
if host:
@@ -1169,7 +1171,8 @@ class API(base.Base):
display_name=name,
image_ref=image_href,
vm_state=vm_states.ACTIVE,
- task_state=task_states.REBUILDING)
+ task_state=task_states.REBUILDING,
+ progress=0)
rebuild_params = {
"new_pass": admin_password,
diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index d7c23c65d..884f1f78f 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -882,7 +882,9 @@ class ComputeManager(manager.SchedulerDependentManager):
migration_ref.instance_uuid)
network_info = self._get_instance_nw_info(context, instance_ref)
- self.driver.destroy(instance_ref, network_info)
+ self.driver.confirm_migration(
+ migration_ref, instance_ref, network_info)
+
usage_info = utils.usage_from_instance(instance_ref)
notifier.notify('compute.%s' % self.host,
'compute.instance.resize.confirm',
@@ -937,7 +939,7 @@ class ComputeManager(manager.SchedulerDependentManager):
local_gb=instance_type['local_gb'],
instance_type_id=instance_type['id'])
- self.driver.revert_migration(instance_ref)
+ self.driver.finish_revert_migration(instance_ref)
self.db.migration_update(context, migration_id,
{'status': 'reverted'})
usage_info = utils.usage_from_instance(instance_ref)
@@ -961,7 +963,8 @@ class ComputeManager(manager.SchedulerDependentManager):
# of the instance down
instance_ref = self.db.instance_get_by_uuid(context, instance_id)
- if instance_ref['host'] == FLAGS.host:
+ same_host = instance_ref['host'] == FLAGS.host
+ if same_host and not FLAGS.allow_resize_to_same_host:
self._instance_update(context,
instance_id,
vm_state=vm_states.ERROR)
@@ -1012,7 +1015,7 @@ class ComputeManager(manager.SchedulerDependentManager):
{'status': 'migrating'})
disk_info = self.driver.migrate_disk_and_power_off(
- instance_ref, migration_ref['dest_host'])
+ context, instance_ref, migration_ref['dest_host'])
self.db.migration_update(context,
migration_id,
{'status': 'post-migrating'})
@@ -1057,8 +1060,8 @@ class ComputeManager(manager.SchedulerDependentManager):
instance_ref.uuid)
network_info = self._get_instance_nw_info(context, instance_ref)
- self.driver.finish_migration(context, instance_ref, disk_info,
- network_info, resize_instance)
+ self.driver.finish_migration(context, migration_ref, instance_ref,
+ disk_info, network_info, resize_instance)
self._instance_update(context,
instance_id,
diff --git a/nova/db/sqlalchemy/migrate_repo/versions/049_add_instances_progress.py b/nova/db/sqlalchemy/migrate_repo/versions/049_add_instances_progress.py
new file mode 100644
index 000000000..d23d52e80
--- /dev/null
+++ b/nova/db/sqlalchemy/migrate_repo/versions/049_add_instances_progress.py
@@ -0,0 +1,43 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 OpenStack LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from sqlalchemy import *
+from migrate import *
+
+from nova import log as logging
+
+meta = MetaData()
+
+instances = Table('instances', meta,
+ Column("id", Integer(), primary_key=True, nullable=False))
+
+# Add progress column to instances table
+progress = Column('progress', Integer())
+
+
+def upgrade(migrate_engine):
+ meta.bind = migrate_engine
+
+ try:
+ instances.create_column(progress)
+ except Exception:
+ logging.error(_("progress column not added to instances table"))
+ raise
+
+
+def downgrade(migrate_engine):
+ meta.bind = migrate_engine
+ instances.drop_column(progress)
diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py
index 33d740fbc..2261a1d09 100644
--- a/nova/db/sqlalchemy/models.py
+++ b/nova/db/sqlalchemy/models.py
@@ -241,6 +241,8 @@ class Instance(BASE, NovaBase):
access_ip_v4 = Column(String(255))
access_ip_v6 = Column(String(255))
+ progress = Column(Integer)
+
class VirtualStorageArray(BASE, NovaBase):
"""
diff --git a/nova/flags.py b/nova/flags.py
index ffb313cec..fab099660 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -434,3 +434,7 @@ DEFINE_list('monkey_patch_modules',
'nova.compute.api:nova.notifier.api.notify_decorator'],
'Module list representing monkey '
'patched module and decorator')
+
+DEFINE_bool('allow_resize_to_same_host', False,
+ 'Allow destination machine to match source for resize. Useful'
+ ' when testing in environments with only one host machine.')
diff --git a/nova/tests/api/openstack/contrib/test_createserverext.py b/nova/tests/api/openstack/contrib/test_createserverext.py
index 03c7d1ec5..dfd6142ca 100644
--- a/nova/tests/api/openstack/contrib/test_createserverext.py
+++ b/nova/tests/api/openstack/contrib/test_createserverext.py
@@ -1,4 +1,4 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# vim: tabstop=5 shiftwidth=4 softtabstop=4
# Copyright 2010-2011 OpenStack LLC.
# All Rights Reserved.
@@ -54,6 +54,7 @@ INSTANCE = {
"created_at": datetime.datetime(2010, 10, 10, 12, 0, 0),
"updated_at": datetime.datetime(2010, 11, 11, 11, 0, 0),
"security_groups": [{"id": 1, "name": "test"}],
+ "progress": 0,
"image_ref": 'http://foo.com/123',
"instance_type": {"flavorid": '124'},
}
@@ -119,7 +120,8 @@ class CreateserverextTest(test.TestCase):
'user_id': 'fake',
'project_id': 'fake',
'created_at': "",
- 'updated_at': ""}]
+ 'updated_at': "",
+ 'progress': 0}]
def set_admin_password(self, *args, **kwargs):
pass
diff --git a/nova/tests/api/openstack/contrib/test_volumes.py b/nova/tests/api/openstack/contrib/test_volumes.py
index 52b65f5e1..ec69c1fd1 100644
--- a/nova/tests/api/openstack/contrib/test_volumes.py
+++ b/nova/tests/api/openstack/contrib/test_volumes.py
@@ -1,4 +1,4 @@
-# Copyright 2011 Josh Durgin
+# Copyright 2013 Josh Durgin
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -43,6 +43,7 @@ def fake_compute_api_create(cls, context, instance_type, image_href, **kwargs):
'project_id': 'fake',
'created_at': datetime.datetime(2010, 10, 10, 12, 0, 0),
'updated_at': datetime.datetime(2010, 11, 11, 11, 0, 0),
+ 'progress': 0
}]
diff --git a/nova/tests/api/openstack/test_server_actions.py b/nova/tests/api/openstack/test_server_actions.py
index 4a215dd74..251b5d126 100644
--- a/nova/tests/api/openstack/test_server_actions.py
+++ b/nova/tests/api/openstack/test_server_actions.py
@@ -91,6 +91,7 @@ def stub_instance(id, metadata=None, image_ref="10", flavor_id="1",
"access_ip_v6": "",
"uuid": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
"virtual_interfaces": [],
+ "progress": 0,
}
instance["fixed_ips"] = {
diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py
index c21fb4a62..b83aad49f 100644
--- a/nova/tests/api/openstack/test_servers.py
+++ b/nova/tests/api/openstack/test_servers.py
@@ -162,7 +162,7 @@ def stub_instance(id, user_id='fake', project_id='fake', private_address=None,
vm_state=None, task_state=None,
reservation_id="", uuid=FAKE_UUID, image_ref="10",
flavor_id="1", interfaces=None, name=None, key_name='',
- access_ipv4=None, access_ipv6=None):
+ access_ipv4=None, access_ipv6=None, progress=0):
metadata = []
metadata.append(InstanceMetadata(key='seq', value=id))
@@ -222,7 +222,8 @@ def stub_instance(id, user_id='fake', project_id='fake', private_address=None,
"access_ip_v4": access_ipv4,
"access_ip_v6": access_ipv6,
"uuid": uuid,
- "virtual_interfaces": interfaces}
+ "virtual_interfaces": interfaces,
+ "progress": progress}
instance["fixed_ips"] = {
"address": private_address,
@@ -548,7 +549,8 @@ class ServersTest(test.TestCase):
},
]
new_return_server = return_server_with_attributes(
- interfaces=interfaces, vm_state=vm_states.ACTIVE)
+ interfaces=interfaces, vm_state=vm_states.ACTIVE,
+ progress=100)
self.stubs.Set(nova.db.api, 'instance_get', new_return_server)
req = webob.Request.blank('/v1.1/fake/servers/1')
@@ -645,7 +647,7 @@ class ServersTest(test.TestCase):
]
new_return_server = return_server_with_attributes(
interfaces=interfaces, vm_state=vm_states.ACTIVE,
- image_ref=image_ref, flavor_id=flavor_id)
+ image_ref=image_ref, flavor_id=flavor_id, progress=100)
self.stubs.Set(nova.db.api, 'instance_get', new_return_server)
req = webob.Request.blank('/v1.1/fake/servers/1')
@@ -1527,6 +1529,7 @@ class ServersTest(test.TestCase):
"created_at": datetime.datetime(2010, 10, 10, 12, 0, 0),
"updated_at": datetime.datetime(2010, 11, 11, 11, 0, 0),
"config_drive": self.config_drive,
+ "progress": 0
}
def server_update(context, id, params):
@@ -3819,7 +3822,8 @@ class ServersViewBuilderV11Test(test.TestCase):
"accessIPv6": "fead::1234",
#"address": ,
#"floating_ips": [{"address":ip} for ip in public_addresses]}
- "uuid": "deadbeef-feed-edee-beef-d0ea7beefedd"}
+ "uuid": "deadbeef-feed-edee-beef-d0ea7beefedd",
+ "progress": 0}
return instance
@@ -3942,6 +3946,7 @@ class ServersViewBuilderV11Test(test.TestCase):
def test_build_server_detail_active_status(self):
#set the power state of the instance to running
self.instance['vm_state'] = vm_states.ACTIVE
+ self.instance['progress'] = 100
image_bookmark = "http://localhost/images/5"
flavor_bookmark = "http://localhost/flavors/1"
expected_server = {
diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 948c7ad40..7ce9e740a 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -584,7 +584,7 @@ class ComputeTestCase(test.TestCase):
pass
self.stubs.Set(self.compute.driver, 'finish_migration', fake)
- self.stubs.Set(self.compute.driver, 'revert_migration', fake)
+ self.stubs.Set(self.compute.driver, 'finish_revert_migration', fake)
self.stubs.Set(self.compute.network_api, 'get_instance_nw_info', fake)
self.compute.run_instance(self.context, instance_id)
diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py
index 8e20e999f..d4180b6f7 100644
--- a/nova/tests/test_virt_drivers.py
+++ b/nova/tests/test_virt_drivers.py
@@ -185,7 +185,8 @@ class _VirtDriverTestCase(test.TestCase):
instance_ref = test_utils.get_test_instance()
network_info = test_utils.get_test_network_info()
self.connection.spawn(self.ctxt, instance_ref, network_info)
- self.connection.migrate_disk_and_power_off(instance_ref, 'dest_host')
+ self.connection.migrate_disk_and_power_off(
+ self.ctxt, instance_ref, 'dest_host')
@catch_notimplementederror
def test_pause(self):
diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py
index 47c6a3c95..2cacd2364 100644
--- a/nova/tests/test_xenapi.py
+++ b/nova/tests/test_xenapi.py
@@ -76,7 +76,7 @@ class XenAPIVolumeTestCase(test.TestCase):
db_fakes.stub_out_db_instance_api(self.stubs)
stubs.stub_out_get_target(self.stubs)
xenapi_fake.reset()
- self.values = {'id': 1,
+ self.instance_values = {'id': 1,
'project_id': self.user_id,
'user_id': 'fake',
'image_ref': 1,
@@ -132,7 +132,7 @@ class XenAPIVolumeTestCase(test.TestCase):
stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests)
conn = xenapi_conn.get_connection(False)
volume = self._create_volume()
- instance = db.instance_create(self.context, self.values)
+ instance = db.instance_create(self.context, self.instance_values)
vm = xenapi_fake.create_vm(instance.name, 'Running')
result = conn.attach_volume(instance.name, volume['id'], '/dev/sdc')
@@ -152,7 +152,7 @@ class XenAPIVolumeTestCase(test.TestCase):
stubs.FakeSessionForVolumeFailedTests)
conn = xenapi_conn.get_connection(False)
volume = self._create_volume()
- instance = db.instance_create(self.context, self.values)
+ instance = db.instance_create(self.context, self.instance_values)
xenapi_fake.create_vm(instance.name, 'Running')
self.assertRaises(Exception,
conn.attach_volume,
@@ -369,7 +369,7 @@ class XenAPIVMTestCase(test.TestCase):
create_record=True, empty_dns=False):
stubs.stubout_loopingcall_start(self.stubs)
if create_record:
- values = {'id': instance_id,
+ instance_values = {'id': instance_id,
'project_id': self.project_id,
'user_id': self.user_id,
'image_ref': image_ref,
@@ -379,7 +379,7 @@ class XenAPIVMTestCase(test.TestCase):
'os_type': os_type,
'hostname': hostname,
'architecture': architecture}
- instance = db.instance_create(self.context, values)
+ instance = db.instance_create(self.context, instance_values)
else:
instance = db.instance_get(self.context, instance_id)
network_info = [({'bridge': 'fa0', 'id': 0, 'injected': True},
@@ -623,28 +623,28 @@ class XenAPIVMTestCase(test.TestCase):
# Ensure that it will not unrescue a non-rescued instance.
self.assertRaises(Exception, conn.unrescue, instance, None)
- def test_revert_migration(self):
+ def test_finish_revert_migration(self):
instance = self._create_instance()
class VMOpsMock():
def __init__(self):
- self.revert_migration_called = False
+ self.finish_revert_migration_called = False
- def revert_migration(self, instance):
- self.revert_migration_called = True
+ def finish_revert_migration(self, instance):
+ self.finish_revert_migration_called = True
stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
conn = xenapi_conn.get_connection(False)
conn._vmops = VMOpsMock()
- conn.revert_migration(instance)
- self.assertTrue(conn._vmops.revert_migration_called)
+ conn.finish_revert_migration(instance)
+ self.assertTrue(conn._vmops.finish_revert_migration_called)
def _create_instance(self, instance_id=1, spawn=True):
"""Creates and spawns a test instance."""
stubs.stubout_loopingcall_start(self.stubs)
- values = {
+ instance_values = {
'id': instance_id,
'project_id': self.project_id,
'user_id': self.user_id,
@@ -654,7 +654,7 @@ class XenAPIVMTestCase(test.TestCase):
'instance_type_id': '3', # m1.large
'os_type': 'linux',
'architecture': 'x86-64'}
- instance = db.instance_create(self.context, values)
+ instance = db.instance_create(self.context, instance_values)
network_info = [({'bridge': 'fa0', 'id': 0, 'injected': False},
{'broadcast': '192.168.0.255',
'dns': ['192.168.0.1'],
@@ -732,7 +732,7 @@ class XenAPIMigrateInstance(test.TestCase):
self.user_id = 'fake'
self.project_id = 'fake'
self.context = context.RequestContext(self.user_id, self.project_id)
- self.values = {'id': 1,
+ self.instance_values = {'id': 1,
'project_id': self.project_id,
'user_id': self.user_id,
'image_ref': 1,
@@ -743,22 +743,34 @@ class XenAPIMigrateInstance(test.TestCase):
'os_type': 'linux',
'architecture': 'x86-64'}
+ migration_values = {
+ 'source_compute': 'nova-compute',
+ 'dest_compute': 'nova-compute',
+ 'dest_host': '10.127.5.114',
+ 'status': 'post-migrating',
+ 'instance_uuid': '15f23e6a-cc6e-4d22-b651-d9bdaac316f7',
+ 'old_instance_type_id': 5,
+ 'new_instance_type_id': 1
+ }
+ self.migration = db.migration_create(
+ context.get_admin_context(), migration_values)
+
fake_utils.stub_out_utils_execute(self.stubs)
stubs.stub_out_migration_methods(self.stubs)
stubs.stubout_get_this_vm_uuid(self.stubs)
glance_stubs.stubout_glance_client(self.stubs)
def test_migrate_disk_and_power_off(self):
- instance = db.instance_create(self.context, self.values)
+ instance = db.instance_create(self.context, self.instance_values)
stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
conn = xenapi_conn.get_connection(False)
- conn.migrate_disk_and_power_off(instance, '127.0.0.1')
+ conn.migrate_disk_and_power_off(self.context, instance, '127.0.0.1')
def test_revert_migrate(self):
- instance = db.instance_create(self.context, self.values)
+ instance = db.instance_create(self.context, self.instance_values)
self.called = False
self.fake_vm_start_called = False
- self.fake_revert_migration_called = False
+ self.fake_finish_revert_migration_called = False
def fake_vm_start(*args, **kwargs):
self.fake_vm_start_called = True
@@ -766,13 +778,14 @@ class XenAPIMigrateInstance(test.TestCase):
def fake_vdi_resize(*args, **kwargs):
self.called = True
- def fake_revert_migration(*args, **kwargs):
- self.fake_revert_migration_called = True
+ def fake_finish_revert_migration(*args, **kwargs):
+ self.fake_finish_revert_migration_called = True
self.stubs.Set(stubs.FakeSessionForMigrationTests,
"VDI_resize_online", fake_vdi_resize)
self.stubs.Set(vmops.VMOps, '_start', fake_vm_start)
- self.stubs.Set(vmops.VMOps, 'revert_migration', fake_revert_migration)
+ self.stubs.Set(vmops.VMOps, 'finish_revert_migration',
+ fake_finish_revert_migration)
stubs.stubout_session(self.stubs, stubs.FakeSessionForMigrationTests)
stubs.stubout_loopingcall_start(self.stubs)
@@ -791,17 +804,17 @@ class XenAPIMigrateInstance(test.TestCase):
'label': 'fake',
'mac': 'DE:AD:BE:EF:00:00',
'rxtx_cap': 3})]
- conn.finish_migration(self.context, instance,
+ conn.finish_migration(self.context, self.migration, instance,
dict(base_copy='hurr', cow='durr'),
network_info, resize_instance=True)
self.assertEqual(self.called, True)
self.assertEqual(self.fake_vm_start_called, True)
- conn.revert_migration(instance)
- self.assertEqual(self.fake_revert_migration_called, True)
+ conn.finish_revert_migration(instance)
+ self.assertEqual(self.fake_finish_revert_migration_called, True)
def test_finish_migrate(self):
- instance = db.instance_create(self.context, self.values)
+ instance = db.instance_create(self.context, self.instance_values)
self.called = False
self.fake_vm_start_called = False
@@ -832,7 +845,7 @@ class XenAPIMigrateInstance(test.TestCase):
'label': 'fake',
'mac': 'DE:AD:BE:EF:00:00',
'rxtx_cap': 3})]
- conn.finish_migration(self.context, instance,
+ conn.finish_migration(self.context, self.migration, instance,
dict(base_copy='hurr', cow='durr'),
network_info, resize_instance=True)
self.assertEqual(self.called, True)
@@ -841,8 +854,9 @@ class XenAPIMigrateInstance(test.TestCase):
def test_finish_migrate_no_local_storage(self):
tiny_type_id = \
instance_types.get_instance_type_by_name('m1.tiny')['id']
- self.values.update({'instance_type_id': tiny_type_id, 'local_gb': 0})
- instance = db.instance_create(self.context, self.values)
+ self.instance_values.update({'instance_type_id': tiny_type_id,
+ 'local_gb': 0})
+ instance = db.instance_create(self.context, self.instance_values)
def fake_vdi_resize(*args, **kwargs):
raise Exception("This shouldn't be called")
@@ -866,12 +880,12 @@ class XenAPIMigrateInstance(test.TestCase):
'label': 'fake',
'mac': 'DE:AD:BE:EF:00:00',
'rxtx_cap': 3})]
- conn.finish_migration(self.context, instance,
+ conn.finish_migration(self.context, self.migration, instance,
dict(base_copy='hurr', cow='durr'),
network_info, resize_instance=True)
def test_finish_migrate_no_resize_vdi(self):
- instance = db.instance_create(self.context, self.values)
+ instance = db.instance_create(self.context, self.instance_values)
def fake_vdi_resize(*args, **kwargs):
raise Exception("This shouldn't be called")
@@ -897,7 +911,7 @@ class XenAPIMigrateInstance(test.TestCase):
'rxtx_cap': 3})]
# Resize instance would be determined by the compute call
- conn.finish_migration(self.context, instance,
+ conn.finish_migration(self.context, self.migration, instance,
dict(base_copy='hurr', cow='durr'),
network_info, resize_instance=False)
diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py
index 647a4c1df..aee279920 100644
--- a/nova/tests/xenapi/stubs.py
+++ b/nova/tests/xenapi/stubs.py
@@ -295,9 +295,12 @@ class FakeSessionForMigrationTests(fake.SessionBase):
vm['is_control_domain'] = False
vm['domid'] = random.randrange(1, 1 << 16)
+ def VM_set_name_label(self, *args):
+ pass
+
def stub_out_migration_methods(stubs):
- def fake_get_snapshot(self, instance):
+ def fake_create_snapshot(self, instance):
return 'vm_ref', dict(image='foo', snap='bar')
@classmethod
@@ -327,7 +330,7 @@ def stub_out_migration_methods(stubs):
stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr)
stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr)
- stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot)
+ stubs.Set(vmops.VMOps, '_create_snapshot', fake_create_snapshot)
stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi)
stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None)
stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path)
diff --git a/nova/utils.py b/nova/utils.py
index 57c93d9d0..c1862f316 100644
--- a/nova/utils.py
+++ b/nova/utils.py
@@ -912,6 +912,20 @@ def convert_to_list_dict(lst, label):
return [{label: x} for x in lst]
+def timefunc(func):
+ """Decorator that logs how long a particular function took to execute"""
+ @functools.wraps(func)
+ def inner(*args, **kwargs):
+ start_time = time.time()
+ try:
+ return func(*args, **kwargs)
+ finally:
+ total_time = time.time() - start_time
+ LOG.debug(_("timefunc: '%(name)s' took %(total_time).2f secs") %
+ dict(name=func.__name__, total_time=total_time))
+ return inner
+
+
def generate_glance_url():
"""Generate the URL to glance."""
# TODO(jk0): This will eventually need to take SSL into consideration
diff --git a/nova/virt/driver.py b/nova/virt/driver.py
index 7edb2cf1a..362d5de57 100644
--- a/nova/virt/driver.py
+++ b/nova/virt/driver.py
@@ -225,12 +225,11 @@ class ComputeDriver(object):
"""
raise NotImplementedError()
- def migrate_disk_and_power_off(self, instance, dest):
+ def migrate_disk_and_power_off(self, context, instance, dest):
"""
Transfers the disk of a running instance in multiple phases, turning
off the instance before the end.
"""
- # TODO(Vek): Need to pass context in for access to auth_token
raise NotImplementedError()
def snapshot(self, context, instance, image_id):
@@ -244,8 +243,8 @@ class ComputeDriver(object):
"""
raise NotImplementedError()
- def finish_migration(self, context, instance, disk_info, network_info,
- resize_instance):
+ def finish_migration(self, context, migration, instance, disk_info,
+ network_info, resize_instance):
"""Completes a resize, turning on the migrated instance
:param network_info:
@@ -253,8 +252,13 @@ class ComputeDriver(object):
"""
raise NotImplementedError()
- def revert_migration(self, instance):
- """Reverts a resize, powering back on the instance"""
+ def confirm_migration(self, migration, instance, network_info):
+ """Confirms a resize, destroying the source VM"""
+ # TODO(Vek): Need to pass context in for access to auth_token
+ raise NotImplementedError()
+
+ def finish_revert_migration(self, instance):
+ """Finish reverting a resize, powering back on the instance"""
# TODO(Vek): Need to pass context in for access to auth_token
raise NotImplementedError()
diff --git a/nova/virt/fake.py b/nova/virt/fake.py
index 96f521ee7..60cc43b01 100644
--- a/nova/virt/fake.py
+++ b/nova/virt/fake.py
@@ -130,10 +130,10 @@ class FakeConnection(driver.ComputeDriver):
def poll_rescued_instances(self, timeout):
pass
- def poll_unconfirmed_resizes(self, resize_confirm_window):
+ def migrate_disk_and_power_off(self, context, instance, dest):
pass
- def migrate_disk_and_power_off(self, instance, dest):
+ def poll_unconfirmed_resizes(self, resize_confirm_window):
pass
def pause(self, instance, callback):
diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py
index 302238c98..5778ca1c2 100644
--- a/nova/virt/xenapi/vm_utils.py
+++ b/nova/virt/xenapi/vm_utils.py
@@ -695,6 +695,10 @@ class VMHelper(HelperBase):
return is_pv
@classmethod
+ def set_vm_name_label(cls, session, vm_ref, name_label):
+ session.get_xenapi().VM.set_name_label(vm_ref, name_label)
+
+ @classmethod
def lookup(cls, session, name_label):
"""Look the instance i up, and returns it if available"""
vm_refs = session.get_xenapi().VM.get_by_name_label(name_label)
diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py
index 988007bae..1dfa5abd1 100644
--- a/nova/virt/xenapi/vmops.py
+++ b/nova/virt/xenapi/vmops.py
@@ -56,6 +56,9 @@ flags.DEFINE_string('xenapi_vif_driver',
'nova.virt.xenapi.vif.XenAPIBridgeDriver',
'The XenAPI VIF driver using XenServer Network APIs.')
+RESIZE_TOTAL_STEPS = 5
+BUILD_TOTAL_STEPS = 4
+
def cmp_version(a, b):
"""Compare two version strings (eg 0.0.1.10 > 0.0.1.9)"""
@@ -111,20 +114,40 @@ class VMOps(object):
instance_infos.append(instance_info)
return instance_infos
- def revert_migration(self, instance):
- vm_ref = VMHelper.lookup(self._session, instance.name)
+ def confirm_migration(self, migration, instance, network_info):
+ name_label = self._get_orig_vm_name_label(instance)
+ vm_ref = VMHelper.lookup(self._session, name_label)
+ return self._destroy(instance, vm_ref, network_info, shutdown=False)
+
+ def finish_revert_migration(self, instance):
+ # NOTE(sirp): the original vm was suffixed with '-orig'; find it using
+ # the old suffix, remove the suffix, then power it back on.
+ name_label = self._get_orig_vm_name_label(instance)
+ vm_ref = VMHelper.lookup(self._session, name_label)
+
+ # Remove the '-orig' suffix (which was added in case the resized VM
+ # ends up on the source host, common during testing)
+ name_label = instance.name
+ VMHelper.set_vm_name_label(self._session, vm_ref, name_label)
+
self._start(instance, vm_ref)
- def finish_migration(self, context, instance, disk_info, network_info,
- resize_instance):
+ def finish_migration(self, context, migration, instance, disk_info,
+ network_info, resize_instance):
vdi_uuid = self.link_disks(instance, disk_info['base_copy'],
disk_info['cow'])
+
vm_ref = self._create_vm(context, instance,
[dict(vdi_type='os', vdi_uuid=vdi_uuid)],
network_info)
if resize_instance:
self.resize_instance(instance, vdi_uuid)
+
+ # 5. Start VM
self._start(instance, vm_ref=vm_ref)
+ self._update_instance_progress(context, instance,
+ step=5,
+ total_steps=RESIZE_TOTAL_STEPS)
def _start(self, instance, vm_ref=None):
"""Power on a VM instance"""
@@ -147,9 +170,35 @@ class VMOps(object):
def spawn(self, context, instance, network_info):
vdis = None
try:
+ # 1. Vanity Step
+ # NOTE(sirp): _create_disk will potentially take a *very* long
+ # time to complete since it has to fetch the image over the
+ # network and images can be several gigs in size. To avoid
+ # progress remaining at 0% for too long, which will appear to be
+ # an error, we insert a "vanity" step to bump the progress up one
+ # notch above 0.
+ self._update_instance_progress(context, instance,
+ step=1,
+ total_steps=BUILD_TOTAL_STEPS)
+
+ # 2. Fetch the Image over the Network
vdis = self._create_disks(context, instance)
+ self._update_instance_progress(context, instance,
+ step=2,
+ total_steps=BUILD_TOTAL_STEPS)
+
+ # 3. Create the VM records
vm_ref = self._create_vm(context, instance, vdis, network_info)
+ self._update_instance_progress(context, instance,
+ step=3,
+ total_steps=BUILD_TOTAL_STEPS)
+
+ # 4. Boot the Instance
self._spawn(instance, vm_ref)
+ self._update_instance_progress(context, instance,
+ step=4,
+ total_steps=BUILD_TOTAL_STEPS)
+
except (self.XenAPI.Failure, OSError, IOError) as spawn_error:
LOG.exception(_("instance %s: Failed to spawn"),
instance.id, exc_info=sys.exc_info())
@@ -169,7 +218,7 @@ class VMOps(object):
if vm_ref is not None:
raise exception.InstanceExists(name=instance_name)
- #ensure enough free memory is available
+ # Ensure enough free memory is available
if not VMHelper.ensure_free_mem(self._session, instance):
LOG.exception(_('instance %(instance_name)s: not enough free '
'memory') % locals())
@@ -501,7 +550,8 @@ class VMOps(object):
"""
template_vm_ref = None
try:
- template_vm_ref, template_vdi_uuids = self._get_snapshot(instance)
+ template_vm_ref, template_vdi_uuids =\
+ self._create_snapshot(instance)
# call plugin to ship snapshot off to glance
VMHelper.upload_image(context,
self._session, instance, template_vdi_uuids, image_id)
@@ -512,7 +562,7 @@ class VMOps(object):
logging.debug(_("Finished snapshot and upload for VM %s"), instance)
- def _get_snapshot(self, instance):
+ def _create_snapshot(self, instance):
#TODO(sirp): Add quiesce and VSS locking support when Windows support
# is added
@@ -529,7 +579,39 @@ class VMOps(object):
% locals())
return
- def migrate_disk_and_power_off(self, instance, dest):
+ def _migrate_vhd(self, instance, vdi_uuid, dest, sr_path):
+ instance_id = instance.id
+ params = {'host': dest,
+ 'vdi_uuid': vdi_uuid,
+ 'instance_id': instance_id,
+ 'sr_path': sr_path}
+
+ task = self._session.async_call_plugin('migration', 'transfer_vhd',
+ {'params': pickle.dumps(params)})
+ self._session.wait_for_task(task, instance_id)
+
+ def _get_orig_vm_name_label(self, instance):
+ return instance.name + '-orig'
+
+ def _update_instance_progress(self, context, instance, step, total_steps):
+ """Update instance progress percent to reflect current step number
+ """
+ # FIXME(sirp): for now we're taking a KISS approach to instance
+ # progress:
+ # Divide the action's workflow into discrete steps and "bump" the
+ # instance's progress field as each step is completed.
+ #
+ # For a first cut this should be fine, however, for large VM images,
+ # the _create_disks step begins to dominate the equation. A
+ # better approximation would use the percentage of the VM image that
+ # has been streamed to the destination host.
+ progress = round(float(step) / total_steps * 100)
+ instance_id = instance['id']
+ LOG.debug(_("Updating instance '%(instance_id)s' progress to"
+ " %(progress)d") % locals())
+ db.instance_update(context, instance_id, {'progress': progress})
+
+ def migrate_disk_and_power_off(self, context, instance, dest):
"""Copies a VHD from one host machine to another.
:param instance: the instance that owns the VHD in question.
@@ -537,6 +619,11 @@ class VMOps(object):
:param disk_type: values are 'primary' or 'cow'.
"""
+ # 0. Zero out the progress to begin
+ self._update_instance_progress(context, instance,
+ step=0,
+ total_steps=RESIZE_TOTAL_STEPS)
+
vm_ref = VMHelper.lookup(self._session, instance.name)
# The primary VDI becomes the COW after the snapshot, and we can
@@ -546,34 +633,43 @@ class VMOps(object):
base_copy_uuid = cow_uuid = None
template_vdi_uuids = template_vm_ref = None
try:
- # transfer the base copy
- template_vm_ref, template_vdi_uuids = self._get_snapshot(instance)
+ # 1. Create Snapshot
+ template_vm_ref, template_vdi_uuids =\
+ self._create_snapshot(instance)
+ self._update_instance_progress(context, instance,
+ step=1,
+ total_steps=RESIZE_TOTAL_STEPS)
+
base_copy_uuid = template_vdi_uuids['image']
vdi_ref, vm_vdi_rec = \
VMHelper.get_vdi_for_vm_safely(self._session, vm_ref)
cow_uuid = vm_vdi_rec['uuid']
- params = {'host': dest,
- 'vdi_uuid': base_copy_uuid,
- 'instance_id': instance.id,
- 'sr_path': VMHelper.get_sr_path(self._session)}
+ sr_path = VMHelper.get_sr_path(self._session)
- task = self._session.async_call_plugin('migration', 'transfer_vhd',
- {'params': pickle.dumps(params)})
- self._session.wait_for_task(task, instance.id)
+ # 2. Transfer the base copy
+ self._migrate_vhd(instance, base_copy_uuid, dest, sr_path)
+ self._update_instance_progress(context, instance,
+ step=2,
+ total_steps=RESIZE_TOTAL_STEPS)
- # Now power down the instance and transfer the COW VHD
+ # 3. Now power down the instance
self._shutdown(instance, vm_ref, hard=False)
-
- params = {'host': dest,
- 'vdi_uuid': cow_uuid,
- 'instance_id': instance.id,
- 'sr_path': VMHelper.get_sr_path(self._session), }
-
- task = self._session.async_call_plugin('migration', 'transfer_vhd',
- {'params': pickle.dumps(params)})
- self._session.wait_for_task(task, instance.id)
-
+ self._update_instance_progress(context, instance,
+ step=3,
+ total_steps=RESIZE_TOTAL_STEPS)
+
+ # 4. Transfer the COW VHD
+ self._migrate_vhd(instance, cow_uuid, dest, sr_path)
+ self._update_instance_progress(context, instance,
+ step=4,
+ total_steps=RESIZE_TOTAL_STEPS)
+
+ # NOTE(sirp): in case we're resizing to the same host (for dev
+ # purposes), apply a suffix to name-label so the two VM records
+ # extant until a confirm_resize don't collide.
+ name_label = self._get_orig_vm_name_label(instance)
+ VMHelper.set_vm_name_label(self._session, vm_ref, name_label)
finally:
if template_vm_ref:
self._destroy(instance, template_vm_ref,
diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py
index 79b02891d..e63a06ad2 100644
--- a/nova/virt/xenapi_conn.py
+++ b/nova/virt/xenapi_conn.py
@@ -189,14 +189,19 @@ class XenAPIConnection(driver.ComputeDriver):
"""Create VM instance"""
self._vmops.spawn(context, instance, network_info)
- def revert_migration(self, instance):
- """Reverts a resize, powering back on the instance"""
- self._vmops.revert_migration(instance)
+ def confirm_migration(self, migration, instance, network_info):
+ """Confirms a resize, destroying the source VM"""
+ # TODO(Vek): Need to pass context in for access to auth_token
+ self._vmops.confirm_migration(migration, instance, network_info)
- def finish_migration(self, context, instance, disk_info, network_info,
- resize_instance=False):
+ def finish_revert_migration(self, instance):
+ """Finish reverting a resize, powering back on the instance"""
+ self._vmops.finish_revert_migration(instance)
+
+ def finish_migration(self, context, migration, instance, disk_info,
+ network_info, resize_instance=False):
"""Completes a resize, turning on the migrated instance"""
- self._vmops.finish_migration(context, instance, disk_info,
+ self._vmops.finish_migration(context, migration, instance, disk_info,
network_info, resize_instance)
def snapshot(self, context, instance, image_id):
@@ -229,10 +234,10 @@ class XenAPIConnection(driver.ComputeDriver):
"""Unpause paused VM instance"""
self._vmops.unpause(instance, callback)
- def migrate_disk_and_power_off(self, instance, dest):
+ def migrate_disk_and_power_off(self, context, instance, dest):
"""Transfers the VHD of a running instance to another host, then shuts
off the instance copies over the COW disk"""
- return self._vmops.migrate_disk_and_power_off(instance, dest)
+ return self._vmops.migrate_disk_and_power_off(context, instance, dest)
def suspend(self, instance, callback):
"""suspend the specified instance"""
diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
index 1a9ac37e9..950b78707 100755
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance
@@ -211,7 +211,11 @@ def _import_vhds(sr_path, staging_path, uuid_stack):
snap_info = prepare_if_exists(staging_path, 'snap.vhd',
image_info[0])
if snap_info:
- paths_to_move.append(snap_info[0])
+ # NOTE(sirp): this is an insert rather than an append since the
+ # 'snapshot' vhd needs to be copied into the SR before the base copy.
+ # If it doesn't, then there is a possibliity that snapwatchd will
+ # delete the base_copy since it is an unreferenced parent.
+ paths_to_move.insert(0, snap_info[0])
# We return this snap as the VDI instead of image.vhd
vdi_return_list.append(dict(vdi_type="os", vdi_uuid=snap_info[1]))
else:
diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration
index ac1c50ad9..4ec3dc7af 100755
--- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration
+++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration
@@ -48,7 +48,7 @@ def move_vhds_into_sr(session, args):
# Discover the copied VHDs locally, and then set up paths to copy
# them to under the SR
- source_image_path = "%s/instance%d" % ('/images/', instance_id)
+ source_image_path = "/images/instance%d" % instance_id
source_base_copy_path = "%s/%s.vhd" % (source_image_path,
old_base_copy_uuid)
source_cow_path = "%s/%s.vhd" % (source_image_path, old_cow_uuid)
@@ -74,9 +74,12 @@ def move_vhds_into_sr(session, args):
(new_cow_path, new_base_copy_path))
subprocess.call(shlex.split('/usr/sbin/vhd-util modify -n %s -p %s' %
(new_cow_path, new_base_copy_path)))
+
logging.debug('Moving VHDs into SR %s' % sr_path)
- shutil.move("%s/%s.vhd" % (temp_vhd_path, new_base_copy_uuid), sr_path)
- shutil.move("%s/%s.vhd" % (temp_vhd_path, new_cow_uuid), sr_path)
+ # NOTE(sirp): COW should be copied before base_copy to avoid snapwatchd
+ # GC'ing an unreferenced base copy VDI
+ shutil.move(new_cow_path, sr_path)
+ shutil.move(new_base_copy_path, sr_path)
logging.debug('Cleaning up temporary SR path %s' % temp_vhd_path)
os.rmdir(temp_vhd_path)
@@ -93,7 +96,7 @@ def transfer_vhd(session, args):
vhd_path = "%s.vhd" % vdi_uuid
source_path = "%s/%s" % (sr_path, vhd_path)
- dest_path = '%s:%sinstance%d/' % (host, '/images/', instance_id)
+ dest_path = '%s:/images/instance%d/' % (host, instance_id)
logging.debug("Preparing to transmit %s to %s" % (source_path,
dest_path))