From 0f56d8ddb02f54ae389380dcd0790e55f2dcb479 Mon Sep 17 00:00:00 2001 From: Hans Lindgren Date: Wed, 17 Apr 2013 15:52:56 +0200 Subject: Optimize instance queries in compute manager Some instance queries against the db through the conductor can be optimized to return a reduced set of instances as needed for the job at hand. Most of these are part of periodic tasks, so the win is kind of big. Compute methods where queries can be made more efficient: _get_instances_on_driver _poll_rebooting_instances _poll_rescued_instances _reclaim_queued_deletes _run_image_cache_manager_pass Resolves bug 1169970. Change-Id: I7c2fab48944e34765b3fff8ce10bc64a5cd826c8 --- nova/tests/compute/test_compute.py | 61 +++++++++++++++++----------------- nova/tests/conductor/test_conductor.py | 18 ---------- nova/tests/test_imagecache.py | 5 +-- 3 files changed, 33 insertions(+), 51 deletions(-) (limited to 'nova/tests') diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py index 88434078d..810d8e91e 100644 --- a/nova/tests/compute/test_compute.py +++ b/nova/tests/compute/test_compute.py @@ -3594,13 +3594,15 @@ class ComputeTestCase(BaseTestCase): deleted_at = (timeutils.utcnow() - datetime.timedelta(hours=1, minutes=5)) instance = self._create_fake_instance({"deleted_at": deleted_at, - "deleted": True}) + "deleted": True}) self.compute.host = instance['host'] self.mox.StubOutWithMock(self.compute, '_get_instances_on_driver') - self.compute._get_instances_on_driver(admin_context).AndReturn( - [instance]) + self.compute._get_instances_on_driver( + admin_context, {'deleted': True, + 'soft_deleted': False, + 'host': self.compute.host}).AndReturn([instance]) self.flags(running_deleted_instance_timeout=3600, running_deleted_instance_action='reap') @@ -3628,13 +3630,11 @@ class ComputeTestCase(BaseTestCase): instance1['deleted'] = True instance1['deleted_at'] = "sometimeago" - instance2 = {} - instance2['deleted'] = False - instance2['deleted_at'] = None - self.mox.StubOutWithMock(self.compute, '_get_instances_on_driver') - self.compute._get_instances_on_driver(admin_context).AndReturn( - [instance1, instance2]) + self.compute._get_instances_on_driver( + admin_context, {'deleted': True, + 'soft_deleted': False, + 'host': self.compute.host}).AndReturn([instance1]) self.mox.StubOutWithMock(timeutils, 'is_older_than') timeutils.is_older_than('sometimeago', @@ -3752,25 +3752,23 @@ class ComputeTestCase(BaseTestCase): instances = [{'uuid': 'fake_uuid1', 'vm_state': vm_states.RESCUED, 'launched_at': timed_out_time}, - {'uuid': 'fake_uuid2', 'vm_state': vm_states.ACTIVE, - 'launched_at': timed_out_time}, - {'uuid': 'fake_uuid3', 'vm_state': vm_states.ACTIVE, - 'launched_at': not_timed_out_time}, - {'uuid': 'fake_uuid4', 'vm_state': vm_states.RESCUED, + {'uuid': 'fake_uuid2', 'vm_state': vm_states.RESCUED, 'launched_at': timed_out_time}, - {'uuid': 'fake_uuid5', 'vm_state': vm_states.RESCUED, + {'uuid': 'fake_uuid3', 'vm_state': vm_states.RESCUED, 'launched_at': not_timed_out_time}] - unrescued_instances = {'fake_uuid1': False, 'fake_uuid4': False} + unrescued_instances = {'fake_uuid1': False, 'fake_uuid2': False} - def fake_instance_get_all_by_host(context, host, columns_to_join): + def fake_instance_get_all_by_filters(context, filters, + columns_to_join): self.assertEqual(columns_to_join, []) return instances def fake_unrescue(context, instance): unrescued_instances[instance['uuid']] = True - self.stubs.Set(self.compute.conductor_api, 'instance_get_all_by_host', - fake_instance_get_all_by_host) + self.stubs.Set(self.compute.conductor_api, + 'instance_get_all_by_filters', + fake_instance_get_all_by_filters) self.stubs.Set(self.compute.conductor_api, 'compute_unrescue', fake_unrescue) @@ -4020,8 +4018,8 @@ class ComputeTestCase(BaseTestCase): self.mox.StubOutWithMock(self.compute, '_legacy_nw_info') self.mox.StubOutWithMock(self.compute.driver, 'destroy') - self.compute._get_instances_on_driver(fake_context).AndReturn( - instances) + self.compute._get_instances_on_driver( + fake_context, {'deleted': False}).AndReturn(instances) self.compute._get_instance_nw_info(fake_context, evacuated_instance).AndReturn( 'fake_network_info') @@ -4073,8 +4071,8 @@ class ComputeTestCase(BaseTestCase): self.mox.StubOutWithMock(self.compute, '_legacy_nw_info') self.mox.StubOutWithMock(self.compute.driver, 'destroy') - self.compute._get_instances_on_driver(fake_context).AndReturn( - instances) + self.compute._get_instances_on_driver( + fake_context, {'deleted': False}).AndReturn(instances) self.compute._get_instance_nw_info(fake_context, evacuated_instance).AndReturn( 'fake_network_info') @@ -4131,8 +4129,8 @@ class ComputeTestCase(BaseTestCase): self.mox.StubOutWithMock(self.compute, '_legacy_nw_info') self.mox.StubOutWithMock(self.compute.driver, 'destroy') - self.compute._get_instances_on_driver(fake_context).AndReturn( - instances) + self.compute._get_instances_on_driver( + fake_context, {'deleted': False}).AndReturn(instances) self.compute._get_instance_nw_info(fake_context, evacuated_instance).AndReturn( 'fake_network_info') @@ -4238,8 +4236,8 @@ class ComputeTestCase(BaseTestCase): self.compute.init_virt_events() # simulate failed instance - self.compute._get_instances_on_driver(fake_context).AndReturn([ - deleted_instance]) + self.compute._get_instances_on_driver( + fake_context, {'deleted': False}).AndReturn([deleted_instance]) self.compute._get_instance_nw_info(fake_context, deleted_instance ).AndRaise(exception.InstanceNotFound( instance_id=deleted_instance['uuid'])) @@ -4398,6 +4396,7 @@ class ComputeTestCase(BaseTestCase): # Test getting instances when driver doesn't support # 'list_instance_uuids' self.compute.host = 'host' + filters = {'host': self.compute.host} fake_context = context.get_admin_context() all_instances = [] @@ -4413,19 +4412,19 @@ class ComputeTestCase(BaseTestCase): self.mox.StubOutWithMock(self.compute.driver, 'list_instances') self.mox.StubOutWithMock(self.compute.conductor_api, - 'instance_get_all_by_host') + 'instance_get_all_by_filters') self.compute.driver.list_instance_uuids().AndRaise( NotImplementedError()) self.compute.driver.list_instances().AndReturn( [inst['name'] for inst in driver_instances]) - self.compute.conductor_api.instance_get_all_by_host( - fake_context, self.compute.host, + self.compute.conductor_api.instance_get_all_by_filters( + fake_context, filters, columns_to_join=[]).AndReturn(all_instances) self.mox.ReplayAll() - result = self.compute._get_instances_on_driver(fake_context) + result = self.compute._get_instances_on_driver(fake_context, filters) self.assertEqual(driver_instances, result) def test_instance_usage_audit(self): diff --git a/nova/tests/conductor/test_conductor.py b/nova/tests/conductor/test_conductor.py index fee919a33..9fc184bea 100644 --- a/nova/tests/conductor/test_conductor.py +++ b/nova/tests/conductor/test_conductor.py @@ -341,12 +341,6 @@ class _BaseTestCase(object): self.context, fake_inst) self.assertEqual(result, 'fake-result') - def test_instance_get_all_hung_in_rebooting(self): - self.mox.StubOutWithMock(db, 'instance_get_all_hung_in_rebooting') - db.instance_get_all_hung_in_rebooting(self.context, 123) - self.mox.ReplayAll() - self.conductor.instance_get_all_hung_in_rebooting(self.context, 123) - def test_instance_get_active_by_window_joined(self): self.mox.StubOutWithMock(db, 'instance_get_active_by_window_joined') db.instance_get_active_by_window_joined(self.context, 'fake-begin', @@ -978,18 +972,6 @@ class ConductorAPITestCase(_BaseTestCase, test.TestCase): self.conductor.block_device_mapping_destroy_by_instance_and_volume( self.context, fake_inst, 'fake-volume') - def test_instance_get_all(self): - self.mox.StubOutWithMock(db, 'instance_get_all_by_filters') - db.instance_get_all(self.context) - db.instance_get_all_by_filters(self.context, {'name': 'fake-inst'}, - 'updated_at', 'asc', - columns_to_join=None) - self.mox.ReplayAll() - self.conductor.instance_get_all(self.context) - self.conductor.instance_get_all_by_filters(self.context, - {'name': 'fake-inst'}, - 'updated_at', 'asc') - def _test_stubbed(self, name, *args, **kwargs): if args and isinstance(args[0], FakeContext): ctxt = args[0] diff --git a/nova/tests/test_imagecache.py b/nova/tests/test_imagecache.py index 02cc7337a..d771eff83 100644 --- a/nova/tests/test_imagecache.py +++ b/nova/tests/test_imagecache.py @@ -945,7 +945,7 @@ class ImageCacheManagerTestCase(test.TestCase): def test_compute_manager(self): was = {'called': False} - def fake_get_all(context, *args, **kwargs): + def fake_get_all_by_filters(context, *args, **kwargs): was['called'] = True return [{'image_ref': '1', 'host': CONF.host, @@ -963,7 +963,8 @@ class ImageCacheManagerTestCase(test.TestCase): with utils.tempdir() as tmpdir: self.flags(instances_path=tmpdir) - self.stubs.Set(db, 'instance_get_all', fake_get_all) + self.stubs.Set(db, 'instance_get_all_by_filters', + fake_get_all_by_filters) compute = importutils.import_object(CONF.compute_manager) self.flags(use_local=True, group='conductor') compute.conductor_api = conductor.API() -- cgit