From 0dbd2c5d3360a71658a146cb3947047e797a56f6 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Mon, 14 Jan 2013 04:27:50 +0000 Subject: Cells: Add support for compute HostAPI() This will allow the hosts API extension to work. Calls to get service information from the DB are proxied via cells. When using the host extension, the 'id' and 'host' fields for service/compute will be modified to have the cell prepended to it. One can do a 'show' on a 'path!to!cell@id' to see details for a particular host in the 'path!to!cell' cell, for instance. Implements bp/nova-compute-cells Change-Id: I8137fa64d2578acfc3d2457c231de2b0d69b96d8 --- nova/tests/cells/test_cells_manager.py | 86 +++++++++++++- nova/tests/cells/test_cells_messaging.py | 139 +++++++++++++++++++++- nova/tests/cells/test_cells_rpcapi.py | 35 ++++++ nova/tests/compute/test_host_api.py | 190 +++++++++++++++++++++++++++++-- 4 files changed, 430 insertions(+), 20 deletions(-) (limited to 'nova/tests') diff --git a/nova/tests/cells/test_cells_manager.py b/nova/tests/cells/test_cells_manager.py index ef165f4ed..d3d412af7 100644 --- a/nova/tests/cells/test_cells_manager.py +++ b/nova/tests/cells/test_cells_manager.py @@ -15,15 +15,29 @@ """ Tests For CellsManager """ +import copy import datetime from nova.cells import messaging from nova.cells import utils as cells_utils from nova import context +from nova.openstack.common import cfg +from nova.openstack.common import rpc from nova.openstack.common import timeutils from nova import test from nova.tests.cells import fakes +CONF = cfg.CONF +CONF.import_opt('compute_topic', 'nova.compute.rpcapi') + + +FAKE_COMPUTE_NODES = [dict(id=1), dict(id=2)] +FAKE_SERVICES = [dict(id=1, host='host1', + compute_node=[FAKE_COMPUTE_NODES[0]]), + dict(id=2, host='host2', + compute_node=[FAKE_COMPUTE_NODES[1]]), + dict(id=3, host='host3', compute_node=[])] + class CellsManagerClassTestCase(test.TestCase): """Test case for CellsManager class.""" @@ -46,6 +60,14 @@ class CellsManagerClassTestCase(test.TestCase): expected_responses.append(('cell%s' % x, x)) return expected_responses, responses + def _get_fake_response(self, raw_response=None, exc=False): + if exc: + return messaging.Response('fake', test.TestingException(), + True) + if raw_response is None: + raw_response = 'fake-response' + return messaging.Response('fake', raw_response, False) + def test_get_cell_info_for_neighbors(self): self.mox.StubOutWithMock(self.cells_manager.state_manager, 'get_cell_info_for_neighbors') @@ -109,17 +131,13 @@ class CellsManagerClassTestCase(test.TestCase): cell_name = 'fake-cell-name' method_info = 'fake-method-info' - fake_response = messaging.Response('fake', 'fake', False) - self.mox.StubOutWithMock(self.msg_runner, 'run_compute_api_method') - self.mox.StubOutWithMock(fake_response, - 'value_or_raise') + fake_response = self._get_fake_response() self.msg_runner.run_compute_api_method(self.ctxt, cell_name, method_info, True).AndReturn(fake_response) - fake_response.value_or_raise().AndReturn('fake-response') self.mox.ReplayAll() response = self.cells_manager.run_compute_api_method( self.ctxt, cell_name=cell_name, method_info=method_info, @@ -237,3 +255,61 @@ class CellsManagerClassTestCase(test.TestCase): project_id='fake-project', updated_since='fake-time', deleted='fake-deleted') + + def test_service_get_all(self): + responses = [] + expected_response = [] + # 3 cells... so 3 responses. Each response is a list of services. + # Manager should turn these into a single list of responses. + for i in xrange(3): + cell_name = 'path!to!cell%i' % i + services = [] + for service in FAKE_SERVICES: + services.append(copy.deepcopy(service)) + expected_service = copy.deepcopy(service) + cells_utils.add_cell_to_service(expected_service, cell_name) + expected_response.append(expected_service) + response = messaging.Response(cell_name, services, False) + responses.append(response) + + self.mox.StubOutWithMock(self.msg_runner, + 'service_get_all') + self.msg_runner.service_get_all(self.ctxt, + 'fake-filters').AndReturn(responses) + self.mox.ReplayAll() + response = self.cells_manager.service_get_all(self.ctxt, + filters='fake-filters') + self.assertEqual(expected_response, response) + + def test_service_get_by_compute_host(self): + self.mox.StubOutWithMock(self.msg_runner, + 'service_get_by_compute_host') + fake_cell = 'fake-cell' + fake_response = messaging.Response(fake_cell, FAKE_SERVICES[0], + False) + expected_response = copy.deepcopy(FAKE_SERVICES[0]) + cells_utils.add_cell_to_service(expected_response, fake_cell) + + cell_and_host = cells_utils.cell_with_item('fake-cell', 'fake-host') + self.msg_runner.service_get_by_compute_host(self.ctxt, + fake_cell, 'fake-host').AndReturn(fake_response) + self.mox.ReplayAll() + response = self.cells_manager.service_get_by_compute_host(self.ctxt, + host_name=cell_and_host) + self.assertEqual(expected_response, response) + + def test_proxy_rpc_to_manager(self): + self.mox.StubOutWithMock(self.msg_runner, + 'proxy_rpc_to_manager') + fake_response = self._get_fake_response() + cell_and_host = cells_utils.cell_with_item('fake-cell', 'fake-host') + topic = rpc.queue_get_for(self.ctxt, CONF.compute_topic, + cell_and_host) + self.msg_runner.proxy_rpc_to_manager(self.ctxt, 'fake-cell', + 'fake-host', topic, 'fake-rpc-msg', + True, -1).AndReturn(fake_response) + self.mox.ReplayAll() + response = self.cells_manager.proxy_rpc_to_manager(self.ctxt, + topic=topic, rpc_message='fake-rpc-msg', call=True, + timeout=-1) + self.assertEqual('fake-response', response) diff --git a/nova/tests/cells/test_cells_messaging.py b/nova/tests/cells/test_cells_messaging.py index da45721ed..f6c51325a 100644 --- a/nova/tests/cells/test_cells_messaging.py +++ b/nova/tests/cells/test_cells_messaging.py @@ -14,13 +14,12 @@ """ Tests For Cells Messaging module """ -import mox - from nova.cells import messaging from nova.cells import utils as cells_utils from nova import context from nova import exception from nova.openstack.common import cfg +from nova.openstack.common import rpc from nova.openstack.common import timeutils from nova import test from nova.tests.cells import fakes @@ -604,7 +603,7 @@ class CellsTargetedMethodsTestCase(test.TestCase): self.tgt_cell_name, host_sched_kwargs) - def test_call_compute_api_method(self): + def test_run_compute_api_method(self): instance_uuid = 'fake_instance_uuid' method_info = {'method': 'reboot', @@ -614,8 +613,7 @@ class CellsTargetedMethodsTestCase(test.TestCase): self.mox.StubOutWithMock(self.tgt_db_inst, 'instance_get_by_uuid') self.tgt_db_inst.instance_get_by_uuid(self.ctxt, - instance_uuid).AndReturn( - 'fake_instance') + instance_uuid).AndReturn('fake_instance') self.tgt_compute_api.reboot(self.ctxt, 'fake_instance', 2, 3, arg1='val1', arg2='val2').AndReturn('fake_result') self.mox.ReplayAll() @@ -628,7 +626,7 @@ class CellsTargetedMethodsTestCase(test.TestCase): result = response.value_or_raise() self.assertEqual('fake_result', result) - def test_call_compute_api_method_unknown_instance(self): + def test_run_compute_api_method_unknown_instance(self): # Unknown instance should send a broadcast up that instance # is gone. instance_uuid = 'fake_instance_uuid' @@ -727,6 +725,70 @@ class CellsTargetedMethodsTestCase(test.TestCase): self.src_msg_runner.ask_children_for_capacities(self.ctxt) + def test_service_get_by_compute_host(self): + fake_host_name = 'fake-host-name' + + self.mox.StubOutWithMock(self.tgt_db_inst, + 'service_get_by_compute_host') + + self.tgt_db_inst.service_get_by_compute_host(self.ctxt, + fake_host_name).AndReturn('fake-service') + self.mox.ReplayAll() + + response = self.src_msg_runner.service_get_by_compute_host( + self.ctxt, + self.tgt_cell_name, + fake_host_name) + result = response.value_or_raise() + self.assertEqual('fake-service', result) + + def test_proxy_rpc_to_manager_call(self): + fake_topic = 'fake-topic' + fake_rpc_message = 'fake-rpc-message' + fake_host_name = 'fake-host-name' + + self.mox.StubOutWithMock(self.tgt_db_inst, + 'service_get_by_compute_host') + self.mox.StubOutWithMock(rpc, 'call') + + self.tgt_db_inst.service_get_by_compute_host(self.ctxt, + fake_host_name) + rpc.call(self.ctxt, fake_topic, + fake_rpc_message, timeout=5).AndReturn('fake_result') + + self.mox.ReplayAll() + + response = self.src_msg_runner.proxy_rpc_to_manager( + self.ctxt, + self.tgt_cell_name, + fake_host_name, + fake_topic, + fake_rpc_message, True, timeout=5) + result = response.value_or_raise() + self.assertEqual('fake_result', result) + + def test_proxy_rpc_to_manager_cast(self): + fake_topic = 'fake-topic' + fake_rpc_message = 'fake-rpc-message' + fake_host_name = 'fake-host-name' + + self.mox.StubOutWithMock(self.tgt_db_inst, + 'service_get_by_compute_host') + self.mox.StubOutWithMock(rpc, 'cast') + + self.tgt_db_inst.service_get_by_compute_host(self.ctxt, + fake_host_name) + rpc.cast(self.ctxt, fake_topic, fake_rpc_message) + + self.mox.ReplayAll() + + self.src_msg_runner.proxy_rpc_to_manager( + self.ctxt, + self.tgt_cell_name, + fake_host_name, + fake_topic, + fake_rpc_message, False, timeout=None) + class CellsBroadcastMethodsTestCase(test.TestCase): """Test case for _BroadcastMessageMethods class. Most of these @@ -756,6 +818,13 @@ class CellsBroadcastMethodsTestCase(test.TestCase): self.src_db_inst = methods_cls.db self.src_compute_api = methods_cls.compute_api + if not up: + # fudge things so we only have 1 child to broadcast to + state_manager = self.src_msg_runner.state_manager + for cell in state_manager.get_child_cells(): + if cell.name != 'child-cell2': + del state_manager.child_cells[cell.name] + self.mid_msg_runner = fakes.get_message_runner(mid_cell) methods_cls = self.mid_msg_runner.methods_by_type['broadcast'] self.mid_methods_cls = methods_cls @@ -958,3 +1027,61 @@ class CellsBroadcastMethodsTestCase(test.TestCase): self.src_msg_runner.sync_instances(self.ctxt, project_id, updated_since_raw, deleted) + + def test_service_get_all_with_disabled(self): + # Reset this, as this is a broadcast down. + self._setup_attrs(up=False) + + ctxt = self.ctxt.elevated() + + self.mox.StubOutWithMock(self.src_db_inst, 'service_get_all') + self.mox.StubOutWithMock(self.mid_db_inst, 'service_get_all') + self.mox.StubOutWithMock(self.tgt_db_inst, 'service_get_all') + + self.src_db_inst.service_get_all(ctxt, + disabled=None).AndReturn([1, 2]) + self.mid_db_inst.service_get_all(ctxt, + disabled=None).AndReturn([3]) + self.tgt_db_inst.service_get_all(ctxt, + disabled=None).AndReturn([4, 5]) + + self.mox.ReplayAll() + + responses = self.src_msg_runner.service_get_all(ctxt, + filters={}) + response_values = [(resp.cell_name, resp.value_or_raise()) + for resp in responses] + expected = [('api-cell!child-cell2!grandchild-cell1', [4, 5]), + ('api-cell!child-cell2', [3]), + ('api-cell', [1, 2])] + self.assertEqual(expected, response_values) + + def test_service_get_all_without_disabled(self): + # Reset this, as this is a broadcast down. + self._setup_attrs(up=False) + disabled = False + filters = {'disabled': disabled} + + ctxt = self.ctxt.elevated() + + self.mox.StubOutWithMock(self.src_db_inst, 'service_get_all') + self.mox.StubOutWithMock(self.mid_db_inst, 'service_get_all') + self.mox.StubOutWithMock(self.tgt_db_inst, 'service_get_all') + + self.src_db_inst.service_get_all(ctxt, + disabled=disabled).AndReturn([1, 2]) + self.mid_db_inst.service_get_all(ctxt, + disabled=disabled).AndReturn([3]) + self.tgt_db_inst.service_get_all(ctxt, + disabled=disabled).AndReturn([4, 5]) + + self.mox.ReplayAll() + + responses = self.src_msg_runner.service_get_all(ctxt, + filters=filters) + response_values = [(resp.cell_name, resp.value_or_raise()) + for resp in responses] + expected = [('api-cell!child-cell2!grandchild-cell1', [4, 5]), + ('api-cell!child-cell2', [3]), + ('api-cell', [1, 2])] + self.assertEqual(expected, response_values) diff --git a/nova/tests/cells/test_cells_rpcapi.py b/nova/tests/cells/test_cells_rpcapi.py index 5e045aca9..876bc5ce5 100644 --- a/nova/tests/cells/test_cells_rpcapi.py +++ b/nova/tests/cells/test_cells_rpcapi.py @@ -224,3 +224,38 @@ class CellsAPITestCase(test.TestCase): 'deleted': True} self._check_result(call_info, 'sync_instances', expected_args, version='1.1') + + def test_service_get_all(self): + call_info = self._stub_rpc_method('call', 'fake_response') + fake_filters = {'key1': 'val1', 'key2': 'val2'} + result = self.cells_rpcapi.service_get_all(self.fake_context, + filters=fake_filters) + + expected_args = {'filters': fake_filters} + self._check_result(call_info, 'service_get_all', expected_args, + version='1.2') + self.assertEqual(result, 'fake_response') + + def test_service_get_by_compute_host(self): + call_info = self._stub_rpc_method('call', 'fake_response') + result = self.cells_rpcapi.service_get_by_compute_host( + self.fake_context, host_name='fake-host-name') + expected_args = {'host_name': 'fake-host-name'} + self._check_result(call_info, 'service_get_by_compute_host', + expected_args, + version='1.2') + self.assertEqual(result, 'fake_response') + + def test_proxy_rpc_to_manager(self): + call_info = self._stub_rpc_method('call', 'fake_response') + result = self.cells_rpcapi.proxy_rpc_to_manager( + self.fake_context, rpc_message='fake-msg', + topic='fake-topic', call=True, timeout=-1) + expected_args = {'rpc_message': 'fake-msg', + 'topic': 'fake-topic', + 'call': True, + 'timeout': -1} + self._check_result(call_info, 'proxy_rpc_to_manager', + expected_args, + version='1.2') + self.assertEqual(result, 'fake_response') diff --git a/nova/tests/compute/test_host_api.py b/nova/tests/compute/test_host_api.py index 95d3c4926..151743715 100644 --- a/nova/tests/compute/test_host_api.py +++ b/nova/tests/compute/test_host_api.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nova.cells import utils as cells_utils from nova import compute from nova.compute import rpcapi as compute_rpcapi from nova import context @@ -84,6 +85,42 @@ class ComputeHostAPITestCase(test.TestCase): 'fake_mode') self.assertEqual('fake-result', result) + def test_service_get_all_no_zones(self): + services = [dict(id=1, key1='val1', key2='val2', topic='compute', + host='host1'), + dict(id=2, key1='val2', key3='val3', topic='compute', + host='host2')] + + self.mox.StubOutWithMock(self.host_api.db, + 'service_get_all') + + # Test no filters + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt) + self.mox.VerifyAll() + self.assertEqual(services, result) + + # Test no filters #2 + self.mox.ResetAll() + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt, filters={}) + self.mox.VerifyAll() + self.assertEqual(services, result) + + # Test w/ filter + self.mox.ResetAll() + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt, + filters=dict(key1='val2')) + self.mox.VerifyAll() + self.assertEqual([services[1]], result) + def test_service_get_all(self): services = [dict(id=1, key1='val1', key2='val2', topic='compute', host='host1'), @@ -99,28 +136,163 @@ class ComputeHostAPITestCase(test.TestCase): 'service_get_all') # Test no filters - self.host_api.db.service_get_all(self.ctxt, False).AndReturn( - services) + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) self.mox.ReplayAll() - result = self.host_api.service_get_all(self.ctxt) + result = self.host_api.service_get_all(self.ctxt, set_zones=True) self.mox.VerifyAll() self.assertEqual(exp_services, result) # Test no filters #2 self.mox.ResetAll() - self.host_api.db.service_get_all(self.ctxt, False).AndReturn( - services) + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) self.mox.ReplayAll() - result = self.host_api.service_get_all(self.ctxt, filters={}) + result = self.host_api.service_get_all(self.ctxt, filters={}, + set_zones=True) self.mox.VerifyAll() self.assertEqual(exp_services, result) # Test w/ filter self.mox.ResetAll() - self.host_api.db.service_get_all(self.ctxt, False).AndReturn( - services) + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) self.mox.ReplayAll() result = self.host_api.service_get_all(self.ctxt, - filters=dict(key1='val2')) + filters=dict(key1='val2'), + set_zones=True) self.mox.VerifyAll() self.assertEqual([exp_services[1]], result) + + # Test w/ zone filter but no set_zones arg. + self.mox.ResetAll() + self.host_api.db.service_get_all(self.ctxt, + disabled=None).AndReturn(services) + self.mox.ReplayAll() + filters = {'availability_zone': 'nova'} + result = self.host_api.service_get_all(self.ctxt, + filters=filters) + self.mox.VerifyAll() + self.assertEqual(exp_services, result) + + def test_service_get_by_compute_host(self): + self.mox.StubOutWithMock(self.host_api.db, + 'service_get_by_compute_host') + + self.host_api.db.service_get_by_compute_host(self.ctxt, + 'fake-host').AndReturn('fake-response') + self.mox.ReplayAll() + result = self.host_api.service_get_by_compute_host(self.ctxt, + 'fake-host') + self.assertEqual('fake-response', result) + + def test_instance_get_all_by_host(self): + self.mox.StubOutWithMock(self.host_api.db, + 'instance_get_all_by_host') + + self.host_api.db.instance_get_all_by_host(self.ctxt, + 'fake-host').AndReturn(['fake-responses']) + self.mox.ReplayAll() + result = self.host_api.instance_get_all_by_host(self.ctxt, + 'fake-host') + self.assertEqual(['fake-responses'], result) + + +class ComputeHostAPICellsTestCase(ComputeHostAPITestCase): + def setUp(self): + self.flags(compute_api_class='nova.compute.cells_api.ComputeCellsAPI') + super(ComputeHostAPICellsTestCase, self).setUp() + + def _mock_rpc_call(self, expected_message, result=None): + if result is None: + result = 'fake-result' + # Wrapped with cells call + expected_message = {'method': 'proxy_rpc_to_manager', + 'args': {'topic': 'compute.fake_host', + 'rpc_message': expected_message, + 'call': True, + 'timeout': None}, + 'version': '1.2'} + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(self.ctxt, 'cells', expected_message, + None).AndReturn(result) + + def test_service_get_all_no_zones(self): + services = [dict(id=1, key1='val1', key2='val2', topic='compute', + host='host1'), + dict(id=2, key1='val2', key3='val3', topic='compute', + host='host2')] + + fake_filters = {'key1': 'val1'} + self.mox.StubOutWithMock(self.host_api.cells_rpcapi, + 'service_get_all') + self.host_api.cells_rpcapi.service_get_all(self.ctxt, + filters=fake_filters).AndReturn(services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt, + filters=fake_filters) + self.assertEqual(services, result) + + def test_service_get_all(self): + services = [dict(id=1, key1='val1', key2='val2', topic='compute', + host='host1'), + dict(id=2, key1='val2', key3='val3', topic='compute', + host='host2')] + exp_services = [] + for service in services: + exp_service = {} + exp_service.update(availability_zone='nova', **service) + exp_services.append(exp_service) + + fake_filters = {'key1': 'val1'} + self.mox.StubOutWithMock(self.host_api.cells_rpcapi, + 'service_get_all') + self.host_api.cells_rpcapi.service_get_all(self.ctxt, + filters=fake_filters).AndReturn(services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt, + filters=fake_filters, + set_zones=True) + self.mox.VerifyAll() + self.assertEqual(exp_services, result) + + # Test w/ zone filter but no set_zones arg. + self.mox.ResetAll() + fake_filters = {'availability_zone': 'nova'} + # Zone filter is done client-size, so should be stripped + # from this call. + self.host_api.cells_rpcapi.service_get_all(self.ctxt, + filters={}).AndReturn(services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt, + filters=fake_filters) + self.mox.VerifyAll() + self.assertEqual(exp_services, result) + + def test_service_get_by_compute_host(self): + self.mox.StubOutWithMock(self.host_api.cells_rpcapi, + 'service_get_by_compute_host') + + self.host_api.cells_rpcapi.service_get_by_compute_host(self.ctxt, + 'fake-host').AndReturn('fake-response') + self.mox.ReplayAll() + result = self.host_api.service_get_by_compute_host(self.ctxt, + 'fake-host') + self.assertEqual('fake-response', result) + + def test_instance_get_all_by_host(self): + instances = [dict(id=1, cell_name='cell1', host='host1'), + dict(id=2, cell_name='cell2', host='host1'), + dict(id=3, cell_name='cell1', host='host2')] + + self.mox.StubOutWithMock(self.host_api.db, + 'instance_get_all_by_host') + + self.host_api.db.instance_get_all_by_host(self.ctxt, + 'fake-host').AndReturn(instances) + self.mox.ReplayAll() + expected_result = [instances[0], instances[2]] + cell_and_host = cells_utils.cell_with_item('cell1', 'fake-host') + result = self.host_api.instance_get_all_by_host(self.ctxt, + cell_and_host) + self.assertEqual(expected_result, result) -- cgit