From f9a868e86ce11f786538547c301b805bd68a1697 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Fri, 13 Apr 2012 05:54:48 +0000 Subject: Cells: Add the main code. This introduces *EXPERIMENTAL* compute cells functionality as a way to scale nova in a more distributed fashion without having to use complicated technologies like DB and message queue clustering. Cells are configured as a tree and the top level cell should contain nova-api without any nova-computes while child cells contain everything except nova-api. One can think of a cell as a normal nova deployment in that each cell has its own DB server and message queue broker. The top level cell keeps a subset of data about ALL instances in all cells in its DB. Child cells send messages to the top level cell when instances change state. Data in 1 child cell is not shared with another child cell. A new service, nova-cells, is introduced that handles communication between cells and picking of a cell for new instances. This service is required for every cell. Communication between cells is pluggable with the only option currently implemented being communnication via RPC. Cells scheduling is separate from host scheduling. nova-cells first picks a cell (currently randomly -- future patches add filtering/weighing functionality and decisions can be based on broadcasts of capacity/capabilities). Once a cell has been selected and the new build request has reached its nova-cells service, it'll be sent over to the host scheduler in that cell and the build proceeds as it does without cells. New config options are introduced for enabling and configuring the cells code. Cells is disabled by default. All of the config options below go under a '[cells]' section in nova.conf. These are the options that one may want to tweak: enable -- Turn on cells code (default is False) name -- Name of the current cell. capabilities -- List of arbitrary key=value pairs defining capabilities of the current cell. These are sent to parent cells, but aren't used in scheduling until later filter/weight support is added. call_timeout -- How long to wait for replies from a calls between cells When using cells, the compute API class must be changed in the API cell, so that requests can be proxied via nova-cells down to the correct cell properly. Thus, config requirements for API cell: -- [DEFAULT] compute_api_class=nova.compute.cells_api.ComputeCellsAPI. [cells] enable=True name=api-cell -- Config requirements for child cell: -- [cells] enable=True name=child-cell1 -- Another requirement is populating the 'cells' DB table in each cell. Each cell needs to know about its parent and children and how to communicate with them (message broker location, credentials, etc). Implements blueprint nova-compute-cells DocImpact Change-Id: I1b52788ea9d7753365d175abf39bdbc22ba822fe --- nova/tests/cells/__init__.py | 19 + nova/tests/cells/fakes.py | 191 +++++++ nova/tests/cells/test_cells_manager.py | 151 +++++ nova/tests/cells/test_cells_messaging.py | 913 ++++++++++++++++++++++++++++++ nova/tests/cells/test_cells_rpc_driver.py | 218 +++++++ nova/tests/cells/test_cells_rpcapi.py | 206 +++++++ nova/tests/cells/test_cells_scheduler.py | 206 +++++++ nova/tests/compute/test_compute.py | 4 + nova/tests/compute/test_compute_cells.py | 99 ++++ 9 files changed, 2007 insertions(+) create mode 100644 nova/tests/cells/__init__.py create mode 100644 nova/tests/cells/fakes.py create mode 100644 nova/tests/cells/test_cells_manager.py create mode 100644 nova/tests/cells/test_cells_messaging.py create mode 100644 nova/tests/cells/test_cells_rpc_driver.py create mode 100644 nova/tests/cells/test_cells_rpcapi.py create mode 100644 nova/tests/cells/test_cells_scheduler.py create mode 100644 nova/tests/compute/test_compute_cells.py (limited to 'nova/tests') diff --git a/nova/tests/cells/__init__.py b/nova/tests/cells/__init__.py new file mode 100644 index 000000000..d1bf725f7 --- /dev/null +++ b/nova/tests/cells/__init__.py @@ -0,0 +1,19 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(vish): this forces the fixtures from tests/__init.py:setup() to work +from nova.tests import * diff --git a/nova/tests/cells/fakes.py b/nova/tests/cells/fakes.py new file mode 100644 index 000000000..a9de530d1 --- /dev/null +++ b/nova/tests/cells/fakes.py @@ -0,0 +1,191 @@ +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Fakes For Cells tests. +""" + +from nova.cells import driver +from nova.cells import manager as cells_manager +from nova.cells import messaging +from nova.cells import state as cells_state +import nova.db +from nova.db import base +from nova.openstack.common import cfg + +CONF = cfg.CONF +CONF.import_opt('name', 'nova.cells.opts', group='cells') + + +# Fake Cell Hierarchy +FAKE_TOP_LEVEL_CELL_NAME = 'api-cell' +FAKE_CELL_LAYOUT = [{'child-cell1': []}, + {'child-cell2': [{'grandchild-cell1': []}]}, + {'child-cell3': [{'grandchild-cell2': []}, + {'grandchild-cell3': []}]}, + {'child-cell4': []}] + +# build_cell_stub_infos() below will take the above layout and create +# a fake view of the DB from the perspective of each of the cells. +# For each cell, a CellStubInfo will be created with this info. +CELL_NAME_TO_STUB_INFO = {} + + +class FakeDBApi(object): + def __init__(self, cell_db_entries): + self.cell_db_entries = cell_db_entries + + def __getattr__(self, key): + return getattr(nova.db, key) + + def cell_get_all(self, ctxt): + return self.cell_db_entries + + def compute_node_get_all(self, ctxt): + return [] + + +class FakeCellsDriver(driver.BaseCellsDriver): + pass + + +class FakeCellState(cells_state.CellState): + def send_message(self, message): + message_runner = get_message_runner(self.name) + orig_ctxt = message.ctxt + json_message = message.to_json() + message = message_runner.message_from_json(json_message) + # Restore this so we can use mox and verify same context + message.ctxt = orig_ctxt + message.process() + + +class FakeCellStateManager(cells_state.CellStateManager): + def __init__(self, *args, **kwargs): + super(FakeCellStateManager, self).__init__(*args, + cell_state_cls=FakeCellState, **kwargs) + + +class FakeCellsManager(cells_manager.CellsManager): + def __init__(self, *args, **kwargs): + super(FakeCellsManager, self).__init__(*args, + cell_state_manager=FakeCellStateManager, + **kwargs) + + +class CellStubInfo(object): + def __init__(self, test_case, cell_name, db_entries): + self.test_case = test_case + self.cell_name = cell_name + self.db_entries = db_entries + + def fake_base_init(_self, *args, **kwargs): + _self.db = FakeDBApi(db_entries) + + test_case.stubs.Set(base.Base, '__init__', fake_base_init) + self.cells_manager = FakeCellsManager() + # Fix the cell name, as it normally uses CONF.cells.name + msg_runner = self.cells_manager.msg_runner + msg_runner.our_name = self.cell_name + self.cells_manager.state_manager.my_cell_state.name = self.cell_name + + +def _build_cell_stub_info(test_case, our_name, parent_path, children): + cell_db_entries = [] + cur_db_id = 1 + sep_char = messaging._PATH_CELL_SEP + if parent_path: + cell_db_entries.append( + dict(id=cur_db_id, + name=parent_path.split(sep_char)[-1], + is_parent=True, + username='username%s' % cur_db_id, + password='password%s' % cur_db_id, + rpc_host='rpc_host%s' % cur_db_id, + rpc_port='rpc_port%s' % cur_db_id, + rpc_virtual_host='rpc_vhost%s' % cur_db_id)) + cur_db_id += 1 + our_path = parent_path + sep_char + our_name + else: + our_path = our_name + for child in children: + for child_name, grandchildren in child.items(): + _build_cell_stub_info(test_case, child_name, our_path, + grandchildren) + cell_entry = dict(id=cur_db_id, + name=child_name, + username='username%s' % cur_db_id, + password='password%s' % cur_db_id, + rpc_host='rpc_host%s' % cur_db_id, + rpc_port='rpc_port%s' % cur_db_id, + rpc_virtual_host='rpc_vhost%s' % cur_db_id, + is_parent=False) + cell_db_entries.append(cell_entry) + cur_db_id += 1 + stub_info = CellStubInfo(test_case, our_name, cell_db_entries) + CELL_NAME_TO_STUB_INFO[our_name] = stub_info + + +def _build_cell_stub_infos(test_case): + _build_cell_stub_info(test_case, FAKE_TOP_LEVEL_CELL_NAME, '', + FAKE_CELL_LAYOUT) + + +def init(test_case): + global CELL_NAME_TO_STUB_INFO + test_case.flags(driver='nova.tests.cells.fakes.FakeCellsDriver', + group='cells') + CELL_NAME_TO_STUB_INFO = {} + _build_cell_stub_infos(test_case) + + +def _get_cell_stub_info(cell_name): + return CELL_NAME_TO_STUB_INFO[cell_name] + + +def get_state_manager(cell_name): + return _get_cell_stub_info(cell_name).cells_manager.state_manager + + +def get_cell_state(cur_cell_name, tgt_cell_name): + state_manager = get_state_manager(cur_cell_name) + cell = state_manager.child_cells.get(tgt_cell_name) + if cell is None: + cell = state_manager.parent_cells.get(tgt_cell_name) + return cell + + +def get_cells_manager(cell_name): + return _get_cell_stub_info(cell_name).cells_manager + + +def get_message_runner(cell_name): + return _get_cell_stub_info(cell_name).cells_manager.msg_runner + + +def stub_tgt_method(test_case, cell_name, method_name, method): + msg_runner = get_message_runner(cell_name) + tgt_msg_methods = msg_runner.methods_by_type['targeted'] + setattr(tgt_msg_methods, method_name, method) + + +def stub_bcast_method(test_case, cell_name, method_name, method): + msg_runner = get_message_runner(cell_name) + tgt_msg_methods = msg_runner.methods_by_type['broadcast'] + setattr(tgt_msg_methods, method_name, method) + + +def stub_bcast_methods(test_case, method_name, method): + for cell_name in CELL_NAME_TO_STUB_INFO.keys(): + stub_bcast_method(test_case, cell_name, method_name, method) diff --git a/nova/tests/cells/test_cells_manager.py b/nova/tests/cells/test_cells_manager.py new file mode 100644 index 000000000..5a2b83145 --- /dev/null +++ b/nova/tests/cells/test_cells_manager.py @@ -0,0 +1,151 @@ +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For CellsManager +""" +from nova.cells import messaging +from nova import context +from nova import test +from nova.tests.cells import fakes + + +class CellsManagerClassTestCase(test.TestCase): + """Test case for CellsManager class""" + + def setUp(self): + super(CellsManagerClassTestCase, self).setUp() + fakes.init(self) + # pick a child cell to use for tests. + self.our_cell = 'grandchild-cell1' + self.cells_manager = fakes.get_cells_manager(self.our_cell) + self.msg_runner = self.cells_manager.msg_runner + self.driver = self.cells_manager.driver + self.ctxt = 'fake_context' + + def test_post_start_hook_child_cell(self): + self.mox.StubOutWithMock(self.driver, 'start_consumers') + self.mox.StubOutWithMock(context, 'get_admin_context') + self.mox.StubOutWithMock(self.cells_manager, '_update_our_parents') + + self.driver.start_consumers(self.msg_runner) + context.get_admin_context().AndReturn(self.ctxt) + self.cells_manager._update_our_parents(self.ctxt) + self.mox.ReplayAll() + self.cells_manager.post_start_hook() + + def test_post_start_hook_middle_cell(self): + cells_manager = fakes.get_cells_manager('child-cell2') + msg_runner = cells_manager.msg_runner + driver = cells_manager.driver + + self.mox.StubOutWithMock(driver, 'start_consumers') + self.mox.StubOutWithMock(context, 'get_admin_context') + self.mox.StubOutWithMock(msg_runner, + 'ask_children_for_capabilities') + self.mox.StubOutWithMock(msg_runner, + 'ask_children_for_capacities') + + driver.start_consumers(msg_runner) + context.get_admin_context().AndReturn(self.ctxt) + msg_runner.ask_children_for_capabilities(self.ctxt) + msg_runner.ask_children_for_capacities(self.ctxt) + self.mox.ReplayAll() + cells_manager.post_start_hook() + + def test_update_our_parents(self): + self.mox.StubOutWithMock(self.msg_runner, + 'tell_parents_our_capabilities') + self.mox.StubOutWithMock(self.msg_runner, + 'tell_parents_our_capacities') + + self.msg_runner.tell_parents_our_capabilities(self.ctxt) + self.msg_runner.tell_parents_our_capacities(self.ctxt) + self.mox.ReplayAll() + self.cells_manager._update_our_parents(self.ctxt) + + def test_schedule_run_instance(self): + host_sched_kwargs = 'fake_host_sched_kwargs_silently_passed' + self.mox.StubOutWithMock(self.msg_runner, 'schedule_run_instance') + our_cell = self.msg_runner.state_manager.get_my_state() + self.msg_runner.schedule_run_instance(self.ctxt, our_cell, + host_sched_kwargs) + self.mox.ReplayAll() + self.cells_manager.schedule_run_instance(self.ctxt, + host_sched_kwargs=host_sched_kwargs) + + def test_run_compute_api_method(self): + # Args should just be silently passed through + cell_name = 'fake-cell-name' + method_info = 'fake-method-info' + + fake_response = messaging.Response('fake', 'fake', False) + + self.mox.StubOutWithMock(self.msg_runner, + 'run_compute_api_method') + self.mox.StubOutWithMock(fake_response, + 'value_or_raise') + self.msg_runner.run_compute_api_method(self.ctxt, + cell_name, + method_info, + True).AndReturn(fake_response) + fake_response.value_or_raise().AndReturn('fake-response') + self.mox.ReplayAll() + response = self.cells_manager.run_compute_api_method( + self.ctxt, cell_name=cell_name, method_info=method_info, + call=True) + self.assertEqual('fake-response', response) + + def test_instance_update_at_top(self): + self.mox.StubOutWithMock(self.msg_runner, 'instance_update_at_top') + self.msg_runner.instance_update_at_top(self.ctxt, 'fake-instance') + self.mox.ReplayAll() + self.cells_manager.instance_update_at_top(self.ctxt, + instance='fake-instance') + + def test_instance_destroy_at_top(self): + self.mox.StubOutWithMock(self.msg_runner, 'instance_destroy_at_top') + self.msg_runner.instance_destroy_at_top(self.ctxt, 'fake-instance') + self.mox.ReplayAll() + self.cells_manager.instance_destroy_at_top(self.ctxt, + instance='fake-instance') + + def test_instance_delete_everywhere(self): + self.mox.StubOutWithMock(self.msg_runner, + 'instance_delete_everywhere') + self.msg_runner.instance_delete_everywhere(self.ctxt, + 'fake-instance', + 'fake-type') + self.mox.ReplayAll() + self.cells_manager.instance_delete_everywhere( + self.ctxt, instance='fake-instance', + delete_type='fake-type') + + def test_instance_fault_create_at_top(self): + self.mox.StubOutWithMock(self.msg_runner, + 'instance_fault_create_at_top') + self.msg_runner.instance_fault_create_at_top(self.ctxt, + 'fake-fault') + self.mox.ReplayAll() + self.cells_manager.instance_fault_create_at_top( + self.ctxt, instance_fault='fake-fault') + + def test_bw_usage_update_at_top(self): + self.mox.StubOutWithMock(self.msg_runner, + 'bw_usage_update_at_top') + self.msg_runner.bw_usage_update_at_top(self.ctxt, + 'fake-bw-info') + self.mox.ReplayAll() + self.cells_manager.bw_usage_update_at_top( + self.ctxt, bw_update_info='fake-bw-info') diff --git a/nova/tests/cells/test_cells_messaging.py b/nova/tests/cells/test_cells_messaging.py new file mode 100644 index 000000000..d728c9474 --- /dev/null +++ b/nova/tests/cells/test_cells_messaging.py @@ -0,0 +1,913 @@ +# Copyright (c) 2012 Rackspace Hosting # All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Cells Messaging module +""" + +from nova.cells import messaging +from nova import context +from nova import exception +from nova.openstack.common import cfg +from nova import test +from nova.tests.cells import fakes + + +CONF = cfg.CONF +CONF.import_opt('host', 'nova.config') +CONF.import_opt('name', 'nova.cells.opts', group='cells') +CONF.import_opt('allowed_rpc_exception_modules', + 'nova.openstack.common.rpc') + + +class CellsMessageClassesTestCase(test.TestCase): + """Test case for the main Cells Message classes.""" + def setUp(self): + super(CellsMessageClassesTestCase, self).setUp() + fakes.init(self) + self.ctxt = context.RequestContext('fake', 'fake') + # Need to be able to deserialize test.TestingException. + allowed_modules = CONF.allowed_rpc_exception_modules + allowed_modules.append('nova.test') + self.flags(allowed_rpc_exception_modules=allowed_modules) + self.our_name = 'api-cell' + self.msg_runner = fakes.get_message_runner(self.our_name) + self.state_manager = self.msg_runner.state_manager + + def test_reverse_path(self): + path = 'a!b!c!d' + expected = 'd!c!b!a' + rev_path = messaging._reverse_path(path) + self.assertEqual(rev_path, expected) + + def test_response_cell_name_from_path(self): + # test array with tuples of inputs/expected outputs + test_paths = [('cell1', 'cell1'), + ('cell1!cell2', 'cell2!cell1'), + ('cell1!cell2!cell3', 'cell3!cell2!cell1')] + + for test_input, expected_output in test_paths: + self.assertEqual(expected_output, + messaging._response_cell_name_from_path(test_input)) + + def test_response_cell_name_from_path_neighbor_only(self): + # test array with tuples of inputs/expected outputs + test_paths = [('cell1', 'cell1'), + ('cell1!cell2', 'cell2!cell1'), + ('cell1!cell2!cell3', 'cell3!cell2')] + + for test_input, expected_output in test_paths: + self.assertEqual(expected_output, + messaging._response_cell_name_from_path(test_input, + neighbor_only=True)) + + def test_targeted_message(self): + self.flags(max_hop_count=99, group='cells') + target_cell = 'api-cell!child-cell2!grandchild-cell1' + method = 'fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell) + self.assertEqual(self.ctxt, tgt_message.ctxt) + self.assertEqual(method, tgt_message.method_name) + self.assertEqual(method_kwargs, tgt_message.method_kwargs) + self.assertEqual(direction, tgt_message.direction) + self.assertEqual(target_cell, target_cell) + self.assertFalse(tgt_message.fanout) + self.assertFalse(tgt_message.need_response) + self.assertEqual(self.our_name, tgt_message.routing_path) + self.assertEqual(1, tgt_message.hop_count) + self.assertEqual(99, tgt_message.max_hop_count) + self.assertFalse(tgt_message.is_broadcast) + # Correct next hop? + next_hop = tgt_message._get_next_hop() + child_cell = self.state_manager.get_child_cell('child-cell2') + self.assertEqual(child_cell, next_hop) + + def test_create_targeted_message_with_response(self): + self.flags(max_hop_count=99, group='cells') + our_name = 'child-cell1' + target_cell = 'child-cell1!api-cell' + msg_runner = fakes.get_message_runner(our_name) + method = 'fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'up' + tgt_message = messaging._TargetedMessage(msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell, + need_response=True) + self.assertEqual(self.ctxt, tgt_message.ctxt) + self.assertEqual(method, tgt_message.method_name) + self.assertEqual(method_kwargs, tgt_message.method_kwargs) + self.assertEqual(direction, tgt_message.direction) + self.assertEqual(target_cell, target_cell) + self.assertFalse(tgt_message.fanout) + self.assertTrue(tgt_message.need_response) + self.assertEqual(our_name, tgt_message.routing_path) + self.assertEqual(1, tgt_message.hop_count) + self.assertEqual(99, tgt_message.max_hop_count) + self.assertFalse(tgt_message.is_broadcast) + # Correct next hop? + next_hop = tgt_message._get_next_hop() + parent_cell = msg_runner.state_manager.get_parent_cell('api-cell') + self.assertEqual(parent_cell, next_hop) + + def test_targeted_message_when_target_is_cell_state(self): + method = 'fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + target_cell = self.state_manager.get_child_cell('child-cell2') + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell) + self.assertEqual('api-cell!child-cell2', tgt_message.target_cell) + # Correct next hop? + next_hop = tgt_message._get_next_hop() + self.assertEqual(target_cell, next_hop) + + def test_targeted_message_when_target_cell_state_is_me(self): + method = 'fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + target_cell = self.state_manager.get_my_state() + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell) + self.assertEqual('api-cell', tgt_message.target_cell) + # Correct next hop? + next_hop = tgt_message._get_next_hop() + self.assertEqual(target_cell, next_hop) + + def test_create_broadcast_message(self): + self.flags(max_hop_count=99, group='cells') + self.flags(name='api-cell', max_hop_count=99, group='cells') + method = 'fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction) + self.assertEqual(self.ctxt, bcast_message.ctxt) + self.assertEqual(method, bcast_message.method_name) + self.assertEqual(method_kwargs, bcast_message.method_kwargs) + self.assertEqual(direction, bcast_message.direction) + self.assertFalse(bcast_message.fanout) + self.assertFalse(bcast_message.need_response) + self.assertEqual(self.our_name, bcast_message.routing_path) + self.assertEqual(1, bcast_message.hop_count) + self.assertEqual(99, bcast_message.max_hop_count) + self.assertTrue(bcast_message.is_broadcast) + # Correct next hops? + next_hops = bcast_message._get_next_hops() + child_cells = self.state_manager.get_child_cells() + self.assertEqual(child_cells, next_hops) + + def test_create_broadcast_message_with_response(self): + self.flags(max_hop_count=99, group='cells') + our_name = 'child-cell1' + msg_runner = fakes.get_message_runner(our_name) + method = 'fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'up' + bcast_message = messaging._BroadcastMessage(msg_runner, self.ctxt, + method, method_kwargs, direction, need_response=True) + self.assertEqual(self.ctxt, bcast_message.ctxt) + self.assertEqual(method, bcast_message.method_name) + self.assertEqual(method_kwargs, bcast_message.method_kwargs) + self.assertEqual(direction, bcast_message.direction) + self.assertFalse(bcast_message.fanout) + self.assertTrue(bcast_message.need_response) + self.assertEqual(our_name, bcast_message.routing_path) + self.assertEqual(1, bcast_message.hop_count) + self.assertEqual(99, bcast_message.max_hop_count) + self.assertTrue(bcast_message.is_broadcast) + # Correct next hops? + next_hops = bcast_message._get_next_hops() + parent_cells = msg_runner.state_manager.get_parent_cells() + self.assertEqual(parent_cells, next_hops) + + def test_self_targeted_message(self): + target_cell = 'api-cell' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + call_info = {} + + def our_fake_method(message, **kwargs): + call_info['context'] = message.ctxt + call_info['routing_path'] = message.routing_path + call_info['kwargs'] = kwargs + + fakes.stub_tgt_method(self, 'api-cell', 'our_fake_method', + our_fake_method) + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell) + tgt_message.process() + + self.assertEqual(self.ctxt, call_info['context']) + self.assertEqual(method_kwargs, call_info['kwargs']) + self.assertEqual(target_cell, call_info['routing_path']) + + def test_child_targeted_message(self): + target_cell = 'api-cell!child-cell1' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + call_info = {} + + def our_fake_method(message, **kwargs): + call_info['context'] = message.ctxt + call_info['routing_path'] = message.routing_path + call_info['kwargs'] = kwargs + + fakes.stub_tgt_method(self, 'child-cell1', 'our_fake_method', + our_fake_method) + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell) + tgt_message.process() + + self.assertEqual(self.ctxt, call_info['context']) + self.assertEqual(method_kwargs, call_info['kwargs']) + self.assertEqual(target_cell, call_info['routing_path']) + + def test_grandchild_targeted_message(self): + target_cell = 'api-cell!child-cell2!grandchild-cell1' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + call_info = {} + + def our_fake_method(message, **kwargs): + call_info['context'] = message.ctxt + call_info['routing_path'] = message.routing_path + call_info['kwargs'] = kwargs + + fakes.stub_tgt_method(self, 'grandchild-cell1', 'our_fake_method', + our_fake_method) + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell) + tgt_message.process() + + self.assertEqual(self.ctxt, call_info['context']) + self.assertEqual(method_kwargs, call_info['kwargs']) + self.assertEqual(target_cell, call_info['routing_path']) + + def test_grandchild_targeted_message_with_response(self): + target_cell = 'api-cell!child-cell2!grandchild-cell1' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + call_info = {} + + def our_fake_method(message, **kwargs): + call_info['context'] = message.ctxt + call_info['routing_path'] = message.routing_path + call_info['kwargs'] = kwargs + return 'our_fake_response' + + fakes.stub_tgt_method(self, 'grandchild-cell1', 'our_fake_method', + our_fake_method) + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell, + need_response=True) + response = tgt_message.process() + + self.assertEqual(self.ctxt, call_info['context']) + self.assertEqual(method_kwargs, call_info['kwargs']) + self.assertEqual(target_cell, call_info['routing_path']) + self.assertFalse(response.failure) + self.assertTrue(response.value_or_raise(), 'our_fake_response') + + def test_grandchild_targeted_message_with_error(self): + target_cell = 'api-cell!child-cell2!grandchild-cell1' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + def our_fake_method(message, **kwargs): + raise test.TestingException('this should be returned') + + fakes.stub_tgt_method(self, 'grandchild-cell1', 'our_fake_method', + our_fake_method) + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell, + need_response=True) + response = tgt_message.process() + self.assertTrue(response.failure) + self.assertRaises(test.TestingException, response.value_or_raise) + + def test_grandchild_targeted_message_max_hops(self): + self.flags(max_hop_count=2, group='cells') + target_cell = 'api-cell!child-cell2!grandchild-cell1' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + def our_fake_method(message, **kwargs): + raise test.TestingException('should not be reached') + + fakes.stub_tgt_method(self, 'grandchild-cell1', 'our_fake_method', + our_fake_method) + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell, + need_response=True) + response = tgt_message.process() + self.assertTrue(response.failure) + self.assertRaises(exception.CellMaxHopCountReached, + response.value_or_raise) + + def test_targeted_message_invalid_cell(self): + target_cell = 'api-cell!child-cell2!grandchild-cell4' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell, + need_response=True) + response = tgt_message.process() + self.assertTrue(response.failure) + self.assertRaises(exception.CellRoutingInconsistency, + response.value_or_raise) + + def test_targeted_message_invalid_cell2(self): + target_cell = 'unknown-cell!child-cell2' + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + tgt_message = messaging._TargetedMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, direction, + target_cell, + need_response=True) + response = tgt_message.process() + self.assertTrue(response.failure) + self.assertRaises(exception.CellRoutingInconsistency, + response.value_or_raise) + + def test_broadcast_routing(self): + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + cells = set() + + def our_fake_method(message, **kwargs): + cells.add(message.routing_path) + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, + direction, + run_locally=True) + bcast_message.process() + # fakes creates 8 cells (including ourself). + self.assertEqual(len(cells), 8) + + def test_broadcast_routing_up(self): + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'up' + msg_runner = fakes.get_message_runner('grandchild-cell3') + + cells = set() + + def our_fake_method(message, **kwargs): + cells.add(message.routing_path) + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + + bcast_message = messaging._BroadcastMessage(msg_runner, self.ctxt, + method, method_kwargs, + direction, + run_locally=True) + bcast_message.process() + # Paths are reversed, since going 'up' + expected = set(['grandchild-cell3', 'grandchild-cell3!child-cell3', + 'grandchild-cell3!child-cell3!api-cell']) + self.assertEqual(expected, cells) + + def test_broadcast_routing_without_ourselves(self): + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + cells = set() + + def our_fake_method(message, **kwargs): + cells.add(message.routing_path) + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, + direction, + run_locally=False) + bcast_message.process() + # fakes creates 8 cells (including ourself). So we should see + # only 7 here. + self.assertEqual(len(cells), 7) + + def test_broadcast_routing_with_response(self): + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + def our_fake_method(message, **kwargs): + return 'response-%s' % message.routing_path + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, + direction, + run_locally=True, + need_response=True) + responses = bcast_message.process() + self.assertEqual(len(responses), 8) + for response in responses: + self.assertFalse(response.failure) + self.assertEqual('response-%s' % response.cell_name, + response.value_or_raise()) + + def test_broadcast_routing_with_response_max_hops(self): + self.flags(max_hop_count=2, group='cells') + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + def our_fake_method(message, **kwargs): + return 'response-%s' % message.routing_path + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, + direction, + run_locally=True, + need_response=True) + responses = bcast_message.process() + # Should only get responses from our immediate children (and + # ourselves) + self.assertEqual(len(responses), 5) + for response in responses: + self.assertFalse(response.failure) + self.assertEqual('response-%s' % response.cell_name, + response.value_or_raise()) + + def test_broadcast_routing_with_all_erroring(self): + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + def our_fake_method(message, **kwargs): + raise test.TestingException('fake failure') + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, + direction, + run_locally=True, + need_response=True) + responses = bcast_message.process() + self.assertEqual(len(responses), 8) + for response in responses: + self.assertTrue(response.failure) + self.assertRaises(test.TestingException, response.value_or_raise) + + def test_broadcast_routing_with_two_erroring(self): + method = 'our_fake_method' + method_kwargs = dict(arg1=1, arg2=2) + direction = 'down' + + def our_fake_method_failing(message, **kwargs): + raise test.TestingException('fake failure') + + def our_fake_method(message, **kwargs): + return 'response-%s' % message.routing_path + + fakes.stub_bcast_methods(self, 'our_fake_method', our_fake_method) + fakes.stub_bcast_method(self, 'child-cell2', 'our_fake_method', + our_fake_method_failing) + fakes.stub_bcast_method(self, 'grandchild-cell3', 'our_fake_method', + our_fake_method_failing) + + bcast_message = messaging._BroadcastMessage(self.msg_runner, + self.ctxt, method, + method_kwargs, + direction, + run_locally=True, + need_response=True) + responses = bcast_message.process() + self.assertEqual(len(responses), 8) + failure_responses = [resp for resp in responses if resp.failure] + success_responses = [resp for resp in responses if not resp.failure] + self.assertEqual(len(failure_responses), 2) + self.assertEqual(len(success_responses), 6) + + for response in success_responses: + self.assertFalse(response.failure) + self.assertEqual('response-%s' % response.cell_name, + response.value_or_raise()) + + for response in failure_responses: + self.assertIn(response.cell_name, ['api-cell!child-cell2', + 'api-cell!child-cell3!grandchild-cell3']) + self.assertTrue(response.failure) + self.assertRaises(test.TestingException, response.value_or_raise) + + +class CellsTargetedMethodsTestCase(test.TestCase): + """Test case for _TargetedMessageMethods class. Most of these + tests actually test the full path from the MessageRunner through + to the functionality of the message method. Hits 2 birds with 1 + stone, even though it's a little more than a unit test. + """ + def setUp(self): + super(CellsTargetedMethodsTestCase, self).setUp() + fakes.init(self) + self.ctxt = context.RequestContext('fake', 'fake') + self._setup_attrs('api-cell', 'api-cell!child-cell2') + + def _setup_attrs(self, source_cell, target_cell): + self.tgt_cell_name = target_cell + self.src_msg_runner = fakes.get_message_runner(source_cell) + self.src_state_manager = self.src_msg_runner.state_manager + tgt_shortname = target_cell.split('!')[-1] + self.tgt_cell_mgr = fakes.get_cells_manager(tgt_shortname) + self.tgt_msg_runner = self.tgt_cell_mgr.msg_runner + self.tgt_scheduler = self.tgt_msg_runner.scheduler + self.tgt_state_manager = self.tgt_msg_runner.state_manager + methods_cls = self.tgt_msg_runner.methods_by_type['targeted'] + self.tgt_methods_cls = methods_cls + self.tgt_compute_api = methods_cls.compute_api + self.tgt_db_inst = methods_cls.db + + def test_schedule_run_instance(self): + host_sched_kwargs = {'filter_properties': {}, + 'key1': 'value1', + 'key2': 'value2'} + self.mox.StubOutWithMock(self.tgt_scheduler, 'run_instance') + self.tgt_scheduler.run_instance(self.ctxt, host_sched_kwargs) + self.mox.ReplayAll() + self.src_msg_runner.schedule_run_instance(self.ctxt, + self.tgt_cell_name, + host_sched_kwargs) + + def test_call_compute_api_method(self): + + instance_uuid = 'fake_instance_uuid' + method_info = {'method': 'reboot', + 'method_args': (instance_uuid, 2, 3), + 'method_kwargs': {'arg1': 'val1', 'arg2': 'val2'}} + self.mox.StubOutWithMock(self.tgt_compute_api, 'reboot') + self.mox.StubOutWithMock(self.tgt_db_inst, 'instance_get_by_uuid') + + self.tgt_db_inst.instance_get_by_uuid(self.ctxt, + instance_uuid).AndReturn( + 'fake_instance') + self.tgt_compute_api.reboot(self.ctxt, 'fake_instance', 2, 3, + arg1='val1', arg2='val2').AndReturn('fake_result') + self.mox.ReplayAll() + + response = self.src_msg_runner.run_compute_api_method( + self.ctxt, + self.tgt_cell_name, + method_info, + True) + result = response.value_or_raise() + self.assertEqual('fake_result', result) + + def test_call_compute_api_method_unknown_instance(self): + # Unknown instance should send a broadcast up that instance + # is gone. + instance_uuid = 'fake_instance_uuid' + instance = {'uuid': instance_uuid} + method_info = {'method': 'reboot', + 'method_args': (instance_uuid, 2, 3), + 'method_kwargs': {'arg1': 'val1', 'arg2': 'val2'}} + + self.mox.StubOutWithMock(self.tgt_db_inst, 'instance_get_by_uuid') + self.mox.StubOutWithMock(self.tgt_msg_runner, + 'instance_destroy_at_top') + + self.tgt_db_inst.instance_get_by_uuid(self.ctxt, + 'fake_instance_uuid').AndRaise( + exception.InstanceNotFound(instance_id=instance_uuid)) + self.tgt_msg_runner.instance_destroy_at_top(self.ctxt, instance) + + self.mox.ReplayAll() + + response = self.src_msg_runner.run_compute_api_method( + self.ctxt, + self.tgt_cell_name, + method_info, + True) + self.assertRaises(exception.InstanceNotFound, + response.value_or_raise) + + def test_update_capabilities(self): + # Route up to API + self._setup_attrs('child-cell2', 'child-cell2!api-cell') + capabs = {'cap1': set(['val1', 'val2']), + 'cap2': set(['val3'])} + # The list(set([])) seems silly, but we can't assume the order + # of the list... This behavior should match the code we're + # testing... which is check that a set was converted to a list. + expected_capabs = {'cap1': list(set(['val1', 'val2'])), + 'cap2': ['val3']} + self.mox.StubOutWithMock(self.src_state_manager, + 'get_our_capabilities') + self.mox.StubOutWithMock(self.tgt_state_manager, + 'update_cell_capabilities') + self.mox.StubOutWithMock(self.tgt_msg_runner, + 'tell_parents_our_capabilities') + self.src_state_manager.get_our_capabilities().AndReturn(capabs) + self.tgt_state_manager.update_cell_capabilities('child-cell2', + expected_capabs) + self.tgt_msg_runner.tell_parents_our_capabilities(self.ctxt) + + self.mox.ReplayAll() + + self.src_msg_runner.tell_parents_our_capabilities(self.ctxt) + + def test_update_capacities(self): + self._setup_attrs('child-cell2', 'child-cell2!api-cell') + capacs = 'fake_capacs' + self.mox.StubOutWithMock(self.src_state_manager, + 'get_our_capacities') + self.mox.StubOutWithMock(self.tgt_state_manager, + 'update_cell_capacities') + self.mox.StubOutWithMock(self.tgt_msg_runner, + 'tell_parents_our_capacities') + self.src_state_manager.get_our_capacities().AndReturn(capacs) + self.tgt_state_manager.update_cell_capacities('child-cell2', + capacs) + self.tgt_msg_runner.tell_parents_our_capacities(self.ctxt) + + self.mox.ReplayAll() + + self.src_msg_runner.tell_parents_our_capacities(self.ctxt) + + def test_announce_capabilities(self): + self._setup_attrs('api-cell', 'api-cell!child-cell1') + # To make this easier to test, make us only have 1 child cell. + cell_state = self.src_state_manager.child_cells['child-cell1'] + self.src_state_manager.child_cells = {'child-cell1': cell_state} + + self.mox.StubOutWithMock(self.tgt_msg_runner, + 'tell_parents_our_capabilities') + self.tgt_msg_runner.tell_parents_our_capabilities(self.ctxt) + + self.mox.ReplayAll() + + self.src_msg_runner.ask_children_for_capabilities(self.ctxt) + + def test_announce_capacities(self): + self._setup_attrs('api-cell', 'api-cell!child-cell1') + # To make this easier to test, make us only have 1 child cell. + cell_state = self.src_state_manager.child_cells['child-cell1'] + self.src_state_manager.child_cells = {'child-cell1': cell_state} + + self.mox.StubOutWithMock(self.tgt_msg_runner, + 'tell_parents_our_capacities') + self.tgt_msg_runner.tell_parents_our_capacities(self.ctxt) + + self.mox.ReplayAll() + + self.src_msg_runner.ask_children_for_capacities(self.ctxt) + + +class CellsBroadcastMethodsTestCase(test.TestCase): + """Test case for _BroadcastMessageMethods class. Most of these + tests actually test the full path from the MessageRunner through + to the functionality of the message method. Hits 2 birds with 1 + stone, even though it's a little more than a unit test. + """ + + def setUp(self): + super(CellsBroadcastMethodsTestCase, self).setUp() + fakes.init(self) + self.ctxt = context.RequestContext('fake', 'fake') + self._setup_attrs() + + def _setup_attrs(self, up=True): + mid_cell = 'child-cell2' + if up: + src_cell = 'grandchild-cell1' + tgt_cell = 'api-cell' + else: + src_cell = 'api-cell' + tgt_cell = 'grandchild-cell1' + + self.src_msg_runner = fakes.get_message_runner(src_cell) + methods_cls = self.src_msg_runner.methods_by_type['broadcast'] + self.src_methods_cls = methods_cls + self.src_db_inst = methods_cls.db + self.src_compute_api = methods_cls.compute_api + + self.mid_msg_runner = fakes.get_message_runner(mid_cell) + methods_cls = self.mid_msg_runner.methods_by_type['broadcast'] + self.mid_methods_cls = methods_cls + self.mid_db_inst = methods_cls.db + self.mid_compute_api = methods_cls.compute_api + + self.tgt_msg_runner = fakes.get_message_runner(tgt_cell) + methods_cls = self.tgt_msg_runner.methods_by_type['broadcast'] + self.tgt_methods_cls = methods_cls + self.tgt_db_inst = methods_cls.db + self.tgt_compute_api = methods_cls.compute_api + + def test_at_the_top(self): + self.assertTrue(self.tgt_methods_cls._at_the_top()) + self.assertFalse(self.mid_methods_cls._at_the_top()) + self.assertFalse(self.src_methods_cls._at_the_top()) + + def test_instance_update_at_top(self): + fake_info_cache = {'id': 1, + 'instance': 'fake_instance', + 'other': 'moo'} + fake_sys_metadata = [{'id': 1, + 'key': 'key1', + 'value': 'value1'}, + {'id': 2, + 'key': 'key2', + 'value': 'value2'}] + fake_instance = {'id': 2, + 'uuid': 'fake_uuid', + 'security_groups': 'fake', + 'instance_type': 'fake', + 'volumes': 'fake', + 'cell_name': 'fake', + 'name': 'fake', + 'metadata': 'fake', + 'info_cache': fake_info_cache, + 'system_metadata': fake_sys_metadata, + 'other': 'meow'} + expected_sys_metadata = {'key1': 'value1', + 'key2': 'value2'} + expected_info_cache = {'other': 'moo'} + expected_instance = {'system_metadata': expected_sys_metadata, + 'other': 'meow', + 'uuid': 'fake_uuid'} + + # To show these should not be called in src/mid-level cell + self.mox.StubOutWithMock(self.src_db_inst, 'instance_update') + self.mox.StubOutWithMock(self.src_db_inst, + 'instance_info_cache_update') + self.mox.StubOutWithMock(self.mid_db_inst, 'instance_update') + self.mox.StubOutWithMock(self.mid_db_inst, + 'instance_info_cache_update') + + self.mox.StubOutWithMock(self.tgt_db_inst, 'instance_update') + self.mox.StubOutWithMock(self.tgt_db_inst, + 'instance_info_cache_update') + self.tgt_db_inst.instance_update(self.ctxt, 'fake_uuid', + expected_instance, + update_cells=False) + self.tgt_db_inst.instance_info_cache_update(self.ctxt, 'fake_uuid', + expected_info_cache, + update_cells=False) + self.mox.ReplayAll() + + self.src_msg_runner.instance_update_at_top(self.ctxt, fake_instance) + + def test_instance_destroy_at_top(self): + fake_instance = {'uuid': 'fake_uuid'} + + # To show these should not be called in src/mid-level cell + self.mox.StubOutWithMock(self.src_db_inst, 'instance_destroy') + + self.mox.StubOutWithMock(self.tgt_db_inst, 'instance_destroy') + self.tgt_db_inst.instance_destroy(self.ctxt, 'fake_uuid', + update_cells=False) + self.mox.ReplayAll() + + self.src_msg_runner.instance_destroy_at_top(self.ctxt, fake_instance) + + def test_instance_hard_delete_everywhere(self): + # Reset this, as this is a broadcast down. + self._setup_attrs(up=False) + instance = {'uuid': 'meow'} + + # Should not be called in src (API cell) + self.mox.StubOutWithMock(self.src_compute_api, 'delete') + + self.mox.StubOutWithMock(self.mid_compute_api, 'delete') + self.mox.StubOutWithMock(self.tgt_compute_api, 'delete') + + self.mid_compute_api.delete(self.ctxt, instance) + self.tgt_compute_api.delete(self.ctxt, instance) + + self.mox.ReplayAll() + + self.src_msg_runner.instance_delete_everywhere(self.ctxt, + instance, 'hard') + + def test_instance_soft_delete_everywhere(self): + # Reset this, as this is a broadcast down. + self._setup_attrs(up=False) + instance = {'uuid': 'meow'} + + # Should not be called in src (API cell) + self.mox.StubOutWithMock(self.src_compute_api, 'soft_delete') + + self.mox.StubOutWithMock(self.mid_compute_api, 'soft_delete') + self.mox.StubOutWithMock(self.tgt_compute_api, 'soft_delete') + + self.mid_compute_api.soft_delete(self.ctxt, instance) + self.tgt_compute_api.soft_delete(self.ctxt, instance) + + self.mox.ReplayAll() + + self.src_msg_runner.instance_delete_everywhere(self.ctxt, + instance, 'soft') + + def test_instance_fault_create_at_top(self): + fake_instance_fault = {'id': 1, + 'other stuff': 2, + 'more stuff': 3} + expected_instance_fault = {'other stuff': 2, + 'more stuff': 3} + + # Shouldn't be called for these 2 cells + self.mox.StubOutWithMock(self.src_db_inst, 'instance_fault_create') + self.mox.StubOutWithMock(self.mid_db_inst, 'instance_fault_create') + + self.mox.StubOutWithMock(self.tgt_db_inst, 'instance_fault_create') + self.tgt_db_inst.instance_fault_create(self.ctxt, + expected_instance_fault) + self.mox.ReplayAll() + + self.src_msg_runner.instance_fault_create_at_top(self.ctxt, + fake_instance_fault) + + def test_bw_usage_update_at_top(self): + fake_bw_update_info = {'uuid': 'fake_uuid', + 'mac': 'fake_mac', + 'start_period': 'fake_start_period', + 'bw_in': 'fake_bw_in', + 'bw_out': 'fake_bw_out', + 'last_ctr_in': 'fake_last_ctr_in', + 'last_ctr_out': 'fake_last_ctr_out', + 'last_refreshed': 'fake_last_refreshed'} + + # Shouldn't be called for these 2 cells + self.mox.StubOutWithMock(self.src_db_inst, 'bw_usage_update') + self.mox.StubOutWithMock(self.mid_db_inst, 'bw_usage_update') + + self.mox.StubOutWithMock(self.tgt_db_inst, 'bw_usage_update') + self.tgt_db_inst.bw_usage_update(self.ctxt, **fake_bw_update_info) + + self.mox.ReplayAll() + + self.src_msg_runner.bw_usage_update_at_top(self.ctxt, + fake_bw_update_info) diff --git a/nova/tests/cells/test_cells_rpc_driver.py b/nova/tests/cells/test_cells_rpc_driver.py new file mode 100644 index 000000000..a44fe9376 --- /dev/null +++ b/nova/tests/cells/test_cells_rpc_driver.py @@ -0,0 +1,218 @@ +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Cells RPC Communication Driver +""" + +from nova.cells import messaging +from nova.cells import rpc_driver +from nova import context +from nova.openstack.common import cfg +from nova.openstack.common import rpc +from nova.openstack.common.rpc import dispatcher as rpc_dispatcher +from nova import test +from nova.tests.cells import fakes + +CONF = cfg.CONF +CONF.import_opt('rpc_driver_queue_base', 'nova.cells.rpc_driver', + group='cells') + + +class CellsRPCDriverTestCase(test.TestCase): + """Test case for Cells communication via RPC.""" + + def setUp(self): + super(CellsRPCDriverTestCase, self).setUp() + fakes.init(self) + self.ctxt = context.RequestContext('fake', 'fake') + self.driver = rpc_driver.CellsRPCDriver() + + def test_start_consumers(self): + self.flags(rpc_driver_queue_base='cells.intercell42', group='cells') + rpc_consumers = [] + rpc_conns = [] + fake_msg_runner = fakes.get_message_runner('api-cell') + call_info = {} + + class FakeInterCellRPCDispatcher(object): + def __init__(_self, msg_runner): + self.assertEqual(fake_msg_runner, msg_runner) + call_info['intercell_dispatcher'] = _self + + class FakeRPCDispatcher(object): + def __init__(_self, proxy_objs): + self.assertEqual([call_info['intercell_dispatcher']], + proxy_objs) + call_info['rpc_dispatcher'] = _self + + class FakeRPCConn(object): + def create_consumer(_self, topic, proxy_obj, **kwargs): + self.assertEqual(call_info['rpc_dispatcher'], proxy_obj) + rpc_consumers.append((topic, kwargs)) + + def consume_in_thread(_self): + pass + + def _fake_create_connection(new): + self.assertTrue(new) + fake_conn = FakeRPCConn() + rpc_conns.append(fake_conn) + return fake_conn + + self.stubs.Set(rpc, 'create_connection', _fake_create_connection) + self.stubs.Set(rpc_driver, 'InterCellRPCDispatcher', + FakeInterCellRPCDispatcher) + self.stubs.Set(rpc_dispatcher, 'RpcDispatcher', FakeRPCDispatcher) + + self.driver.start_consumers(fake_msg_runner) + + for message_type in ['broadcast', 'response', 'targeted']: + topic = 'cells.intercell42.' + message_type + self.assertIn((topic, {'fanout': True}), rpc_consumers) + self.assertIn((topic, {'fanout': False}), rpc_consumers) + self.assertEqual(rpc_conns, self.driver.rpc_connections) + + def test_stop_consumers(self): + call_info = {'closed': []} + + class FakeRPCConn(object): + def close(self): + call_info['closed'].append(self) + + fake_conns = [FakeRPCConn() for x in xrange(5)] + self.driver.rpc_connections = fake_conns + self.driver.stop_consumers() + self.assertEqual(fake_conns, call_info['closed']) + + def test_send_message_to_cell_cast(self): + msg_runner = fakes.get_message_runner('api-cell') + cell_state = fakes.get_cell_state('api-cell', 'child-cell2') + message = messaging._TargetedMessage(msg_runner, + self.ctxt, 'fake', 'fake', 'down', cell_state, fanout=False) + + call_info = {} + + def _fake_make_msg(method, **kwargs): + call_info['rpc_method'] = method + call_info['rpc_kwargs'] = kwargs + return 'fake-message' + + def _fake_cast_to_server(*args, **kwargs): + call_info['cast_args'] = args + call_info['cast_kwargs'] = kwargs + + self.stubs.Set(rpc, 'cast_to_server', _fake_cast_to_server) + self.stubs.Set(self.driver.intercell_rpcapi, 'make_msg', + _fake_make_msg) + self.stubs.Set(self.driver.intercell_rpcapi, 'cast_to_server', + _fake_cast_to_server) + + self.driver.send_message_to_cell(cell_state, message) + expected_server_params = {'hostname': 'rpc_host2', + 'password': 'password2', + 'port': 'rpc_port2', + 'username': 'username2', + 'virtual_host': 'rpc_vhost2'} + expected_cast_args = (self.ctxt, expected_server_params, + 'fake-message') + expected_cast_kwargs = {'topic': 'cells.intercell.targeted'} + expected_rpc_kwargs = {'message': message.to_json()} + self.assertEqual(expected_cast_args, call_info['cast_args']) + self.assertEqual(expected_cast_kwargs, call_info['cast_kwargs']) + self.assertEqual('process_message', call_info['rpc_method']) + self.assertEqual(expected_rpc_kwargs, call_info['rpc_kwargs']) + + def test_send_message_to_cell_fanout_cast(self): + msg_runner = fakes.get_message_runner('api-cell') + cell_state = fakes.get_cell_state('api-cell', 'child-cell2') + message = messaging._TargetedMessage(msg_runner, + self.ctxt, 'fake', 'fake', 'down', cell_state, fanout=True) + + call_info = {} + + def _fake_make_msg(method, **kwargs): + call_info['rpc_method'] = method + call_info['rpc_kwargs'] = kwargs + return 'fake-message' + + def _fake_fanout_cast_to_server(*args, **kwargs): + call_info['cast_args'] = args + call_info['cast_kwargs'] = kwargs + + self.stubs.Set(rpc, 'fanout_cast_to_server', + _fake_fanout_cast_to_server) + self.stubs.Set(self.driver.intercell_rpcapi, 'make_msg', + _fake_make_msg) + self.stubs.Set(self.driver.intercell_rpcapi, + 'fanout_cast_to_server', _fake_fanout_cast_to_server) + + self.driver.send_message_to_cell(cell_state, message) + expected_server_params = {'hostname': 'rpc_host2', + 'password': 'password2', + 'port': 'rpc_port2', + 'username': 'username2', + 'virtual_host': 'rpc_vhost2'} + expected_cast_args = (self.ctxt, expected_server_params, + 'fake-message') + expected_cast_kwargs = {'topic': 'cells.intercell.targeted'} + expected_rpc_kwargs = {'message': message.to_json()} + self.assertEqual(expected_cast_args, call_info['cast_args']) + self.assertEqual(expected_cast_kwargs, call_info['cast_kwargs']) + self.assertEqual('process_message', call_info['rpc_method']) + self.assertEqual(expected_rpc_kwargs, call_info['rpc_kwargs']) + + def test_rpc_topic_uses_message_type(self): + self.flags(rpc_driver_queue_base='cells.intercell42', group='cells') + msg_runner = fakes.get_message_runner('api-cell') + cell_state = fakes.get_cell_state('api-cell', 'child-cell2') + message = messaging._BroadcastMessage(msg_runner, + self.ctxt, 'fake', 'fake', 'down', fanout=True) + message.message_type = 'fake-message-type' + + call_info = {} + + def _fake_fanout_cast_to_server(*args, **kwargs): + call_info['topic'] = kwargs.get('topic') + + self.stubs.Set(self.driver.intercell_rpcapi, + 'fanout_cast_to_server', _fake_fanout_cast_to_server) + + self.driver.send_message_to_cell(cell_state, message) + self.assertEqual('cells.intercell42.fake-message-type', + call_info['topic']) + + def test_process_message(self): + msg_runner = fakes.get_message_runner('api-cell') + dispatcher = rpc_driver.InterCellRPCDispatcher(msg_runner) + message = messaging._BroadcastMessage(msg_runner, + self.ctxt, 'fake', 'fake', 'down', fanout=True) + + call_info = {} + + def _fake_message_from_json(json_message): + call_info['json_message'] = json_message + self.assertEqual(message.to_json(), json_message) + return message + + def _fake_process(): + call_info['process_called'] = True + + self.stubs.Set(msg_runner, 'message_from_json', + _fake_message_from_json) + self.stubs.Set(message, 'process', _fake_process) + + dispatcher.process_message(self.ctxt, message.to_json()) + self.assertEqual(message.to_json(), call_info['json_message']) + self.assertTrue(call_info['process_called']) diff --git a/nova/tests/cells/test_cells_rpcapi.py b/nova/tests/cells/test_cells_rpcapi.py new file mode 100644 index 000000000..b51bfa0c1 --- /dev/null +++ b/nova/tests/cells/test_cells_rpcapi.py @@ -0,0 +1,206 @@ +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Cells RPCAPI +""" + +from nova.cells import rpcapi as cells_rpcapi +from nova.openstack.common import cfg +from nova.openstack.common import rpc +from nova import test + +CONF = cfg.CONF +CONF.import_opt('topic', 'nova.cells.opts', group='cells') + + +class CellsAPITestCase(test.TestCase): + """Test case for cells.api interfaces.""" + + def setUp(self): + super(CellsAPITestCase, self).setUp() + self.fake_topic = 'fake_topic' + self.fake_context = 'fake_context' + self.flags(topic=self.fake_topic, enable=True, group='cells') + self.cells_rpcapi = cells_rpcapi.CellsAPI() + + def _stub_rpc_method(self, rpc_method, result): + call_info = {} + + def fake_rpc_method(ctxt, topic, msg, *args, **kwargs): + call_info['context'] = ctxt + call_info['topic'] = topic + call_info['msg'] = msg + return result + + self.stubs.Set(rpc, rpc_method, fake_rpc_method) + return call_info + + def _check_result(self, call_info, method, args, version=None): + if version is None: + version = self.cells_rpcapi.BASE_RPC_API_VERSION + self.assertEqual(self.fake_context, call_info['context']) + self.assertEqual(self.fake_topic, call_info['topic']) + self.assertEqual(method, call_info['msg']['method']) + self.assertEqual(version, call_info['msg']['version']) + self.assertEqual(args, call_info['msg']['args']) + + def test_cast_compute_api_method(self): + fake_cell_name = 'fake_cell_name' + fake_method = 'fake_method' + fake_method_args = (1, 2) + fake_method_kwargs = {'kwarg1': 10, 'kwarg2': 20} + + expected_method_info = {'method': fake_method, + 'method_args': fake_method_args, + 'method_kwargs': fake_method_kwargs} + expected_args = {'method_info': expected_method_info, + 'cell_name': fake_cell_name, + 'call': False} + + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.cast_compute_api_method(self.fake_context, + fake_cell_name, fake_method, + *fake_method_args, **fake_method_kwargs) + self._check_result(call_info, 'run_compute_api_method', + expected_args) + + def test_call_compute_api_method(self): + fake_cell_name = 'fake_cell_name' + fake_method = 'fake_method' + fake_method_args = (1, 2) + fake_method_kwargs = {'kwarg1': 10, 'kwarg2': 20} + fake_response = 'fake_response' + + expected_method_info = {'method': fake_method, + 'method_args': fake_method_args, + 'method_kwargs': fake_method_kwargs} + expected_args = {'method_info': expected_method_info, + 'cell_name': fake_cell_name, + 'call': True} + + call_info = self._stub_rpc_method('call', fake_response) + + result = self.cells_rpcapi.call_compute_api_method(self.fake_context, + fake_cell_name, fake_method, + *fake_method_args, **fake_method_kwargs) + self._check_result(call_info, 'run_compute_api_method', + expected_args) + self.assertEqual(fake_response, result) + + def test_schedule_run_instance(self): + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.schedule_run_instance( + self.fake_context, arg1=1, arg2=2, arg3=3) + + expected_args = {'host_sched_kwargs': {'arg1': 1, + 'arg2': 2, + 'arg3': 3}} + self._check_result(call_info, 'schedule_run_instance', + expected_args) + + def test_instance_update_at_top(self): + fake_info_cache = {'id': 1, + 'instance': 'fake_instance', + 'other': 'moo'} + fake_sys_metadata = [{'id': 1, + 'key': 'key1', + 'value': 'value1'}, + {'id': 2, + 'key': 'key2', + 'value': 'value2'}] + fake_instance = {'id': 2, + 'security_groups': 'fake', + 'instance_type': 'fake', + 'volumes': 'fake', + 'cell_name': 'fake', + 'name': 'fake', + 'metadata': 'fake', + 'info_cache': fake_info_cache, + 'system_metadata': fake_sys_metadata, + 'other': 'meow'} + + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.instance_update_at_top( + self.fake_context, fake_instance) + + expected_args = {'instance': fake_instance} + self._check_result(call_info, 'instance_update_at_top', + expected_args) + + def test_instance_destroy_at_top(self): + fake_instance = {'uuid': 'fake-uuid'} + + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.instance_destroy_at_top( + self.fake_context, fake_instance) + + expected_args = {'instance': fake_instance} + self._check_result(call_info, 'instance_destroy_at_top', + expected_args) + + def test_instance_delete_everywhere(self): + fake_instance = {'uuid': 'fake-uuid'} + + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.instance_delete_everywhere( + self.fake_context, fake_instance, + 'fake-type') + + expected_args = {'instance': fake_instance, + 'delete_type': 'fake-type'} + self._check_result(call_info, 'instance_delete_everywhere', + expected_args) + + def test_instance_fault_create_at_top(self): + fake_instance_fault = {'id': 2, + 'other': 'meow'} + + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.instance_fault_create_at_top( + self.fake_context, fake_instance_fault) + + expected_args = {'instance_fault': fake_instance_fault} + self._check_result(call_info, 'instance_fault_create_at_top', + expected_args) + + def test_bw_usage_update_at_top(self): + update_args = ('fake_uuid', 'fake_mac', 'fake_start_period', + 'fake_bw_in', 'fake_bw_out', 'fake_ctr_in', + 'fake_ctr_out') + update_kwargs = {'last_refreshed': 'fake_refreshed'} + + call_info = self._stub_rpc_method('cast', None) + + self.cells_rpcapi.bw_usage_update_at_top( + self.fake_context, *update_args, **update_kwargs) + + bw_update_info = {'uuid': 'fake_uuid', + 'mac': 'fake_mac', + 'start_period': 'fake_start_period', + 'bw_in': 'fake_bw_in', + 'bw_out': 'fake_bw_out', + 'last_ctr_in': 'fake_ctr_in', + 'last_ctr_out': 'fake_ctr_out', + 'last_refreshed': 'fake_refreshed'} + + expected_args = {'bw_update_info': bw_update_info} + self._check_result(call_info, 'bw_usage_update_at_top', + expected_args) diff --git a/nova/tests/cells/test_cells_scheduler.py b/nova/tests/cells/test_cells_scheduler.py new file mode 100644 index 000000000..66e7e245e --- /dev/null +++ b/nova/tests/cells/test_cells_scheduler.py @@ -0,0 +1,206 @@ +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For CellsScheduler +""" +import time + +from nova.compute import vm_states +from nova import context +from nova import db +from nova import exception +from nova.openstack.common import cfg +from nova.openstack.common import uuidutils +from nova import test +from nova.tests.cells import fakes + +CONF = cfg.CONF +CONF.import_opt('scheduler_retries', 'nova.cells.scheduler', group='cells') + + +class CellsSchedulerTestCase(test.TestCase): + """Test case for CellsScheduler class""" + + def setUp(self): + super(CellsSchedulerTestCase, self).setUp() + fakes.init(self) + self.msg_runner = fakes.get_message_runner('api-cell') + self.scheduler = self.msg_runner.scheduler + self.state_manager = self.msg_runner.state_manager + self.my_cell_state = self.state_manager.get_my_state() + self.ctxt = context.RequestContext('fake', 'fake') + instance_uuids = [] + for x in xrange(3): + instance_uuids.append(uuidutils.generate_uuid()) + self.instance_uuids = instance_uuids + self.request_spec = {'instance_uuids': instance_uuids, + 'other': 'stuff'} + + def test_create_instances_here(self): + # Just grab the first instance type + inst_type = db.instance_type_get(self.ctxt, 1) + image = {'properties': {}} + instance_props = {'hostname': 'meow', + 'display_name': 'moo', + 'image_ref': 'fake_image_ref', + 'user_id': self.ctxt.user_id, + 'project_id': self.ctxt.project_id} + request_spec = {'instance_type': inst_type, + 'image': image, + 'security_group': ['default'], + 'block_device_mapping': [], + 'instance_properties': instance_props, + 'instance_uuids': self.instance_uuids} + + call_info = {'uuids': []} + + def _fake_instance_update_at_top(_ctxt, instance): + call_info['uuids'].append(instance['uuid']) + + self.stubs.Set(self.msg_runner, 'instance_update_at_top', + _fake_instance_update_at_top) + + self.scheduler._create_instances_here(self.ctxt, request_spec) + self.assertEqual(self.instance_uuids, call_info['uuids']) + + for instance_uuid in self.instance_uuids: + instance = db.instance_get_by_uuid(self.ctxt, instance_uuid) + self.assertEqual('meow', instance['hostname']) + self.assertEqual('moo', instance['display_name']) + self.assertEqual('fake_image_ref', instance['image_ref']) + + def test_run_instance_selects_child_cell(self): + # Make sure there's no capacity info so we're sure to + # select a child cell + our_cell_info = self.state_manager.get_my_state() + our_cell_info.capacities = {} + + call_info = {'times': 0} + + orig_fn = self.msg_runner.schedule_run_instance + + def msg_runner_schedule_run_instance(ctxt, target_cell, + host_sched_kwargs): + # This gets called twice. Once for our running it + # in this cell.. and then it'll get called when the + # child cell is picked. So, first time.. just run it + # like normal. + if not call_info['times']: + call_info['times'] += 1 + return orig_fn(ctxt, target_cell, host_sched_kwargs) + call_info['ctxt'] = ctxt + call_info['target_cell'] = target_cell + call_info['host_sched_kwargs'] = host_sched_kwargs + + self.stubs.Set(self.msg_runner, 'schedule_run_instance', + msg_runner_schedule_run_instance) + + host_sched_kwargs = {'request_spec': self.request_spec} + self.msg_runner.schedule_run_instance(self.ctxt, + self.my_cell_state, host_sched_kwargs) + + self.assertEqual(self.ctxt, call_info['ctxt']) + self.assertEqual(host_sched_kwargs, call_info['host_sched_kwargs']) + child_cells = self.state_manager.get_child_cells() + self.assertIn(call_info['target_cell'], child_cells) + + def test_run_instance_selects_current_cell(self): + # Make sure there's no child cells so that we will be + # selected + self.state_manager.child_cells = {} + + call_info = {} + + def fake_create_instances_here(ctxt, request_spec): + call_info['ctxt'] = ctxt + call_info['request_spec'] = request_spec + + def fake_rpc_run_instance(ctxt, **host_sched_kwargs): + call_info['host_sched_kwargs'] = host_sched_kwargs + + self.stubs.Set(self.scheduler, '_create_instances_here', + fake_create_instances_here) + self.stubs.Set(self.scheduler.scheduler_rpcapi, + 'run_instance', fake_rpc_run_instance) + + host_sched_kwargs = {'request_spec': self.request_spec, + 'other': 'stuff'} + self.msg_runner.schedule_run_instance(self.ctxt, + self.my_cell_state, host_sched_kwargs) + + self.assertEqual(self.ctxt, call_info['ctxt']) + self.assertEqual(self.request_spec, call_info['request_spec']) + self.assertEqual(host_sched_kwargs, call_info['host_sched_kwargs']) + + def test_run_instance_retries_when_no_cells_avail(self): + self.flags(scheduler_retries=7, group='cells') + + host_sched_kwargs = {'request_spec': self.request_spec} + + call_info = {'num_tries': 0, 'errored_uuids': []} + + def fake_run_instance(message, host_sched_kwargs): + call_info['num_tries'] += 1 + raise exception.NoCellsAvailable() + + def fake_sleep(_secs): + return + + def fake_instance_update(ctxt, instance_uuid, values): + self.assertEqual(vm_states.ERROR, values['vm_state']) + call_info['errored_uuids'].append(instance_uuid) + + self.stubs.Set(self.scheduler, '_run_instance', fake_run_instance) + self.stubs.Set(time, 'sleep', fake_sleep) + self.stubs.Set(db, 'instance_update', fake_instance_update) + + self.msg_runner.schedule_run_instance(self.ctxt, + self.my_cell_state, host_sched_kwargs) + + self.assertEqual(8, call_info['num_tries']) + self.assertEqual(self.instance_uuids, call_info['errored_uuids']) + + def test_run_instance_on_random_exception(self): + self.flags(scheduler_retries=7, group='cells') + + host_sched_kwargs = {'request_spec': self.request_spec} + + call_info = {'num_tries': 0, + 'errored_uuids1': [], + 'errored_uuids2': []} + + def fake_run_instance(message, host_sched_kwargs): + call_info['num_tries'] += 1 + raise test.TestingException() + + def fake_instance_update(ctxt, instance_uuid, values): + self.assertEqual(vm_states.ERROR, values['vm_state']) + call_info['errored_uuids1'].append(instance_uuid) + + def fake_instance_update_at_top(ctxt, instance): + self.assertEqual(vm_states.ERROR, instance['vm_state']) + call_info['errored_uuids2'].append(instance['uuid']) + + self.stubs.Set(self.scheduler, '_run_instance', fake_run_instance) + self.stubs.Set(db, 'instance_update', fake_instance_update) + self.stubs.Set(self.msg_runner, 'instance_update_at_top', + fake_instance_update_at_top) + + self.msg_runner.schedule_run_instance(self.ctxt, + self.my_cell_state, host_sched_kwargs) + # Shouldn't retry + self.assertEqual(1, call_info['num_tries']) + self.assertEqual(self.instance_uuids, call_info['errored_uuids1']) + self.assertEqual(self.instance_uuids, call_info['errored_uuids2']) diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py index d335f6675..104fd4e68 100644 --- a/nova/tests/compute/test_compute.py +++ b/nova/tests/compute/test_compute.py @@ -3696,8 +3696,12 @@ class ComputeAPITestCase(BaseTestCase): {'vm_state': vm_states.SOFT_DELETED, 'task_state': None}) + # Ensure quotas are committed self.mox.StubOutWithMock(nova.quota.QUOTAS, 'commit') nova.quota.QUOTAS.commit(mox.IgnoreArg(), mox.IgnoreArg()) + if self.__class__.__name__ == 'CellsComputeAPITestCase': + # Called a 2nd time (for the child cell) when testing cells + nova.quota.QUOTAS.commit(mox.IgnoreArg(), mox.IgnoreArg()) self.mox.ReplayAll() self.compute_api.restore(self.context, instance) diff --git a/nova/tests/compute/test_compute_cells.py b/nova/tests/compute/test_compute_cells.py new file mode 100644 index 000000000..aa4b448d4 --- /dev/null +++ b/nova/tests/compute/test_compute_cells.py @@ -0,0 +1,99 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# Copyright (c) 2012 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Compute w/ Cells +""" +from nova.compute import cells_api as compute_cells_api +from nova.openstack.common import log as logging +from nova.tests.compute import test_compute + + +LOG = logging.getLogger('nova.tests.test_compute_cells') + +ORIG_COMPUTE_API = None + + +def stub_call_to_cells(context, instance, method, *args, **kwargs): + fn = getattr(ORIG_COMPUTE_API, method) + return fn(context, instance, *args, **kwargs) + + +def stub_cast_to_cells(context, instance, method, *args, **kwargs): + fn = getattr(ORIG_COMPUTE_API, method) + fn(context, instance, *args, **kwargs) + + +def deploy_stubs(stubs, api): + stubs.Set(api, '_call_to_cells', stub_call_to_cells) + stubs.Set(api, '_cast_to_cells', stub_cast_to_cells) + + +class CellsComputeAPITestCase(test_compute.ComputeAPITestCase): + def setUp(self): + super(CellsComputeAPITestCase, self).setUp() + global ORIG_COMPUTE_API + ORIG_COMPUTE_API = self.compute_api + + def _fake_cell_read_only(*args, **kwargs): + return False + + def _fake_validate_cell(*args, **kwargs): + return + + def _nop_update(context, instance, **kwargs): + return instance + + self.compute_api = compute_cells_api.ComputeCellsAPI() + self.stubs.Set(self.compute_api, '_cell_read_only', + _fake_cell_read_only) + self.stubs.Set(self.compute_api, '_validate_cell', + _fake_validate_cell) + + # NOTE(belliott) Don't update the instance state + # for the tests at the API layer. Let it happen after + # the stub cast to cells so that expected_task_states + # match. + self.stubs.Set(self.compute_api, 'update', _nop_update) + + deploy_stubs(self.stubs, self.compute_api) + + def tearDown(self): + global ORIG_COMPUTE_API + self.compute_api = ORIG_COMPUTE_API + super(CellsComputeAPITestCase, self).tearDown() + + def test_instance_metadata(self): + self.skipTest("Test is incompatible with cells.") + + def test_live_migrate(self): + self.skipTest("Test is incompatible with cells.") + + def test_get_backdoor_port(self): + self.skipTest("Test is incompatible with cells.") + + +class CellsComputePolicyTestCase(test_compute.ComputePolicyTestCase): + def setUp(self): + super(CellsComputePolicyTestCase, self).setUp() + global ORIG_COMPUTE_API + ORIG_COMPUTE_API = self.compute_api + self.compute_api = compute_cells_api.ComputeCellsAPI() + deploy_stubs(self.stubs, self.compute_api) + + def tearDown(self): + global ORIG_COMPUTE_API + self.compute_api = ORIG_COMPUTE_API + super(CellsComputePolicyTestCase, self).tearDown() -- cgit