From bc0f2235d9b27e604b9264b5c19adce3cf306bc2 Mon Sep 17 00:00:00 2001 From: Brian Elliott Date: Sun, 13 May 2012 21:06:29 +0000 Subject: Added a instance state update notification Added a instance update notification (compute.instance.update) that will report on changes to vm_state and task_state. The goal here is to provide useful insight into instance state transitions. (e.g. BUILDING->ACTIVE) The new notification has minimial dependencies and is intended for wide use across the different layers/packages within nova. Calls in compute api/manager, scheduler, and the virt layer that modify the instance state have been instrumented with this notification. Change-Id: I223eb7eccc8aa079b782f6bb17727cd0b71d18ed --- .../openstack/compute/contrib/test_disk_config.py | 8 +- .../api/openstack/compute/test_server_actions.py | 6 +- nova/tests/api/openstack/compute/test_servers.py | 15 +- nova/tests/scheduler/test_scheduler.py | 46 +++--- nova/tests/test_db_api.py | 12 ++ nova/tests/test_notifications.py | 169 +++++++++++++++++++++ 6 files changed, 228 insertions(+), 28 deletions(-) create mode 100644 nova/tests/test_notifications.py (limited to 'nova/tests') diff --git a/nova/tests/api/openstack/compute/contrib/test_disk_config.py b/nova/tests/api/openstack/compute/contrib/test_disk_config.py index 59ae14ac8..41a1cd607 100644 --- a/nova/tests/api/openstack/compute/contrib/test_disk_config.py +++ b/nova/tests/api/openstack/compute/contrib/test_disk_config.py @@ -92,13 +92,13 @@ class DiskConfigTestCase(test.TestCase): inst['updated_at'] = datetime.datetime(2010, 10, 10, 12, 0, 0) inst['progress'] = 0 inst['name'] = 'instance-1' # this is a property + inst['task_state'] = '' + inst['vm_state'] = '' def fake_instance_get_for_create(context, id_, *args, **kwargs): - return inst + return (inst, inst) - self.stubs.Set(nova.db, 'instance_get', - fake_instance_get_for_create) - self.stubs.Set(nova.db, 'instance_update', + self.stubs.Set(nova.db, 'instance_update_and_get_original', fake_instance_get_for_create) def fake_instance_get_all_for_create(context, *args, **kwargs): diff --git a/nova/tests/api/openstack/compute/test_server_actions.py b/nova/tests/api/openstack/compute/test_server_actions.py index 774700361..04aef91f9 100644 --- a/nova/tests/api/openstack/compute/test_server_actions.py +++ b/nova/tests/api/openstack/compute/test_server_actions.py @@ -38,7 +38,8 @@ def return_server_not_found(context, uuid): def instance_update(context, instance_id, kwargs): - return fakes.stub_instance(instance_id) + inst = fakes.stub_instance(instance_id) + return (inst, inst) class MockSetAdminPassword(object): @@ -59,7 +60,8 @@ class ServerActionsControllerTest(test.TestCase): self.stubs.Set(nova.db, 'instance_get_by_uuid', fakes.fake_instance_get(vm_state=vm_states.ACTIVE, host='fake_host')) - self.stubs.Set(nova.db, 'instance_update', instance_update) + self.stubs.Set(nova.db, 'instance_update_and_get_original', + instance_update) fakes.stub_out_glance(self.stubs) fakes.stub_out_nw_api(self.stubs) diff --git a/nova/tests/api/openstack/compute/test_servers.py b/nova/tests/api/openstack/compute/test_servers.py index 14d1dff75..aefe19581 100644 --- a/nova/tests/api/openstack/compute/test_servers.py +++ b/nova/tests/api/openstack/compute/test_servers.py @@ -72,7 +72,8 @@ def return_security_group(context, instance_id, security_group_id): def instance_update(context, instance_id, values): - return fakes.stub_instance(instance_id, name=values.get('display_name')) + inst = fakes.stub_instance(instance_id, name=values.get('display_name')) + return (inst, inst) def fake_compute_api(cls, req, id): @@ -106,7 +107,8 @@ class ServersControllerTest(test.TestCase): return_servers) self.stubs.Set(nova.db, 'instance_add_security_group', return_security_group) - self.stubs.Set(nova.db, 'instance_update', instance_update) + self.stubs.Set(nova.db, 'instance_update_and_get_original', + instance_update) self.controller = servers.Controller() self.ips_controller = ips.Controller() @@ -1433,7 +1435,9 @@ class ServersControllerCreateTest(test.TestCase): "updated_at": datetime.datetime(2010, 11, 11, 11, 0, 0), "config_drive": None, "progress": 0, - "fixed_ips": [] + "fixed_ips": [], + "task_state": "", + "vm_state": "", } self.instance_cache[instance['id']] = instance return instance @@ -1459,7 +1463,7 @@ class ServersControllerCreateTest(test.TestCase): def server_update(context, instance_id, params): inst = self.instance_cache[instance_id] inst.update(params) - return inst + return (inst, inst) def fake_method(*args, **kwargs): pass @@ -1485,7 +1489,8 @@ class ServersControllerCreateTest(test.TestCase): self.stubs.Set(nova.db, 'instance_get', instance_get) self.stubs.Set(nova.rpc, 'cast', fake_method) self.stubs.Set(nova.rpc, 'call', rpc_call_wrapper) - self.stubs.Set(nova.db, 'instance_update', server_update) + self.stubs.Set(nova.db, 'instance_update_and_get_original', + server_update) self.stubs.Set(nova.db, 'queue_get_for', queue_get_for) self.stubs.Set(nova.network.manager.VlanManager, 'allocate_fixed_ip', fake_method) diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index 23b553d64..b7915ed8e 100644 --- a/nova/tests/scheduler/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -28,6 +28,7 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import notifications from nova import rpc from nova.rpc import common as rpc_common from nova.scheduler import driver @@ -232,9 +233,10 @@ class SchedulerManagerTestCase(test.TestCase): """ fake_instance_uuid = 'fake-instance-id' + inst = {"vm_state": "", "task_state": ""} self._mox_schedule_method_helper('schedule_run_instance') - self.mox.StubOutWithMock(db, 'instance_update') + self.mox.StubOutWithMock(db, 'instance_update_and_get_original') request_spec = {'instance_properties': {'uuid': fake_instance_uuid}} @@ -243,8 +245,8 @@ class SchedulerManagerTestCase(test.TestCase): self.manager.driver.schedule_run_instance(self.context, *self.fake_args, **self.fake_kwargs).AndRaise( exception.NoValidHost(reason="")) - db.instance_update(self.context, fake_instance_uuid, - {'vm_state': vm_states.ERROR}) + db.instance_update_and_get_original(self.context, fake_instance_uuid, + {"vm_state": vm_states.ERROR}).AndReturn((inst, inst)) self.mox.ReplayAll() self.manager.run_instance(self.context, self.topic, @@ -255,10 +257,11 @@ class SchedulerManagerTestCase(test.TestCase): the instance in ACTIVE state """ fake_instance_uuid = 'fake-instance-id' + inst = {"vm_state": "", "task_state": ""} self._mox_schedule_method_helper('schedule_prep_resize') - self.mox.StubOutWithMock(db, 'instance_update') + self.mox.StubOutWithMock(db, 'instance_update_and_get_original') request_spec = {'instance_properties': {'uuid': fake_instance_uuid}} @@ -267,9 +270,9 @@ class SchedulerManagerTestCase(test.TestCase): self.manager.driver.schedule_prep_resize(self.context, *self.fake_args, **self.fake_kwargs).AndRaise( exception.NoValidHost(reason="")) - db.instance_update(self.context, fake_instance_uuid, - {'vm_state': vm_states.ACTIVE, - 'task_state': None}) + db.instance_update_and_get_original(self.context, fake_instance_uuid, + {"vm_state": vm_states.ACTIVE, "task_state": None}).AndReturn( + (inst, inst)) self.mox.ReplayAll() self.manager.prep_resize(self.context, self.topic, @@ -283,7 +286,7 @@ class SchedulerManagerTestCase(test.TestCase): self._mox_schedule_method_helper('schedule_prep_resize') - self.mox.StubOutWithMock(db, 'instance_update') + self.mox.StubOutWithMock(db, 'instance_update_and_get_original') request_spec = {'instance_properties': {'uuid': fake_instance_uuid}} @@ -292,8 +295,13 @@ class SchedulerManagerTestCase(test.TestCase): self.manager.driver.schedule_prep_resize(self.context, *self.fake_args, **self.fake_kwargs).AndRaise( self.AnException('something happened')) - db.instance_update(self.context, fake_instance_uuid, - {'vm_state': vm_states.ERROR}) + + inst = { + "vm_state": "", + "task_state": "", + } + db.instance_update_and_get_original(self.context, fake_instance_uuid, + {"vm_state": vm_states.ERROR}).AndReturn((inst, inst)) self.mox.ReplayAll() @@ -421,7 +429,9 @@ class SchedulerTestCase(test.TestCase): 'power_state': power_state.RUNNING, 'memory_mb': 1024, 'root_gb': 1024, - 'ephemeral_gb': 0} + 'ephemeral_gb': 0, + 'vm_state': '', + 'task_state': ''} def test_live_migration_basic(self): """Test basic schedule_live_migration functionality""" @@ -429,7 +439,7 @@ class SchedulerTestCase(test.TestCase): self.mox.StubOutWithMock(self.driver, '_live_migration_src_check') self.mox.StubOutWithMock(self.driver, '_live_migration_dest_check') self.mox.StubOutWithMock(self.driver, '_live_migration_common_check') - self.mox.StubOutWithMock(db, 'instance_update') + self.mox.StubOutWithMock(db, 'instance_update_and_get_original') self.mox.StubOutWithMock(driver, 'cast_to_compute_host') dest = 'fake_host2' @@ -443,8 +453,9 @@ class SchedulerTestCase(test.TestCase): dest, block_migration, disk_over_commit) self.driver._live_migration_common_check(self.context, instance, dest, block_migration, disk_over_commit) - db.instance_update(self.context, instance['id'], - {'vm_state': vm_states.MIGRATING}) + db.instance_update_and_get_original(self.context, instance['id'], + {"vm_state": vm_states.MIGRATING}).AndReturn( + (instance, instance)) driver.cast_to_compute_host(self.context, instance['host'], 'live_migration', update_db=False, @@ -468,7 +479,7 @@ class SchedulerTestCase(test.TestCase): self.mox.StubOutWithMock(db, 'queue_get_for') self.mox.StubOutWithMock(rpc, 'call') self.mox.StubOutWithMock(rpc, 'cast') - self.mox.StubOutWithMock(db, 'instance_update') + self.mox.StubOutWithMock(db, 'instance_update_and_get_original') self.mox.StubOutWithMock(driver, 'cast_to_compute_host') dest = 'fake_host2' @@ -530,8 +541,9 @@ class SchedulerTestCase(test.TestCase): {'method': 'compare_cpu', 'args': {'cpu_info': 'fake_cpu_info'}}).AndReturn(True) - db.instance_update(self.context, instance['id'], - {'vm_state': vm_states.MIGRATING}) + db.instance_update_and_get_original(self.context, instance['id'], + {"vm_state": vm_states.MIGRATING}).AndReturn( + (instance, instance)) driver.cast_to_compute_host(self.context, instance['host'], 'live_migration', update_db=False, diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py index 33bef98c2..b16ffd335 100644 --- a/nova/tests/test_db_api.py +++ b/nova/tests/test_db_api.py @@ -211,6 +211,18 @@ class DbApiTestCase(test.TestCase): system_meta = db.instance_system_metadata_get(ctxt, instance.uuid) self.assertEqual('baz', system_meta['original_image_ref']) + def test_instance_update_with_and_get_original(self): + ctxt = context.get_admin_context() + + # Create an instance with some metadata + values = {'vm_state': 'building'} + instance = db.instance_create(ctxt, values) + + (old_ref, new_ref) = db.instance_update_and_get_original(ctxt, + instance['id'], {'vm_state': 'needscoffee'}) + self.assertEquals("building", old_ref["vm_state"]) + self.assertEquals("needscoffee", new_ref["vm_state"]) + def test_instance_fault_create(self): """Ensure we can create an instance fault""" ctxt = context.get_admin_context() diff --git a/nova/tests/test_notifications.py b/nova/tests/test_notifications.py new file mode 100644 index 000000000..a9d4f3f3b --- /dev/null +++ b/nova/tests/test_notifications.py @@ -0,0 +1,169 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for common notifcations.""" + +import copy + +from nova.compute import instance_types +from nova.compute import task_states +from nova.compute import vm_states +from nova import context +from nova import db +from nova import flags +import nova.image.fake +from nova import log as logging +from nova import notifications +from nova.notifier import test_notifier +from nova import test +from nova.tests import fake_network + +LOG = logging.getLogger(__name__) +FLAGS = flags.FLAGS +flags.DECLARE('stub_network', 'nova.compute.manager') + + +class NotificationsTestCase(test.TestCase): + + def setUp(self): + + def fake_get_nw_info(cls, ctxt, instance): + self.assertTrue(ctxt.is_admin) + return fake_network.fake_get_instance_nw_info(self.stubs, 1, 1, + spectacular=True) + + super(NotificationsTestCase, self).setUp() + self.stubs.Set(nova.network.API, 'get_instance_nw_info', + fake_get_nw_info) + + self.flags(connection_type='fake', + stub_network=True, + notification_driver='nova.notifier.test_notifier', + network_manager='nova.network.manager.FlatManager', + notify_on_state_change="vm_and_task_state") + + self.user_id = 'fake' + self.project_id = 'fake' + self.context = context.RequestContext(self.user_id, self.project_id) + test_notifier.NOTIFICATIONS = [] + + self.instance = self._wrapped_create() + + def _wrapped_create(self, params=None): + inst = {} + inst['image_ref'] = 1 + inst['user_id'] = self.user_id + inst['project_id'] = self.project_id + type_id = instance_types.get_instance_type_by_name('m1.tiny')['id'] + inst['instance_type_id'] = type_id + inst['root_gb'] = 0 + inst['ephemeral_gb'] = 0 + if params: + inst.update(params) + return db.instance_create(self.context, inst) + + def test_notif_disabled(self): + + # test config disable of the notifcations + self.flags(notify_on_state_change=None) + + old = copy.copy(self.instance) + self.instance["vm_state"] = vm_states.ACTIVE + + notifications.send_update(self.context, old, self.instance) + self.assertEquals(0, len(test_notifier.NOTIFICATIONS)) + + def test_task_notif(self): + + # test config disable of just the task state notifications + self.flags(notify_on_state_change="vm_state") + + # we should not get a notification on task stgate chagne now + old = copy.copy(self.instance) + self.instance["task_state"] = task_states.SPAWNING + + notifications.send_update(self.context, old, self.instance) + self.assertEquals(0, len(test_notifier.NOTIFICATIONS)) + + # ok now enable task state notifcations and re-try + self.flags(notify_on_state_change="vm_and_task_state") + + notifications.send_update(self.context, old, self.instance) + self.assertEquals(1, len(test_notifier.NOTIFICATIONS)) + + def test_send_no_notif(self): + + # test notification on send no initial vm state: + notifications.send_update(self.context, self.instance, self.instance) + self.assertEquals(0, len(test_notifier.NOTIFICATIONS)) + + def test_send_on_vm_change(self): + + # pretend we just transitioned to ACTIVE: + params = {"vm_state": vm_states.ACTIVE} + (old_ref, new_ref) = db.instance_update_and_get_original(self.context, + self.instance["id"], params) + notifications.send_update(self.context, old_ref, new_ref) + + self.assertEquals(1, len(test_notifier.NOTIFICATIONS)) + + def test_send_on_task_change(self): + + # pretend we just transitioned to task SPAWNING: + params = {"task_state": task_states.SPAWNING} + (old_ref, new_ref) = db.instance_update_and_get_original(self.context, + self.instance["id"], params) + print old_ref["task_state"] + print new_ref["task_state"] + notifications.send_update(self.context, old_ref, new_ref) + + self.assertEquals(1, len(test_notifier.NOTIFICATIONS)) + + def test_no_update_with_states(self): + + notifications.send_update_with_states(self.context, self.instance, + vm_states.BUILDING, vm_states.BUILDING, task_states.SPAWNING, + task_states.SPAWNING) + self.assertEquals(0, len(test_notifier.NOTIFICATIONS)) + + def test_vm_update_with_states(self): + + notifications.send_update_with_states(self.context, self.instance, + vm_states.BUILDING, vm_states.ACTIVE, task_states.SPAWNING, + task_states.SPAWNING) + self.assertEquals(1, len(test_notifier.NOTIFICATIONS)) + notif = test_notifier.NOTIFICATIONS[0] + payload = notif["payload"] + + self.assertEquals(vm_states.BUILDING, payload["old_state"]) + self.assertEquals(vm_states.ACTIVE, payload["state"]) + self.assertEquals(task_states.SPAWNING, payload["old_task_state"]) + self.assertEquals(task_states.SPAWNING, payload["new_task_state"]) + + def test_task_update_with_states(self): + + notifications.send_update_with_states(self.context, self.instance, + vm_states.BUILDING, vm_states.BUILDING, task_states.SPAWNING, + None) + self.assertEquals(1, len(test_notifier.NOTIFICATIONS)) + notif = test_notifier.NOTIFICATIONS[0] + payload = notif["payload"] + + self.assertEquals(vm_states.BUILDING, payload["old_state"]) + self.assertEquals(vm_states.BUILDING, payload["state"]) + self.assertEquals(task_states.SPAWNING, payload["old_task_state"]) + self.assertEquals(None, payload["new_task_state"]) -- cgit