From e6e5123cceb874a7ca6dcb16bc401f530439d07a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 11 Sep 2012 12:09:38 -0700 Subject: Allows waiting timers in libvirt to raise NotFound There are cases where an operation will fail when communicating with libvirt. We were eating the exception even though the operation failed, which has the potential to put the instance into an unrecoverable state. This patch allows NotFound exceptions to propogate up so that they are caught by the state handling code and the task state can be set to error. Fixes bug 1002814 Change-Id: Iddc319b24aee0b7132155f50b9d3b0eee9bb3fa8 --- nova/tests/test_libvirt.py | 30 +++----------------------- nova/virt/libvirt/driver.py | 51 ++++++++++++++++++--------------------------- 2 files changed, 23 insertions(+), 58 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index 060a8445a..19839c56e 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -2251,28 +2251,7 @@ class LibvirtConnTestCase(test.TestCase): "uuid": "875a8070-d0b9-4949-8b31-104d125c9a64"} conn.destroy(instance, []) - def test_private_destroy(self): - """Ensure Instance not found skips undefine""" - mock = self.mox.CreateMock(libvirt.virDomain) - mock.destroy() - self.mox.ReplayAll() - - def fake_lookup_by_name(instance_name): - return mock - - def fake_get_info(instance_name): - return {'state': power_state.SHUTDOWN} - - conn = libvirt_driver.LibvirtDriver(False) - self.stubs.Set(conn, '_lookup_by_name', fake_lookup_by_name) - self.stubs.Set(conn, 'get_info', fake_get_info) - instance = {"name": "instancename", "id": "instanceid", - "uuid": "875a8070-d0b9-4949-8b31-104d125c9a64"} - result = conn._destroy(instance) - self.assertTrue(result) - def test_private_destroy_not_found(self): - """Ensure Instance not found skips undefine""" mock = self.mox.CreateMock(libvirt.virDomain) mock.destroy() self.mox.ReplayAll() @@ -2288,8 +2267,8 @@ class LibvirtConnTestCase(test.TestCase): self.stubs.Set(conn, 'get_info', fake_get_info) instance = {"name": "instancename", "id": "instanceid", "uuid": "875a8070-d0b9-4949-8b31-104d125c9a64"} - result = conn._destroy(instance) - self.assertFalse(result) + # NOTE(vish): verifies destory doesn't raise if the instance disappears + conn._destroy(instance) def test_available_least_handles_missing(self): """Ensure destroy calls managedSaveRemove for saved instance""" @@ -3714,9 +3693,6 @@ class LibvirtDriverTestCase(test.TestCase): self.assertEquals(out, disk_info_text) def test_wait_for_running(self): - """Test for nova.virt.libvirt.libvirt_driver.LivirtConnection - ._wait_for_running. """ - def fake_get_info(instance): if instance['name'] == "not_found": raise exception.NotFound @@ -3729,7 +3705,7 @@ class LibvirtDriverTestCase(test.TestCase): fake_get_info) """ instance not found case """ - self.assertRaises(utils.LoopingCallDone, + self.assertRaises(exception.NotFound, self.libvirtconnection._wait_for_running, {'name': 'not_found', 'uuid': 'not_found_uuid'}) diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 7632ad960..daa0840b6 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -472,20 +472,24 @@ class LibvirtDriver(driver.ComputeDriver): def _wait_for_destroy(): """Called at an interval until the VM is gone.""" + # NOTE(vish): If the instance disappears during the destroy + # we ignore it so the cleanup can still be + # attempted because we would prefer destroy to + # never fail. try: state = self.get_info(instance)['state'] except exception.NotFound: LOG.error(_("During wait destroy, instance disappeared."), instance=instance) - raise utils.LoopingCallDone(False) + raise utils.LoopingCallDone() if state == power_state.SHUTDOWN: LOG.info(_("Instance destroyed successfully."), instance=instance) - raise utils.LoopingCallDone(True) + raise utils.LoopingCallDone() timer = utils.LoopingCall(_wait_for_destroy) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() def destroy(self, instance, network_info, block_device_info=None): self._destroy(instance) @@ -895,7 +899,7 @@ class LibvirtDriver(driver.ComputeDriver): instance=instance) self._create_domain(domain=dom) timer = utils.LoopingCall(self._wait_for_running, instance) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() greenthread.sleep(1) return False @@ -922,20 +926,15 @@ class LibvirtDriver(driver.ComputeDriver): def _wait_for_reboot(): """Called at an interval until the VM is running again.""" - try: - state = self.get_info(instance)['state'] - except exception.NotFound: - LOG.error(_("During reboot, instance disappeared."), - instance=instance) - raise utils.LoopingCallDone + state = self.get_info(instance)['state'] if state == power_state.RUNNING: LOG.info(_("Instance rebooted successfully."), instance=instance) - raise utils.LoopingCallDone + raise utils.LoopingCallDone() timer = utils.LoopingCall(_wait_for_reboot) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() @exception.wrap_exception() def pause(self, instance): @@ -960,7 +959,7 @@ class LibvirtDriver(driver.ComputeDriver): dom = self._lookup_by_name(instance['name']) self._create_domain(domain=dom) timer = utils.LoopingCall(self._wait_for_running, instance) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() @exception.wrap_exception() def suspend(self, instance): @@ -1064,20 +1063,15 @@ class LibvirtDriver(driver.ComputeDriver): def _wait_for_boot(): """Called at an interval until the VM is running.""" - try: - state = self.get_info(instance)['state'] - except exception.NotFound: - LOG.error(_("During spawn, instance disappeared."), - instance=instance) - raise utils.LoopingCallDone + state = self.get_info(instance)['state'] if state == power_state.RUNNING: LOG.info(_("Instance spawned successfully."), instance=instance) - raise utils.LoopingCallDone + raise utils.LoopingCallDone() timer = utils.LoopingCall(_wait_for_boot) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() def _flush_libvirt_console(self, pty): out, err = utils.execute('dd', @@ -2509,7 +2503,7 @@ class LibvirtDriver(driver.ComputeDriver): post_method(ctxt, instance_ref, dest, block_migration) timer.f = wait_for_live_migration - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() def pre_live_migration(self, context, instance_ref, block_device_info, network_info): @@ -2824,16 +2818,11 @@ class LibvirtDriver(driver.ComputeDriver): return disk_info_text def _wait_for_running(self, instance): - try: - state = self.get_info(instance)['state'] - except exception.NotFound: - LOG.error(_("During wait running, instance disappeared."), - instance=instance) - raise utils.LoopingCallDone(False) + state = self.get_info(instance)['state'] if state == power_state.RUNNING: LOG.info(_("Instance running successfully."), instance=instance) - raise utils.LoopingCallDone(True) + raise utils.LoopingCallDone() @exception.wrap_exception() def finish_migration(self, context, migration, instance, disk_info, @@ -2882,7 +2871,7 @@ class LibvirtDriver(driver.ComputeDriver): block_device_info=None) self._create_domain_and_network(xml, instance, network_info) timer = utils.LoopingCall(self._wait_for_running, instance) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() @exception.wrap_exception() def finish_revert_migration(self, instance, network_info): @@ -2898,7 +2887,7 @@ class LibvirtDriver(driver.ComputeDriver): self._create_domain_and_network(xml, instance, network_info) timer = utils.LoopingCall(self._wait_for_running, instance) - return timer.start(interval=0.5).wait() + timer.start(interval=0.5).wait() def confirm_migration(self, migration, instance, network_info): """Confirms a resize, destroying the source VM""" -- cgit