Ignore InstanceNotFound when trying to set instance to ERROR

Fixes bug 948632

There's a race condition where an instance can be deleted from an API
call... immediately before some other error occurs during build.

Also: compute tests should not test raising quantum exceptions, as
those could never possibly make it back to us.  rpc.call only raises
RemoteError... so use that instead.

Change-Id: Iacfe511a37e937eaec1d6213bf73153aa4d4e9bf

2 files changed, 31 insertions, 6 deletions

diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 3d27d94cc..4b640b6ef 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -216,9 +216,13 @@ class ComputeManager(manager.SchedulerDependentManager):
         return self.db.instance_update(context, instance_id, kwargs)
 
     def _set_instance_error_state(self, context, instance_uuid):
-        self._instance_update(context,
-                              instance_uuid,
-                              vm_state=vm_states.ERROR)
+        try:
+            self._instance_update(context,
+                    instance_uuid, vm_state=vm_states.ERROR)
+        except exception.InstanceNotFound:
+            LOG.debug(_("Instance %(instance_uuid)s has been destroyed "
+                    "from under us while trying to set it to ERROR") %
+                    locals())
 
     def init_host(self):
         """Initialization for a standalone compute service."""
diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 3b784dfa1..ef416543d 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -43,10 +43,10 @@ from nova import exception
 from nova import flags
 from nova.image import fake as fake_image
 from nova import log as logging
-from nova.network.quantum import client as quantum_client
 from nova.notifier import test_notifier
 import nova.policy
 from nova import rpc
+from nova.rpc import common as rpc_common
 from nova.scheduler import driver as scheduler_driver
 from nova import test
 from nova.tests import fake_network
@@ -942,13 +942,13 @@ class ComputeTestCase(BaseTestCase):
                 mox.IgnoreArg(),
                 mox.IgnoreArg(),
                 requested_networks=None,
-                vpn=False).AndRaise(quantum_client.QuantumServerException())
+                vpn=False).AndRaise(rpc_common.RemoteError())
 
         self.flags(stub_network=False)
 
         self.mox.ReplayAll()
 
-        self.assertRaises(quantum_client.QuantumServerException,
+        self.assertRaises(rpc_common.RemoteError,
                           self.compute.run_instance,
                           self.context,
                           instance_uuid)
@@ -959,6 +959,27 @@ class ComputeTestCase(BaseTestCase):
 
         self.compute.terminate_instance(self.context, instance['uuid'])
 
+    def test_instance_set_to_error_on_deleted_instance_doesnt_raise(self):
+        """Test that we don't raise InstanceNotFound when trying to set
+        an instance to ERROR that has already been deleted from under us.
+        The original exception should be re-raised.
+        """
+        instance = self._create_fake_instance()
+        instance_uuid = instance['uuid']
+
+        def fake_allocate_network(context, instance, requested_networks):
+            # Remove the instance to simulate race condition
+            self.compute.terminate_instance(self.context, instance['uuid'])
+            raise rpc_common.RemoteError()
+
+        self.stubs.Set(self.compute, '_allocate_network',
+                       fake_allocate_network)
+
+        self.assertRaises(rpc_common.RemoteError,
+                          self.compute.run_instance,
+                          self.context,
+                          instance_uuid)
+
     def test_network_is_deallocated_on_spawn_failure(self):
         """When a spawn fails the network must be deallocated"""
         instance = self._create_fake_instance()