Generalize console error handling during build.

There is a race condition where you can create an instance and then
quickly request a VNC or Spice console before the instance is ready.
The way this went down was different depending on the virt driver in
use.  The libvirt driver would raise InstanceNotFound.  The xenapi
driver would raise InstanceNotReady.

This patch moves the handling of this race that was in the xenapi driver
up to the compute manager.  Now, all of the virt drivers that support
this method (libvirt, xenapi, vmware) will all raise InstanceNotFound in
this case, and the compute manager will convert it into InstanceNotReady
IFF the vm_state is BUILDING.

Related to bug 1154327.

Change-Id: I68f4a6db8aac26c6f731c985d97299ee38c34448

3 files changed, 66 insertions, 24 deletions

diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 3d3a094a2..2ced60da3 100755
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -2580,7 +2580,8 @@ class ComputeManager(manager.SchedulerDependentManager):
         else:
             return '\n'.join(log.split('\n')[-int(length):])
 
-    @rpc_common.client_exceptions(exception.ConsoleTypeInvalid)
+    @rpc_common.client_exceptions(exception.ConsoleTypeInvalid,
+            exception.InstanceNotReady, exception.InstanceNotFound)
     @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
     @wrap_instance_fault
     def get_vnc_console(self, context, console_type, instance):
@@ -2601,15 +2602,21 @@ class ComputeManager(manager.SchedulerDependentManager):
         else:
             raise exception.ConsoleTypeInvalid(console_type=console_type)
 
-        # Retrieve connect info from driver, and then decorate with our
-        # access info token
-        connect_info = self.driver.get_vnc_console(instance)
-        connect_info['token'] = token
-        connect_info['access_url'] = access_url
+        try:
+            # Retrieve connect info from driver, and then decorate with our
+            # access info token
+            connect_info = self.driver.get_vnc_console(instance)
+            connect_info['token'] = token
+            connect_info['access_url'] = access_url
+        except exception.InstanceNotFound:
+            if instance['vm_state'] != vm_states.BUILDING:
+                raise
+            raise exception.InstanceNotReady(instance_id=instance['uuid'])
 
         return connect_info
 
-    @rpc_common.client_exceptions(exception.ConsoleTypeInvalid)
+    @rpc_common.client_exceptions(exception.ConsoleTypeInvalid,
+            exception.InstanceNotReady, exception.InstanceNotFound)
     @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
     @wrap_instance_fault
     def get_spice_console(self, context, console_type, instance):
@@ -2629,14 +2636,21 @@ class ComputeManager(manager.SchedulerDependentManager):
         else:
             raise exception.ConsoleTypeInvalid(console_type=console_type)
 
-        # Retrieve connect info from driver, and then decorate with our
-        # access info token
-        connect_info = self.driver.get_spice_console(instance)
-        connect_info['token'] = token
-        connect_info['access_url'] = access_url
+        try:
+            # Retrieve connect info from driver, and then decorate with our
+            # access info token
+            connect_info = self.driver.get_spice_console(instance)
+            connect_info['token'] = token
+            connect_info['access_url'] = access_url
+        except exception.InstanceNotFound:
+            if instance['vm_state'] != vm_states.BUILDING:
+                raise
+            raise exception.InstanceNotReady(instance_id=instance['uuid'])
 
         return connect_info
 
+    @rpc_common.client_exceptions(exception.ConsoleTypeInvalid,
+            exception.InstanceNotReady, exception.InstanceNotFound)
     @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
     @wrap_instance_fault
     def validate_console_port(self, ctxt, instance, port, console_type):
diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py
index 7be3cf9a0..99fb383b3 100644
--- a/nova/tests/compute/test_compute.py
+++ b/nova/tests/compute/test_compute.py
@@ -1622,6 +1622,44 @@ class ComputeTestCase(BaseTestCase):
 
         self.compute.terminate_instance(self.context, instance=instance)
 
+    def test_vnc_console_instance_not_ready(self):
+        self.flags(vnc_enabled=True)
+        self.flags(enabled=False, group='spice')
+        instance = self._create_fake_instance(
+                params={'vm_state': vm_states.BUILDING})
+        instance = jsonutils.to_primitive(instance)
+
+        def fake_driver_get_console(*args, **kwargs):
+            raise exception.InstanceNotFound(instance_id=instance['uuid'])
+
+        self.stubs.Set(self.compute.driver, "get_vnc_console",
+                       fake_driver_get_console)
+
+        self.stub_out_client_exceptions()
+
+        self.assertRaises(exception.InstanceNotReady,
+                self.compute.get_vnc_console, self.context, 'novnc',
+                instance=instance)
+
+    def test_spice_console_instance_not_ready(self):
+        self.flags(vnc_enabled=False)
+        self.flags(enabled=True, group='spice')
+        instance = self._create_fake_instance(
+                params={'vm_state': vm_states.BUILDING})
+        instance = jsonutils.to_primitive(instance)
+
+        def fake_driver_get_console(*args, **kwargs):
+            raise exception.InstanceNotFound(instance_id=instance['uuid'])
+
+        self.stubs.Set(self.compute.driver, "get_spice_console",
+                       fake_driver_get_console)
+
+        self.stub_out_client_exceptions()
+
+        self.assertRaises(exception.InstanceNotReady,
+                self.compute.get_spice_console, self.context, 'spice-html5',
+                instance=instance)
+
     def test_diagnostics(self):
         # Make sure we can get diagnostics for an instance.
         expected_diagnostic = {'cpu0_time': 17300000000,
diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py
index 56dd5bd3d..fd028ecef 100644
--- a/nova/virt/xenapi/vmops.py
+++ b/nova/virt/xenapi/vmops.py
@@ -33,7 +33,6 @@ from nova.compute import instance_types
 from nova.compute import power_state
 from nova.compute import task_states
 from nova.compute import vm_mode
-from nova.compute import vm_states
 from nova import context as nova_context
 from nova import exception
 from nova.openstack.common import excutils
@@ -1325,20 +1324,11 @@ class VMOps(object):
 
     def get_vnc_console(self, instance):
         """Return connection info for a vnc console."""
-        # NOTE(johannes): This can fail if the VM object hasn't been created
-        # yet on the dom0. Since that step happens fairly late in the build
-        # process, there's a potential for a race condition here. Until the
-        # VM object is created, return back a 409 error instead of a 404
-        # error.
         try:
             vm_ref = self._get_vm_opaque_ref(instance)
         except exception.NotFound:
-            if instance['vm_state'] != vm_states.BUILDING:
-                raise
-
-            LOG.info(_('Fetching VM ref while BUILDING failed'),
-                     instance=instance)
-            raise exception.InstanceNotReady(instance_id=instance['uuid'])
+            # The compute manager expects InstanceNotFound for this case.
+            raise exception.InstanceNotFound(instance_id=instance['uuid'])
 
         session_id = self._session.get_session_id()
         path = "/console?ref=%s&session_id=%s" % (str(vm_ref), session_id)