Handle compute node not available for live migration

This patch handles exception.ComputeServiceUnavailable by restoring
instance's vm_state and instance's task_state after live migration
failure caused by unavailable source/dest compute node.

Raises detailed HTTPBadRequest explanation for this exception.

Fixes bug 973393 and bug 1051881

Change-Id: If825b61fad9c4e3030f2e6c5002907255eaf3661

6 files changed, 121 insertions, 12 deletions

diff --git a/nova/api/openstack/compute/contrib/admin_actions.py b/nova/api/openstack/compute/contrib/admin_actions.py
index 1c053ea59..dc3ee8fc4 100644
--- a/nova/api/openstack/compute/contrib/admin_actions.py
+++ b/nova/api/openstack/compute/contrib/admin_actions.py
@@ -282,6 +282,8 @@ class AdminActionsController(wsgi.Controller):
             instance = self.compute_api.get(context, id)
             self.compute_api.live_migrate(context, instance, block_migration,
                                           disk_over_commit, host)
+        except exception.ComputeServiceUnavailable as ex:
+            raise exc.HTTPBadRequest(explanation=str(ex))
         except Exception:
             msg = _("Live migration of instance %(id)s to host %(host)s"
                     " failed") % locals()
diff --git a/nova/exception.py b/nova/exception.py
index 6915c14bb..92bb8b993 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -358,7 +358,7 @@ class ComputeResourcesUnavailable(ServiceUnavailable):
 
 
 class ComputeServiceUnavailable(ServiceUnavailable):
-    message = _("Compute service is unavailable at this time.")
+    message = _("Compute service of %(host)s is unavailable at this time.")
 
 
 class UnableToMigrateToSelf(Invalid):
diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py
index 01bef4185..dc945a515 100644
--- a/nova/scheduler/driver.py
+++ b/nova/scheduler/driver.py
@@ -209,7 +209,10 @@ class Scheduler(object):
         """
 
         # Checking dest exists and compute node.
-        dservice_ref = db.service_get_by_compute_host(context, dest)
+        try:
+            dservice_ref = db.service_get_by_compute_host(context, dest)
+        except exception.NotFound:
+            raise exception.ComputeServiceUnavailable(host=dest)
 
         # Checking dest host is alive.
         if not self.servicegroup_api.service_is_up(dservice_ref):
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index e6bf1a293..830d61852 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -24,6 +24,7 @@ Scheduler Service
 import sys
 
 from nova.compute import rpcapi as compute_rpcapi
+from nova.compute import task_states
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova.conductor import api as conductor_api
@@ -92,6 +93,16 @@ class SchedulerManager(manager.Manager):
             return self.driver.schedule_live_migration(
                 context, instance, dest,
                 block_migration, disk_over_commit)
+        except exception.ComputeServiceUnavailable as ex:
+            request_spec = {'instance_properties': {
+                'uuid': instance['uuid'], },
+            }
+            with excutils.save_and_reraise_exception():
+                self._set_vm_state_and_notify('live_migration',
+                            dict(vm_state=instance['vm_state'],
+                                 task_state=None,
+                                 expected_task_state=task_states.MIGRATING,),
+                                              context, ex, request_spec)
         except Exception as ex:
             with excutils.save_and_reraise_exception():
                 self._set_vm_state_and_notify('live_migration',
diff --git a/nova/tests/api/openstack/compute/contrib/test_admin_actions.py b/nova/tests/api/openstack/compute/contrib/test_admin_actions.py
index dfb687cf4..76351e489 100644
--- a/nova/tests/api/openstack/compute/contrib/test_admin_actions.py
+++ b/nova/tests/api/openstack/compute/contrib/test_admin_actions.py
@@ -64,13 +64,6 @@ def fake_compute_api_get(self, context, instance_id):
             'task_state': None}
 
 
-def fake_scheduler_api_live_migration(self, context, dest,
-                                      block_migration=False,
-                                      disk_over_commit=False, instance=None,
-                                      instance_id=None, topic=None):
-    return None
-
-
 class AdminActionsTest(test.TestCase):
 
     _actions = ('pause', 'unpause', 'suspend', 'resume', 'migrate',
@@ -93,9 +86,6 @@ class AdminActionsTest(test.TestCase):
         self.UUID = uuid.uuid4()
         for _method in self._methods:
             self.stubs.Set(compute_api.API, _method, fake_compute_api)
-        self.stubs.Set(scheduler_rpcapi.SchedulerAPI,
-                       'live_migration',
-                       fake_scheduler_api_live_migration)
         self.flags(
             osapi_compute_extension=[
                 'nova.api.openstack.compute.contrib.select_extensions'],
@@ -150,7 +140,16 @@ class AdminActionsTest(test.TestCase):
                         task_state, expected_task_state):
             return None
 
+        def fake_scheduler_api_live_migration(self, context, dest,
+                                        block_migration=False,
+                                        disk_over_commit=False, instance=None,
+                                        instance_id=None, topic=None):
+            return None
+
         self.stubs.Set(compute_api.API, 'update', fake_update)
+        self.stubs.Set(scheduler_rpcapi.SchedulerAPI,
+                       'live_migration',
+                       fake_scheduler_api_live_migration)
 
         res = req.get_response(app)
         self.assertEqual(res.status_int, 202)
@@ -174,6 +173,44 @@ class AdminActionsTest(test.TestCase):
         res = req.get_response(app)
         self.assertEqual(res.status_int, 400)
 
+    def test_migrate_live_compute_service_unavailable(self):
+        ctxt = context.get_admin_context()
+        ctxt.user_id = 'fake'
+        ctxt.project_id = 'fake'
+        ctxt.is_admin = True
+        app = fakes.wsgi_app(fake_auth_context=ctxt, init_only=('servers',))
+        req = webob.Request.blank('/v2/fake/servers/%s/action' % self.UUID)
+        req.method = 'POST'
+        req.body = jsonutils.dumps({
+            'os-migrateLive': {
+                'host': 'hostname',
+                'block_migration': False,
+                'disk_over_commit': False,
+            }
+        })
+        req.content_type = 'application/json'
+
+        def fake_update(inst, context, instance,
+                        task_state, expected_task_state):
+            return None
+
+        def fake_scheduler_api_live_migration(context, dest,
+                                        block_migration=False,
+                                        disk_over_commit=False, instance=None,
+                                        instance_id=None, topic=None):
+            raise exception.ComputeServiceUnavailable(host='host')
+
+        self.stubs.Set(compute_api.API, 'update', fake_update)
+        self.stubs.Set(scheduler_rpcapi.SchedulerAPI,
+                       'live_migration',
+                       fake_scheduler_api_live_migration)
+
+        res = req.get_response(app)
+        self.assertEqual(res.status_int, 400)
+        self.assertIn(
+            unicode(exception.ComputeServiceUnavailable(host='host')),
+            res.body)
+
 
 class CreateBackupTests(test.TestCase):
 
diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py
index 142d8ea0e..2cb15ee0c 100644
--- a/nova/tests/scheduler/test_scheduler.py
+++ b/nova/tests/scheduler/test_scheduler.py
@@ -24,6 +24,7 @@ import mox
 from nova.compute import api as compute_api
 from nova.compute import power_state
 from nova.compute import rpcapi as compute_rpcapi
+from nova.compute import task_states
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova.conductor import api as conductor_api
@@ -197,6 +198,38 @@ class SchedulerManagerTestCase(test.TestCase):
         self.manager.run_instance(self.context, request_spec,
                 None, None, None, None, {})
 
+    def test_live_migration_compute_service_notavailable(self):
+        inst = {"uuid": "fake-instance-id",
+                "vm_state": vm_states.ACTIVE,
+                "task_state": task_states.MIGRATING, }
+
+        dest = 'fake_host'
+        block_migration = False
+        disk_over_commit = False
+
+        self._mox_schedule_method_helper('schedule_live_migration')
+        self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
+        self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
+
+        self.manager.driver.schedule_live_migration(self.context,
+                    inst, dest, block_migration, disk_over_commit).AndRaise(
+                    exception.ComputeServiceUnavailable(host="src"))
+        db.instance_update_and_get_original(self.context, inst["uuid"],
+                                {"vm_state": inst['vm_state'],
+                                 "task_state": None,
+                                 "expected_task_state": task_states.MIGRATING,
+                                }).AndReturn((inst, inst))
+        compute_utils.add_instance_fault_from_exc(self.context,
+                                mox.IsA(conductor_api.LocalAPI), inst,
+                                mox.IsA(exception.ComputeServiceUnavailable),
+                                mox.IgnoreArg())
+
+        self.mox.ReplayAll()
+        self.assertRaises(exception.ComputeServiceUnavailable,
+                          self.manager.live_migration,
+                          self.context, inst, dest, block_migration,
+                          disk_over_commit)
+
     def test_prep_resize_no_valid_host_back_in_active_state(self):
         fake_instance_uuid = 'fake-instance-id'
         inst = {"vm_state": "", "task_state": ""}
@@ -502,6 +535,29 @@ class SchedulerTestCase(test.TestCase):
                 block_migration=block_migration,
                 disk_over_commit=disk_over_commit)
 
+    def test_live_migration_compute_dest_not_exist(self):
+        # Raise exception when dest compute node does not exist.
+
+        self.mox.StubOutWithMock(self.driver, '_live_migration_src_check')
+        self.mox.StubOutWithMock(db, 'service_get_by_compute_host')
+
+        dest = 'fake_host2'
+        block_migration = False
+        disk_over_commit = False
+        instance = self._live_migration_instance()
+
+        self.driver._live_migration_src_check(self.context, instance)
+        # Compute down
+        db.service_get_by_compute_host(self.context,
+                            dest).AndRaise(exception.NotFound())
+
+        self.mox.ReplayAll()
+        self.assertRaises(exception.ComputeServiceUnavailable,
+                self.driver.schedule_live_migration, self.context,
+                instance=instance, dest=dest,
+                block_migration=block_migration,
+                disk_over_commit=disk_over_commit)
+
     def test_live_migration_compute_dest_not_alive(self):
         # Raise exception when dest compute node is not alive.