Reset migrating task state for MigrationError exceptions

Create an exception type for pre-check errors and add it to the list
of exceptions for which instance task state is reset to ACTIVE to
avoid cases where an instance get stuck in MIGRATING state.

Convert existing instances of raise MigrationError in scheduler and
driver pre-checks to have them raise the new MigrationPreCheckError
exception instead.

In addition, also make sure that expected exceptions are passed over
RPC silently.

Resolves bug 1171526.

Change-Id: I00fa0962e555f895dd98861684ec7767fed3b37b

7 files changed, 34 insertions, 9 deletions

diff --git a/nova/exception.py b/nova/exception.py
index 698b49c2b..c81362dcf 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -946,6 +946,10 @@ class MigrationError(NovaException):
     message = _("Migration error") + ": %(reason)s"
 
 
+class MigrationPreCheckError(MigrationError):
+    message = _("Migration pre-check error") + ": %(reason)s"
+
+
 class MalformedRequestBody(NovaException):
     message = _("Malformed message body: %(reason)s")
 
diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py
index 44960a856..92c7779e8 100644
--- a/nova/scheduler/driver.py
+++ b/nova/scheduler/driver.py
@@ -335,7 +335,7 @@ class Scheduler(object):
             reason = _("Unable to migrate %(instance_uuid)s to %(dest)s: "
                        "Lack of memory(host:%(avail)s <= "
                        "instance:%(mem_inst)s)")
-            raise exception.MigrationError(reason=reason % locals())
+            raise exception.MigrationPreCheckError(reason=reason % locals())
 
     def _get_compute_info(self, context, host):
         """get compute node's information specified by key
diff --git a/nova/scheduler/filter_scheduler.py b/nova/scheduler/filter_scheduler.py
index 5afa8dc91..8f99db0ae 100644
--- a/nova/scheduler/filter_scheduler.py
+++ b/nova/scheduler/filter_scheduler.py
@@ -392,4 +392,4 @@ class FilterScheduler(driver.Scheduler):
                         "Lack of memory")
                       % {'instance_uuid': instance_uuid,
                          'dest': dest})
-            raise exception.MigrationError(reason=reason)
+            raise exception.MigrationPreCheckError(reason=reason)
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index c71b34963..65b171eba 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -40,6 +40,7 @@ from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging
 from nova.openstack.common.notifier import api as notifier
 from nova.openstack.common import periodic_task
+from nova.openstack.common.rpc import common as rpc_common
 from nova import quota
 
 
@@ -90,6 +91,14 @@ class SchedulerManager(manager.Manager):
         #function removed in RPC API 2.3
         pass
 
+    @rpc_common.client_exceptions(exception.NoValidHost,
+                                  exception.ComputeServiceUnavailable,
+                                  exception.InvalidHypervisorType,
+                                  exception.UnableToMigrateToSelf,
+                                  exception.DestinationHypervisorTooOld,
+                                  exception.InvalidLocalStorage,
+                                  exception.InvalidSharedStorage,
+                                  exception.MigrationPreCheckError)
     def live_migration(self, context, instance, dest,
                        block_migration, disk_over_commit):
         try:
@@ -102,7 +111,8 @@ class SchedulerManager(manager.Manager):
                 exception.UnableToMigrateToSelf,
                 exception.DestinationHypervisorTooOld,
                 exception.InvalidLocalStorage,
-                exception.InvalidSharedStorage) as ex:
+                exception.InvalidSharedStorage,
+                exception.MigrationPreCheckError) as ex:
             request_spec = {'instance_properties': {
                 'uuid': instance['uuid'], },
             }
diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py
index 92b5f4ca4..461b81b4b 100644
--- a/nova/tests/scheduler/test_scheduler.py
+++ b/nova/tests/scheduler/test_scheduler.py
@@ -36,6 +36,7 @@ from nova.image import glance
 from nova.openstack.common import jsonutils
 from nova.openstack.common.notifier import api as notifier
 from nova.openstack.common import rpc
+from nova.openstack.common.rpc import common as rpc_common
 from nova.scheduler import driver
 from nova.scheduler import manager
 from nova import servicegroup
@@ -65,6 +66,12 @@ class SchedulerManagerTestCase(test.TestCase):
         self.fake_kwargs = {'cat': 'meow', 'dog': 'woof'}
         fake_instance_actions.stub_out_action_events(self.stubs)
 
+    def stub_out_client_exceptions(self):
+        def passthru(exceptions, func, *args, **kwargs):
+            return func(*args, **kwargs)
+
+        self.stubs.Set(rpc_common, 'catch_client_exception', passthru)
+
     def test_1_correct_init(self):
         # Correct scheduler driver
         manager = self.manager
@@ -231,6 +238,7 @@ class SchedulerManagerTestCase(test.TestCase):
                                 mox.IgnoreArg())
 
         self.mox.ReplayAll()
+        self.stub_out_client_exceptions()
         self.assertRaises(exception.NoValidHost,
                           self.manager.live_migration,
                           self.context, inst, dest, block_migration,
@@ -263,6 +271,7 @@ class SchedulerManagerTestCase(test.TestCase):
                                 mox.IgnoreArg())
 
         self.mox.ReplayAll()
+        self.stub_out_client_exceptions()
         self.assertRaises(exception.ComputeServiceUnavailable,
                           self.manager.live_migration,
                           self.context, inst, dest, block_migration,
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index dc0af8539..c365c5fdd 100755
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -2968,7 +2968,7 @@ class LibvirtDriver(driver.ComputeDriver):
                        "Disk of instance is too large(available"
                        " on destination host:%(available)s "
                        "< need:%(necessary)s)")
-            raise exception.MigrationError(reason=reason % locals())
+            raise exception.MigrationPreCheckError(reason=reason % locals())
 
     def _compare_cpu(self, cpu_info):
         """Checks the host cpu is compatible to a cpu given by xml.
diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py
index 48413ed7c..e7dd3cff0 100644
--- a/nova/virt/xenapi/vmops.py
+++ b/nova/virt/xenapi/vmops.py
@@ -1625,7 +1625,7 @@ class VMOps(object):
         except KeyError:
             reason = _('Destination host:%(hostname)s must be in the same '
                        'aggregate as the source server')
-            raise exception.MigrationError(reason=reason % locals())
+            raise exception.MigrationPreCheckError(reason=reason % locals())
 
     def _ensure_host_in_aggregate(self, context, hostname):
         self._get_host_uuid_from_aggregate(context, hostname)
@@ -1644,7 +1644,8 @@ class VMOps(object):
         pifs = self._session.call_xenapi('PIF.get_all_records_where',
                                          expr)
         if len(pifs) != 1:
-            raise exception.MigrationError('No suitable network for migrate')
+            msg = _('No suitable network for migrate')
+            raise exception.MigrationPreCheckError(reason=msg)
 
         nwref = pifs[pifs.keys()[0]]['network']
         try:
@@ -1655,7 +1656,8 @@ class VMOps(object):
                                                      options)
         except self._session.XenAPI.Failure as exc:
             LOG.exception(exc)
-            raise exception.MigrationError(_('Migrate Receive failed'))
+            msg = _('Migrate Receive failed')
+            raise exception.MigrationPreCheckError(reason=msg)
         return migrate_data
 
     def check_can_live_migrate_destination(self, ctxt, instance_ref,
@@ -1703,8 +1705,8 @@ class VMOps(object):
                     "VM.assert_can_migrate", vm_ref, migrate_data)
             except self._session.XenAPI.Failure as exc:
                 LOG.exception(exc)
-                raise exception.MigrationError(_('VM.assert_can_migrate'
-                                                 'failed'))
+                msg = _('VM.assert_can_migrate failed')
+                raise exception.MigrationPreCheckError(reason=msg)
         return dest_check_data
 
     def _generate_vdi_map(self, destination_sr_ref, vm_ref):