Reset instance to ACTIVE when no hosts found

bug 928521

modified nova/scheduler/manager.py to reset vm_state to ACTIVE and set
task_state to None when prep_resize raises a NoHostsFound

refactored run_instance and prep_resize so they don't go through
_schedule and now must be implemented in driver

Changed behavior to set vm_state to error on any other exception in
prep_resize.

Change behavior to change instance vm_state to ERROR on exceptions

Added tests that the vm_state gets updated

Added tests that schedule_prep_resize and schedule_run_instance
have no implementation in the Driver base class

Had to adjust methods and tests for Multi scheduler to reflect the
new Scheduler contract

Change-Id: Ibcac7ef0df3456793a2132beb7a711849510da80

3 files changed, 90 insertions, 34 deletions

diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py
index 5ee4ed343..74b5c308b 100644
--- a/nova/scheduler/driver.py
+++ b/nova/scheduler/driver.py
@@ -177,9 +177,19 @@ class Scheduler(object):
         return instance
 
     def schedule(self, context, topic, method, *_args, **_kwargs):
-        """Must override at least this method for scheduler to work."""
+        """Must override schedule method for scheduler to work."""
         raise NotImplementedError(_("Must implement a fallback schedule"))
 
+    def schedule_prep_resize(self, context, request_spec, *_args, **_kwargs):
+        """Must override schedule_prep_resize method for scheduler to work."""
+        msg = _("Driver must implement schedule_prep_resize")
+        raise NotImplementedError(msg)
+
+    def schedule_run_instance(self, context, request_spec, *_args, **_kwargs):
+        """Must override schedule_run_instance method for scheduler to work."""
+        msg = _("Driver must implement schedule_run_instance")
+        raise NotImplementedError(msg)
+
     def schedule_live_migration(self, context, instance_id, dest,
                                 block_migration=False,
                                 disk_over_commit=False):
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index 0390f5ced..57bd445ff 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -75,62 +75,104 @@ class SchedulerManager(manager.Manager):
 
     def _schedule(self, method, context, topic, *args, **kwargs):
         """Tries to call schedule_* method on the driver to retrieve host.
-
         Falls back to schedule(context, topic) if method doesn't exist.
         """
-        driver_method = 'schedule_%s' % method
+        driver_method_name = 'schedule_%s' % method
         try:
-            real_meth = getattr(self.driver, driver_method)
+            driver_method = getattr(self.driver, driver_method_name)
             args = (context,) + args
         except AttributeError, e:
-            LOG.warning(_("Driver Method %(driver_method)s missing: %(e)s."
-                          "Reverting to schedule()") % locals())
-            real_meth = self.driver.schedule
+            LOG.warning(_("Driver Method %(driver_method_name)s missing: "
+                       "%(e)s. Reverting to schedule()") % locals())
+            driver_method = self.driver.schedule
             args = (context, topic, method) + args
 
         # Scheduler methods are responsible for casting.
         try:
-            return real_meth(*args, **kwargs)
+            return driver_method(*args, **kwargs)
+        except Exception as ex:
+            with utils.save_and_reraise_exception():
+                self._set_vm_state_and_notify(method,
+                                             {'vm_state': vm_states.ERROR},
+                                             context, ex, *args, **kwargs)
+
+    def run_instance(self, context, topic, *args, **kwargs):
+        """Tries to call schedule_run_instance on the driver.
+        Sets instance vm_state to ERROR on exceptions
+        """
+        args = (context,) + args
+        try:
+            return self.driver.schedule_run_instance(*args, **kwargs)
+        except exception.NoValidHost as ex:
+            # don't reraise
+            self._set_vm_state_and_notify('run_instance',
+                                         {'vm_state': vm_states.ERROR},
+                                          context, ex, *args, **kwargs)
+        except Exception as ex:
+            with utils.save_and_reraise_exception():
+                self._set_vm_state_and_notify('run_instance',
+                                             {'vm_state': vm_states.ERROR},
+                                             context, ex, *args, **kwargs)
+
+    def prep_resize(self, context, topic, *args, **kwargs):
+        """Tries to call schedule_prep_resize on the driver.
+        Sets instance vm_state to ACTIVE on NoHostFound
+        Sets vm_state to ERROR on other exceptions
+        """
+        args = (context,) + args
+        try:
+            return self.driver.schedule_prep_resize(*args, **kwargs)
         except exception.NoValidHost as ex:
-            self._set_instance_error(method, context, ex, *args, **kwargs)
+            self._set_vm_state_and_notify('prep_resize',
+                                         {'vm_state': vm_states.ACTIVE,
+                                          'task_state': None},
+                                         context, ex, *args, **kwargs)
         except Exception as ex:
             with utils.save_and_reraise_exception():
-                self._set_instance_error(method, context, ex, *args, **kwargs)
+                self._set_vm_state_and_notify('prep_resize',
+                                             {'vm_state': vm_states.ERROR},
+                                             context, ex, *args, **kwargs)
+
+    def _set_vm_state_and_notify(self, method, updates, context, ex,
+                                *args, **kwargs):
+        """changes VM state and notifies"""
+        # FIXME(comstud): Re-factor this somehow. Not sure this belongs in the
+        # scheduler manager like this. We should make this easier.
+        # run_instance only sends a request_spec, and an instance may or may
+        # not have been created in the API (or scheduler) already. If it was
+        # created, there's a 'uuid' set in the instance_properties of the
+        # request_spec.
+        # (littleidea): I refactored this a bit, and I agree
+        # it should be easier :)
+        # The refactoring could go further but trying to minimize changes
+        # for essex timeframe
 
-    def _set_instance_error(self, method, context, ex, *args, **kwargs):
-        """Sets VM to Error state"""
         LOG.warning(_("Failed to schedule_%(method)s: %(ex)s") % locals())
-        # FIXME(comstud): Re-factor this somehow.  Not sure this belongs
-        # in the scheduler manager like this.  Needs to support more than
-        # run_instance
-        if method != "run_instance":
-            return
-        # FIXME(comstud): We should make this easier.  run_instance
-        # only sends a request_spec, and an instance may or may not
-        # have been created in the API (or scheduler) already.  If it
-        # was created, there's a 'uuid' set in the instance_properties
-        # of the request_spec.
+
+        vm_state = updates['vm_state']
         request_spec = kwargs.get('request_spec', {})
         properties = request_spec.get('instance_properties', {})
         instance_uuid = properties.get('uuid', {})
+
         if instance_uuid:
-            LOG.warning(_("Setting instance %(instance_uuid)s to "
-                    "ERROR state.") % locals())
-            db.instance_update(context, instance_uuid,
-                    {'vm_state': vm_states.ERROR})
+            state = vm_state.upper()
+            msg = _("Setting instance %(instance_uuid)s to %(state)s state.")
+            LOG.warning(msg % locals())
+            db.instance_update(context, instance_uuid, updates)
 
         payload = dict(request_spec=request_spec,
                        instance_properties=properties,
                        instance_id=instance_uuid,
-                       state=vm_states.ERROR,
+                       state=vm_state,
                        method=method,
                        reason=ex)
+
         notifier.notify(notifier.publisher_id("scheduler"),
-                        'scheduler.run_instance', notifier.ERROR, payload)
+                        'scheduler.' + method, notifier.ERROR, payload)
 
     # NOTE (masumotok) : This method should be moved to nova.api.ec2.admin.
-    #                    Based on bexar design summit discussion,
-    #                    just put this here for bexar release.
+    # Based on bexar design summit discussion,
+    # just put this here for bexar release.
     def show_host_resources(self, context, host):
         """Shows the physical/usage resource given by hosts.
 
diff --git a/nova/scheduler/multi.py b/nova/scheduler/multi.py
index 3bec060ab..d29370ffd 100644
--- a/nova/scheduler/multi.py
+++ b/nova/scheduler/multi.py
@@ -41,10 +41,8 @@ FLAGS = flags.FLAGS
 FLAGS.register_opts(multi_scheduler_opts)
 
 # A mapping of methods to topics so we can figure out which driver to use.
-_METHOD_MAP = {'run_instance': 'compute',
-               'prep_resize': 'compute',
-               'live_migration': 'compute',
-               'create_volume': 'volume',
+# There are currently no compute methods proxied through the map
+_METHOD_MAP = {'create_volume': 'volume',
                'create_volumes': 'volume'}
 
 
@@ -75,3 +73,9 @@ class MultiScheduler(driver.Scheduler):
     def schedule(self, context, topic, method, *_args, **_kwargs):
         return self.drivers[topic].schedule(context, topic,
                 method, *_args, **_kwargs)
+
+    def schedule_run_instance(self, *args, **kwargs):
+        return self.drivers['compute'].schedule_run_instance(*args, **kwargs)
+
+    def schedule_prep_resize(self, *args, **kwargs):
+        return self.drivers['compute'].schedule_prep_resize(*args, **kwargs)