Reset instance to ACTIVE when no hosts found

bug 928521

modified nova/scheduler/manager.py to reset vm_state to ACTIVE and set
task_state to None when prep_resize raises a NoHostsFound

refactored run_instance and prep_resize so they don't go through
_schedule and now must be implemented in driver

Changed behavior to set vm_state to error on any other exception in
prep_resize.

Change behavior to change instance vm_state to ERROR on exceptions

Added tests that the vm_state gets updated

Added tests that schedule_prep_resize and schedule_run_instance
have no implementation in the Driver base class

Had to adjust methods and tests for Multi scheduler to reflect the
new Scheduler contract

Change-Id: Ibcac7ef0df3456793a2132beb7a711849510da80

2 files changed, 136 insertions, 17 deletions

diff --git a/nova/tests/scheduler/test_multi_scheduler.py b/nova/tests/scheduler/test_multi_scheduler.py
index 7468a2447..3d7220c2d 100644
--- a/nova/tests/scheduler/test_multi_scheduler.py
+++ b/nova/tests/scheduler/test_multi_scheduler.py
@@ -28,13 +28,7 @@ from nova.tests.scheduler import test_scheduler
 class FakeComputeScheduler(driver.Scheduler):
     is_fake_compute = True
 
-    def schedule_run_instance(self, *args, **kwargs):
-        pass
-
-    def schedule_live_migration(self, *args, **kwargs):
-        pass
-
-    def schedule_prep_resize(self, *args, **kwargs):
+    def schedule_theoretical(self, *args, **kwargs):
         pass
 
     def schedule(self, *args, **kwargs):
@@ -79,7 +73,8 @@ class MultiDriverTestCase(test_scheduler.SchedulerTestCase):
         compute_driver = mgr.drivers['compute']
         volume_driver = mgr.drivers['volume']
 
-        test_methods = {compute_driver: ['run_instance', 'prep_resize'],
+        #no compute methods are proxied at this time
+        test_methods = {compute_driver: [],
                         volume_driver: ['create_volume', 'create_volumes']}
 
         for driver, methods in test_methods.iteritems():
diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py
index 67f607b07..ccf9d38d9 100644
--- a/nova/tests/scheduler/test_scheduler.py
+++ b/nova/tests/scheduler/test_scheduler.py
@@ -29,6 +29,7 @@ from nova import context
 from nova import db
 from nova import exception
 from nova import flags
+from nova.notifier import api as notifier
 from nova import rpc
 from nova.rpc import common as rpc_common
 from nova.scheduler import driver
@@ -47,6 +48,9 @@ class SchedulerManagerTestCase(test.TestCase):
     driver_cls = driver.Scheduler
     driver_cls_name = 'nova.scheduler.driver.Scheduler'
 
+    class AnException(Exception):
+        pass
+
     def setUp(self):
         super(SchedulerManagerTestCase, self).setUp()
         self.flags(scheduler_driver=self.driver_cls_name)
@@ -183,21 +187,55 @@ class SchedulerManagerTestCase(test.TestCase):
                                  'memory_mb_used': 512}}
         self.assertDictMatch(result, expected)
 
-    def test_run_instance_exception_puts_instance_in_error_state(self):
-        """Test that a NoValidHost exception for run_instance puts
-        the instance in ERROR state and eats the exception.
-        """
-
-        fake_instance_uuid = 'fake-instance-id'
-
+    def _mox_schedule_method_helper(self, method_name):
         # Make sure the method exists that we're going to test call
         def stub_method(*args, **kwargs):
             pass
 
-        setattr(self.manager.driver, 'schedule_run_instance', stub_method)
+        setattr(self.manager.driver, method_name, stub_method)
 
         self.mox.StubOutWithMock(self.manager.driver,
-                'schedule_run_instance')
+                method_name)
+
+    def test_schedule_exeception_changes_state_notifies_and_raises(self):
+        """Test that an exception scheduling calls
+        _set_vm_state_and_notify and reraises
+        """
+        fake_instance_uuid = 'fake-instance-id'
+
+        self._mox_schedule_method_helper('schedule_something')
+
+        self.mox.StubOutWithMock(self.manager, '_set_vm_state_and_notify')
+
+        request_spec = {'instance_properties':
+                {'uuid': fake_instance_uuid}}
+        self.fake_kwargs['request_spec'] = request_spec
+
+        ex = self.AnException('something happened')
+        self.manager.driver.schedule_something(self.context,
+                *self.fake_args, **self.fake_kwargs).AndRaise(ex)
+
+        # Adding the context to the args is kind of gnarly, but thats what
+        # happens. Could be refactored to keep all the context, spec, topic
+        # stuff a bit cleaner.
+        self.manager._set_vm_state_and_notify('something',
+                {'vm_state': vm_states.ERROR}, self.context,
+                ex, *((self.context,) + self.fake_args), **self.fake_kwargs)
+
+        self.mox.ReplayAll()
+
+        self.assertRaises(self.AnException, self.manager.something,
+                         self.context, self.topic,
+                         *self.fake_args, **self.fake_kwargs)
+
+    def test_run_instance_exception_puts_instance_in_error_state(self):
+        """Test that an NoValidHost exception for run_instance puts
+        the instance in ERROR state and eats the exception.
+        """
+
+        fake_instance_uuid = 'fake-instance-id'
+
+        self._mox_schedule_method_helper('schedule_run_instance')
         self.mox.StubOutWithMock(db, 'instance_update')
 
         request_spec = {'instance_properties':
@@ -214,6 +252,57 @@ class SchedulerManagerTestCase(test.TestCase):
         self.manager.run_instance(self.context, self.topic,
                 *self.fake_args, **self.fake_kwargs)
 
+    def test_prep_resize_no_valid_host_back_in_active_state(self):
+        """Test that a NoValidHost exception for prep_resize puts
+        the instance in ACTIVE state
+        """
+        fake_instance_uuid = 'fake-instance-id'
+
+        self._mox_schedule_method_helper('schedule_prep_resize')
+
+        self.mox.StubOutWithMock(db, 'instance_update')
+
+        request_spec = {'instance_properties':
+                {'uuid': fake_instance_uuid}}
+        self.fake_kwargs['request_spec'] = request_spec
+
+        self.manager.driver.schedule_prep_resize(self.context,
+                *self.fake_args, **self.fake_kwargs).AndRaise(
+                        exception.NoValidHost(reason=""))
+        db.instance_update(self.context, fake_instance_uuid,
+                {'vm_state': vm_states.ACTIVE,
+                 'task_state': None})
+
+        self.mox.ReplayAll()
+        self.manager.prep_resize(self.context, self.topic,
+                *self.fake_args, **self.fake_kwargs)
+
+    def test_prep_resize_exception_host_in_error_state_and_raise(self):
+        """Test that a NoValidHost exception for prep_resize puts
+        the instance in ACTIVE state
+        """
+        fake_instance_uuid = 'fake-instance-id'
+
+        self._mox_schedule_method_helper('schedule_prep_resize')
+
+        self.mox.StubOutWithMock(db, 'instance_update')
+
+        request_spec = {'instance_properties':
+                {'uuid': fake_instance_uuid}}
+        self.fake_kwargs['request_spec'] = request_spec
+
+        self.manager.driver.schedule_prep_resize(self.context,
+                *self.fake_args, **self.fake_kwargs).AndRaise(
+                self.AnException('something happened'))
+        db.instance_update(self.context, fake_instance_uuid,
+                {'vm_state': vm_states.ERROR})
+
+        self.mox.ReplayAll()
+
+        self.assertRaises(self.AnException, self.manager.prep_resize,
+                         self.context, self.topic,
+                         *self.fake_args, **self.fake_kwargs)
+
 
 class SchedulerTestCase(test.TestCase):
     """Test case for base scheduler driver class"""
@@ -916,6 +1005,41 @@ class SchedulerTestCase(test.TestCase):
                 block_migration=block_migration)
 
 
+class SchedulerDriverBaseTestCase(SchedulerTestCase):
+    """Test cases for base scheduler driver class methods
+       that can't will fail if the driver is changed"""
+
+    def test_unimplemented_schedule(self):
+        fake_args = (1, 2, 3)
+        fake_kwargs = {'cat': 'meow'}
+
+        self.assertRaises(NotImplementedError, self.driver.schedule,
+                         self.context, self.topic, 'schedule_something',
+                         *fake_args, **fake_kwargs)
+
+    def test_unimplemented_schedule_run_instance(self):
+        fake_args = (1, 2, 3)
+        fake_kwargs = {'cat': 'meow'}
+        fake_request_spec = {'instance_properties':
+                {'uuid': 'uuid'}}
+
+        self.assertRaises(NotImplementedError,
+                         self.driver.schedule_run_instance,
+                         self.context, fake_request_spec,
+                         *fake_args, **fake_kwargs)
+
+    def test_unimplemented_schedule_prep_resize(self):
+        fake_args = (1, 2, 3)
+        fake_kwargs = {'cat': 'meow'}
+        fake_request_spec = {'instance_properties':
+                {'uuid': 'uuid'}}
+
+        self.assertRaises(NotImplementedError,
+                         self.driver.schedule_prep_resize,
+                         self.context, fake_request_spec,
+                         *fake_args, **fake_kwargs)
+
+
 class SchedulerDriverModuleTestCase(test.TestCase):
     """Test case for scheduler driver module methods"""