Merge "Add scheduler retries for prep_resize operations."

3 files changed, 310 insertions, 53 deletions

diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py
index 5d87cc373..9abca178f 100644
--- a/nova/tests/compute/test_compute.py
+++ b/nova/tests/compute/test_compute.py
@@ -21,7 +21,6 @@
 import base64
 import copy
 import datetime
-import functools
 import sys
 import time
 
@@ -88,6 +87,10 @@ class FakeSchedulerAPI(object):
             instance, dest):
         pass
 
+    def prep_resize(self, ctxt, instance, instance_type, image, request_spec,
+            filter_properties, reservations):
+        pass
+
 
 class BaseTestCase(test.TestCase):
 
@@ -5539,28 +5542,36 @@ class DisabledInstanceTypesTestCase(BaseTestCase):
 
 
 class ComputeReschedulingTestCase(BaseTestCase):
-    """Tests related to re-scheduling build requests"""
+    """Tests re-scheduling logic for new build requests"""
 
     def setUp(self):
         super(ComputeReschedulingTestCase, self).setUp()
 
-        self._reschedule = self._reschedule_partial()
+        self.expected_task_state = task_states.SCHEDULING
 
         def fake_update(*args, **kwargs):
             self.updated_task_state = kwargs.get('task_state')
         self.stubs.Set(self.compute, '_instance_update', fake_update)
 
-    def _reschedule_partial(self):
-        uuid = "12-34-56-78-90"
+    def _reschedule(self, request_spec=None, filter_properties=None):
+        if not filter_properties:
+            filter_properties = {}
+
+        instance_uuid = "12-34-56-78-90"
 
-        requested_networks = None
         admin_password = None
         injected_files = None
+        requested_networks = None
         is_first_time = False
 
-        return functools.partial(self.compute._reschedule, self.context, uuid,
-                requested_networks, admin_password, injected_files,
-                is_first_time, request_spec=None, filter_properties={})
+        scheduler_method = self.compute.scheduler_rpcapi.run_instance
+        method_args = (request_spec, admin_password, injected_files,
+                       requested_networks, is_first_time, filter_properties)
+        task_state = task_states.SCHEDULING
+
+        return self.compute._reschedule(self.context, request_spec,
+                filter_properties, instance_uuid, scheduler_method,
+                method_args, self.expected_task_state)
 
     def test_reschedule_no_filter_properties(self):
         """no filter_properties will disable re-scheduling"""
@@ -5584,61 +5595,254 @@ class ComputeReschedulingTestCase(BaseTestCase):
         self.assertTrue(self._reschedule(filter_properties=filter_properties,
             request_spec=request_spec))
         self.assertEqual(1, len(request_spec['instance_uuids']))
-        self.assertEqual(self.updated_task_state, task_states.SCHEDULING)
+        self.assertEqual(self.updated_task_state, self.expected_task_state)
 
 
-class ComputeReschedulingExceptionTestCase(BaseTestCase):
-    """Tests for re-scheduling exception handling logic"""
+class ComputeReschedulingResizeTestCase(ComputeReschedulingTestCase):
+    """Test re-scheduling logic for prep_resize requests"""
 
     def setUp(self):
-        super(ComputeReschedulingExceptionTestCase, self).setUp()
+        super(ComputeReschedulingResizeTestCase, self).setUp()
+        self.expected_task_state = task_states.RESIZE_PREP
 
-        # cause _spawn to raise an exception to test the exception logic:
-        def exploding_spawn(*args, **kwargs):
-            raise test.TestingException()
-        self.stubs.Set(self.compute, '_spawn',
-                exploding_spawn)
+    def _reschedule(self, request_spec=None, filter_properties=None):
+        if not filter_properties:
+            filter_properties = {}
 
-        self.fake_instance = jsonutils.to_primitive(
-                self._create_fake_instance())
-        self.instance_uuid = self.fake_instance['uuid']
+        instance_uuid = "12-34-56-78-90"
 
-    def test_exception_with_rescheduling_disabled(self):
-        """Spawn fails and re-scheduling is disabled."""
-        # this won't be re-scheduled:
-        self.assertRaises(test.TestingException,
-                self.compute._run_instance, self.context,
-                None, {}, None, None, None, None, self.fake_instance)
+        instance = {'uuid': instance_uuid}
+        instance_type = {}
+        image = None
+        reservations = None
+
+        scheduler_method = self.compute.scheduler_rpcapi.prep_resize
+        method_args = (instance, instance_type, image, request_spec,
+                filter_properties, reservations)
+
+        return self.compute._reschedule(self.context, request_spec,
+                filter_properties, instance_uuid, scheduler_method,
+                method_args, self.expected_task_state)
 
-    def test_exception_with_rescheduling_enabled(self):
-        """Spawn fails and re-scheduling is enabled.  Original exception
-        should *not* be re-raised.
+
+class InnerTestingException(Exception):
+    pass
+
+
+class ComputeRescheduleOrReraiseTestCase(BaseTestCase):
+    """Test logic and exception handling around rescheduling or re-raising
+    original exceptions when builds fail.
+    """
+
+    def setUp(self):
+        super(ComputeRescheduleOrReraiseTestCase, self).setUp()
+        self.instance = self._create_fake_instance()
+
+    def test_reschedule_or_reraise_called(self):
+        """Basic sanity check to make sure _reschedule_or_reraise is called
+        when a build fails.
         """
-        # provide the expected status so that this one will be re-scheduled:
-        retry = dict(num_attempts=1)
-        filter_properties = dict(retry=retry)
-        request_spec = dict(num_attempts=1)
-        # Assert that test.TestingException is not raised
-        self.compute._run_instance(self.context, request_spec,
-                                   filter_properties, None, None, None,
-                                   True, self.fake_instance)
-
-    def test_exception_context_cleared(self):
-        """Test with no rescheduling and an additional exception occurs
-        clearing the original build error's exception context.
+        self.mox.StubOutWithMock(self.compute, '_spawn')
+        self.mox.StubOutWithMock(self.compute, '_reschedule_or_reraise')
+
+        self.compute._spawn(mox.IgnoreArg(), self.instance, None, None, None,
+                False, None).AndRaise(test.TestingException("BuildError"))
+        self.compute._reschedule_or_reraise(mox.IgnoreArg(), self.instance,
+                None, None, None, False, None, {})
+
+        self.mox.ReplayAll()
+        self.compute._run_instance(self.context, None, {}, None, None, None,
+                False, self.instance)
+
+    def test_deallocate_network_fail(self):
+        """Test de-allocation of network failing before re-scheduling logic
+        can even run.
         """
-        # clears the original exception context:
-        class FleshWoundException(Exception):
-            pass
+        instance_uuid = self.instance['uuid']
+        self.mox.StubOutWithMock(self.compute, '_deallocate_network')
 
-        def reschedule_explode(*args, **kwargs):
-            raise FleshWoundException()
-        self.stubs.Set(self.compute, '_reschedule', reschedule_explode)
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            exc_info = sys.exc_info()
 
-        # the original exception should now be raised:
-        self.assertRaises(test.TestingException,
-                self.compute._run_instance, self.context,
-                None, {}, None, None, None, None, self.fake_instance)
+            self.compute._deallocate_network(self.context,
+                    self.instance).AndRaise(InnerTestingException("Error"))
+            self.compute._log_original_error(exc_info, instance_uuid)
+
+            self.mox.ReplayAll()
+
+            # should raise the deallocation exception, not the original build
+            # error:
+            self.assertRaises(InnerTestingException,
+                    self.compute._reschedule_or_reraise, self.context,
+                    self.instance, None, None, None, False, None, {})
+
+    def test_reschedule_fail(self):
+        """Test handling of exception from _reschedule"""
+        instance_uuid = self.instance['uuid']
+        method_args = (None, None, None, None, False, {})
+        self.mox.StubOutWithMock(self.compute, '_deallocate_network')
+        self.mox.StubOutWithMock(self.compute, '_reschedule')
+
+        self.compute._deallocate_network(self.context,
+                self.instance)
+        self.compute._reschedule(self.context, None, instance_uuid,
+                {}, self.compute.scheduler_rpcapi.run_instance,
+                method_args, task_states.SCHEDULING).AndRaise(
+                        InnerTestingException("Inner"))
+
+        self.mox.ReplayAll()
+
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            # not re-scheduling, should raise the original build error:
+            self.assertRaises(test.TestingException,
+                    self.compute._reschedule_or_reraise, self.context,
+                    self.instance, None, None, None, False, None, {})
+
+    def test_reschedule_false(self):
+        """Test not-rescheduling, but no nested exception"""
+        instance_uuid = self.instance['uuid']
+        method_args = (None, None, None, None, False, {})
+        self.mox.StubOutWithMock(self.compute, '_deallocate_network')
+        self.mox.StubOutWithMock(self.compute, '_reschedule')
+
+        self.compute._deallocate_network(self.context,
+                self.instance)
+        self.compute._reschedule(self.context, None, instance_uuid,
+                {}, self.compute.scheduler_rpcapi.run_instance, method_args,
+                task_states.SCHEDULING).AndReturn(False)
+
+        self.mox.ReplayAll()
+
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            # re-scheduling is False, the original build error should be
+            # raised here:
+            self.assertRaises(test.TestingException,
+                    self.compute._reschedule_or_reraise, self.context,
+                    self.instance, None, None, None, False, None, {})
+
+    def test_reschedule_true(self):
+        """Test behavior when re-scheduling happens"""
+        instance_uuid = self.instance['uuid']
+        method_args = (None, None, None, None, False, {})
+        self.mox.StubOutWithMock(self.compute, '_deallocate_network')
+        self.mox.StubOutWithMock(self.compute, '_reschedule')
+
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            exc_info = sys.exc_info()
+
+            self.compute._deallocate_network(self.context,
+                    self.instance)
+            self.compute._reschedule(self.context, None, instance_uuid,
+                    {}, self.compute.scheduler_rpcapi.run_instance,
+                    method_args, task_states.SCHEDULING).AndReturn(True)
+            self.compute._log_original_error(exc_info, instance_uuid)
+
+            self.mox.ReplayAll()
+
+            # re-scheduling is True, original error is logged, but nothing
+            # is raised:
+            self.compute._reschedule_or_reraise(self.context, self.instance,
+                    None, None, None, False, None, {})
+
+
+class ComputeRescheduleResizeOrReraiseTestCase(BaseTestCase):
+    """Test logic and exception handling around rescheduling prep resize
+    requests
+    """
+    def setUp(self):
+        super(ComputeRescheduleResizeOrReraiseTestCase, self).setUp()
+        self.instance = self._create_fake_instance()
+        self.instance_uuid = self.instance['uuid']
+        self.instance_type = instance_types.get_instance_type_by_name(
+                "m1.tiny")
+
+    def test_reschedule_resize_or_reraise_called(self):
+        """Verify the rescheduling logic gets called when there is an error
+        during prep_resize.
+        """
+        self.mox.StubOutWithMock(self.compute.db, 'migration_create')
+        self.mox.StubOutWithMock(self.compute, '_reschedule_resize_or_reraise')
+
+        self.compute.db.migration_create(mox.IgnoreArg(),
+                mox.IgnoreArg()).AndRaise(test.TestingException("Original"))
+
+        self.compute._reschedule_resize_or_reraise(mox.IgnoreArg(), None,
+                self.instance, self.instance_type, None, None, None)
+
+        self.mox.ReplayAll()
+
+        self.compute.prep_resize(self.context, None, self.instance,
+                self.instance_type)
+
+    def test_reschedule_fails_with_exception(self):
+        """Original exception should be raised if the _reschedule method
+        raises another exception
+        """
+        method_args = (None, self.instance, self.instance_type, None, None,
+                None)
+        self.mox.StubOutWithMock(self.compute, "_reschedule")
+
+        self.compute._reschedule(self.context, None, None, self.instance_uuid,
+                self.compute.scheduler_rpcapi.prep_resize, method_args,
+                task_states.RESIZE_PREP).AndRaise(
+                        InnerTestingException("Inner"))
+        self.mox.ReplayAll()
+
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            self.assertRaises(test.TestingException,
+                    self.compute._reschedule_resize_or_reraise, self.context,
+                    None, self.instance, self.instance_type, None, {}, {})
+
+    def test_reschedule_false(self):
+        """Original exception should be raised if the resize is not
+        rescheduled.
+        """
+        method_args = (None, self.instance, self.instance_type, None, None,
+                None)
+        self.mox.StubOutWithMock(self.compute, "_reschedule")
+
+        self.compute._reschedule(self.context, None, None, self.instance_uuid,
+                self.compute.scheduler_rpcapi.prep_resize, method_args,
+                task_states.RESIZE_PREP).AndReturn(False)
+        self.mox.ReplayAll()
+
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            self.assertRaises(test.TestingException,
+                    self.compute._reschedule_resize_or_reraise, self.context,
+                    None, self.instance, self.instance_type, None, {}, {})
+
+    def test_reschedule_true(self):
+        """If rescheduled, the original resize exception should be logged"""
+        method_args = (self.instance, self.instance_type, None, {}, {}, None)
+        try:
+            raise test.TestingException("Original")
+        except Exception:
+            exc_info = sys.exc_info()
+
+            self.mox.StubOutWithMock(self.compute, "_reschedule")
+            self.mox.StubOutWithMock(self.compute, "_log_original_error")
+            self.compute._reschedule(self.context, {}, {},
+                    self.instance_uuid,
+                    self.compute.scheduler_rpcapi.prep_resize, method_args,
+                    task_states.RESIZE_PREP).AndReturn(True)
+
+            self.compute._log_original_error(exc_info, self.instance_uuid)
+            self.mox.ReplayAll()
+
+            self.compute._reschedule_resize_or_reraise(self.context, None,
+                    self.instance, self.instance_type, None, {}, {})
 
 
 class ComputeInactiveImageTestCase(BaseTestCase):
diff --git a/nova/tests/compute/test_rpcapi.py b/nova/tests/compute/test_rpcapi.py
index 460fbc24a..7b3d58909 100644
--- a/nova/tests/compute/test_rpcapi.py
+++ b/nova/tests/compute/test_rpcapi.py
@@ -213,7 +213,10 @@ class ComputeRpcAPITestCase(test.TestCase):
         self._test_compute_api('prep_resize', 'cast',
                 instance=self.fake_instance, instance_type='fake_type',
                 image='fake_image', host='host',
-                reservations=list('fake_res'))
+                reservations=list('fake_res'),
+                request_spec='fake_spec',
+                filter_properties={'fakeprop': 'fakeval'},
+                version='2.10')
 
     def test_reboot_instance(self):
         self.maxDiff = None
diff --git a/nova/tests/scheduler/test_filter_scheduler.py b/nova/tests/scheduler/test_filter_scheduler.py
index 394b9b67f..b6bdd359b 100644
--- a/nova/tests/scheduler/test_filter_scheduler.py
+++ b/nova/tests/scheduler/test_filter_scheduler.py
@@ -18,6 +18,7 @@ Tests For Filter Scheduler.
 
 import mox
 
+from nova.compute import instance_types
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova import context
@@ -310,3 +311,52 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
         hosts = filter_properties['retry']['hosts']
         self.assertEqual(1, len(hosts))
         self.assertEqual(host, hosts[0])
+
+    def test_post_select_populate(self):
+        """Test addition of certain filter props after a host is selected"""
+        retry = {'hosts': [], 'num_attempts': 1}
+        filter_properties = {'retry': retry}
+        sched = fakes.FakeFilterScheduler()
+
+        host_state = host_manager.HostState('host', 'compute')
+        host_state.limits['vcpus'] = 5
+        sched._post_select_populate_filter_properties(filter_properties,
+                host_state)
+
+        self.assertEqual('host', filter_properties['retry']['hosts'][0])
+
+        self.assertEqual({'vcpus': 5}, host_state.limits)
+
+    def test_prep_resize_post_populates_retry(self):
+        """Prep resize should add a 'host' entry to the retry dict"""
+        sched = fakes.FakeFilterScheduler()
+
+        image = 'image'
+        instance = db.instance_create(self.context, {})
+
+        instance_properties = {'project_id': 'fake', 'os_type': 'Linux'}
+        instance_type = instance_types.get_instance_type_by_name("m1.tiny")
+        request_spec = {'instance_properties': instance_properties,
+                        'instance_type': instance_type}
+        retry = {'hosts': [], 'num_attempts': 1}
+        filter_properties = {'retry': retry}
+        reservations = None
+
+        host = fakes.FakeHostState('host', 'compute', {})
+        weighted_host = least_cost.WeightedHost(1, host)
+        hosts = [weighted_host]
+
+        self.mox.StubOutWithMock(sched, '_schedule')
+        self.mox.StubOutWithMock(sched.compute_rpcapi, 'prep_resize')
+
+        sched._schedule(self.context, 'compute', request_spec,
+                filter_properties, [instance['uuid']]).AndReturn(hosts)
+        sched.compute_rpcapi.prep_resize(self.context, image, instance,
+                instance_type, 'host', reservations, request_spec=request_spec,
+                filter_properties=filter_properties)
+
+        self.mox.ReplayAll()
+        sched.schedule_prep_resize(self.context, image, request_spec,
+                filter_properties, instance, instance_type, reservations)
+
+        self.assertEqual(['host'], filter_properties['retry']['hosts'])