Merge "Refactor scheduling weights."

3 files changed, 207 insertions, 63 deletions

diff --git a/nova/tests/scheduler/test_filter_scheduler.py b/nova/tests/scheduler/test_filter_scheduler.py
index b8ab45bc2..e9412ba60 100644
--- a/nova/tests/scheduler/test_filter_scheduler.py
+++ b/nova/tests/scheduler/test_filter_scheduler.py
@@ -27,7 +27,7 @@ from nova import exception
 from nova.scheduler import driver
 from nova.scheduler import filter_scheduler
 from nova.scheduler import host_manager
-from nova.scheduler import least_cost
+from nova.scheduler import weights
 from nova.tests.scheduler import fakes
 from nova.tests.scheduler import test_scheduler
 
@@ -145,11 +145,10 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
 
         self.next_weight = 1.0
 
-        def _fake_weighted_sum(functions, hosts, options):
+        def _fake_weigh_objects(_self, functions, hosts, options):
             self.next_weight += 2.0
             host_state = hosts[0]
-            return least_cost.WeightedHost(self.next_weight,
-                    host_state=host_state)
+            return [weights.WeighedHost(host_state, self.next_weight)]
 
         sched = fakes.FakeFilterScheduler()
         fake_context = context.RequestContext('user', 'project',
@@ -157,7 +156,8 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
 
         self.stubs.Set(sched.host_manager, 'get_filtered_hosts',
                 fake_get_filtered_hosts)
-        self.stubs.Set(least_cost, 'weighted_sum', _fake_weighted_sum)
+        self.stubs.Set(weights.HostWeightHandler,
+                'get_weighed_objects', _fake_weigh_objects)
         fakes.mox_host_manager_db_calls(self.mox, fake_context)
 
         request_spec = {'num_instances': 10,
@@ -171,10 +171,10 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
                                                 'vcpus': 1,
                                                 'os_type': 'Linux'}}
         self.mox.ReplayAll()
-        weighted_hosts = sched._schedule(fake_context, request_spec, {})
-        self.assertEquals(len(weighted_hosts), 10)
-        for weighted_host in weighted_hosts:
-            self.assertTrue(weighted_host.host_state is not None)
+        weighed_hosts = sched._schedule(fake_context, request_spec, {})
+        self.assertEquals(len(weighed_hosts), 10)
+        for weighed_host in weighed_hosts:
+            self.assertTrue(weighed_host.obj is not None)
 
     def test_schedule_prep_resize_doesnt_update_host(self):
         fake_context = context.RequestContext('user', 'project',
@@ -184,7 +184,7 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
 
         def _return_hosts(*args, **kwargs):
             host_state = host_manager.HostState('host2', 'node2')
-            return [least_cost.WeightedHost(1.0, host_state=host_state)]
+            return [weights.WeighedHost(host_state, 1.0)]
 
         self.stubs.Set(sched, '_schedule', _return_hosts)
 
@@ -203,19 +203,6 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
                                    instance, {}, None)
         self.assertEqual(info['called'], 0)
 
-    def test_get_cost_functions(self):
-        fixture = fakes.FakeFilterScheduler()
-        fns = fixture.get_cost_functions()
-        self.assertEquals(len(fns), 1)
-        weight, fn = fns[0]
-        self.assertEquals(weight, -1.0)
-        hostinfo = host_manager.HostState('host', 'node')
-        hostinfo.update_from_compute_node(dict(memory_mb=1000,
-                local_gb=0, vcpus=1, disk_available_least=1000,
-                free_disk_mb=1000, free_ram_mb=872, vcpus_used=0,
-                local_gb_used=0, updated_at=None))
-        self.assertEquals(872, fn(hostinfo, {}))
-
     def test_max_attempts(self):
         self.flags(scheduler_max_attempts=4)
 
@@ -332,14 +319,14 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
         reservations = None
 
         host = fakes.FakeHostState('host', 'node', {})
-        weighted_host = least_cost.WeightedHost(1, host)
-        hosts = [weighted_host]
+        weighed_host = weights.WeighedHost(host, 1)
+        weighed_hosts = [weighed_host]
 
         self.mox.StubOutWithMock(sched, '_schedule')
         self.mox.StubOutWithMock(sched.compute_rpcapi, 'prep_resize')
 
-        sched._schedule(self.context, request_spec,
-                filter_properties, [instance['uuid']]).AndReturn(hosts)
+        sched._schedule(self.context, request_spec, filter_properties,
+                [instance['uuid']]).AndReturn(weighed_hosts)
         sched.compute_rpcapi.prep_resize(self.context, image, instance,
                 instance_type, 'host', reservations, request_spec=request_spec,
                 filter_properties=filter_properties)
diff --git a/nova/tests/scheduler/test_least_cost.py b/nova/tests/scheduler/test_least_cost.py
index 1d180d718..f8ed20b43 100644
--- a/nova/tests/scheduler/test_least_cost.py
+++ b/nova/tests/scheduler/test_least_cost.py
@@ -1,4 +1,4 @@
-# Copyright 2011 OpenStack LLC.
+# Copyright 2011-2012 OpenStack LLC.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,27 +15,51 @@
 """
 Tests For Least Cost functions.
 """
+from nova import config
 from nova import context
-from nova.scheduler import host_manager
-from nova.scheduler import least_cost
+from nova.openstack.common import cfg
+from nova.scheduler import weights
+from nova.scheduler.weights import least_cost
 from nova import test
-from nova.tests import matchers
 from nova.tests.scheduler import fakes
 
 
-def offset(hostinfo, options):
+test_least_cost_opts = [
+    cfg.FloatOpt('compute_fake_weigher1_weight',
+             default=2.0,
+             help='How much weight to give the fake_weigher1 function'),
+    cfg.FloatOpt('compute_fake_weigher2_weight',
+             default=1.0,
+             help='How much weight to give the fake_weigher2 function'),
+    ]
+
+CONF = config.CONF
+CONF.import_opt('least_cost_functions', 'nova.scheduler.weights.least_cost')
+CONF.import_opt('compute_fill_first_cost_fn_weight',
+        'nova.scheduler.weights.least_cost')
+CONF.register_opts(test_least_cost_opts)
+
+
+def compute_fake_weigher1(hostinfo, options):
     return hostinfo.free_ram_mb + 10000
 
 
-def scale(hostinfo, options):
+def compute_fake_weigher2(hostinfo, options):
     return hostinfo.free_ram_mb * 2
 
 
 class LeastCostTestCase(test.TestCase):
     def setUp(self):
         super(LeastCostTestCase, self).setUp()
-        self.flags(reserved_host_disk_mb=0, reserved_host_memory_mb=0)
         self.host_manager = fakes.FakeHostManager()
+        self.weight_handler = weights.HostWeightHandler()
+
+    def _get_weighed_host(self, hosts, weight_properties=None):
+        weigher_classes = least_cost.get_least_cost_weighers()
+        if weight_properties is None:
+            weight_properties = {}
+        return self.weight_handler.get_weighed_objects(weigher_classes,
+                hosts, weight_properties)[0]
 
     def _get_all_hosts(self):
         ctxt = context.get_admin_context()
@@ -46,8 +70,39 @@ class LeastCostTestCase(test.TestCase):
         self.mox.ResetAll()
         return host_states
 
-    def test_weighted_sum_happy_day(self):
-        fn_tuples = [(1.0, offset), (1.0, scale)]
+    def test_default_of_spread_first(self):
+        # Default modifier is -1.0, so it turns out that hosts with
+        # the most free memory win
+        hostinfo_list = self._get_all_hosts()
+
+        # host1: free_ram_mb=512
+        # host2: free_ram_mb=1024
+        # host3: free_ram_mb=3072
+        # host4: free_ram_mb=8192
+
+        # so, host1 should win:
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, 8192)
+        self.assertEqual(weighed_host.obj.host, 'host4')
+
+    def test_filling_first(self):
+        self.flags(compute_fill_first_cost_fn_weight=1.0)
+        hostinfo_list = self._get_all_hosts()
+
+        # host1: free_ram_mb=-512
+        # host2: free_ram_mb=-1024
+        # host3: free_ram_mb=-3072
+        # host4: free_ram_mb=-8192
+
+        # so, host1 should win:
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, -512)
+        self.assertEqual(weighed_host.obj.host, 'host1')
+
+    def test_weighted_sum_provided_method(self):
+        fns = ['nova.tests.scheduler.test_least_cost.compute_fake_weigher1',
+               'nova.tests.scheduler.test_least_cost.compute_fake_weigher2']
+        self.flags(least_cost_functions=fns)
         hostinfo_list = self._get_all_hosts()
 
         # host1: free_ram_mb=512
@@ -59,18 +114,17 @@ class LeastCostTestCase(test.TestCase):
         # [10512, 11024, 13072, 18192]
         # [1024,  2048, 6144, 16384]
 
-        # adjusted [ 1.0 * x + 1.0 * y] =
-        # [11536, 13072, 19216, 34576]
+        # adjusted [ 2.0 * x + 1.0 * y] =
+        # [22048, 24096, 32288, 52768]
 
         # so, host1 should win:
-        options = {}
-        weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
-                options)
-        self.assertEqual(weighted_host.weight, 11536)
-        self.assertEqual(weighted_host.host_state.host, 'host1')
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, 52768)
+        self.assertEqual(weighed_host.obj.host, 'host4')
 
     def test_weighted_sum_single_function(self):
-        fn_tuples = [(1.0, offset), ]
+        fns = ['nova.tests.scheduler.test_least_cost.compute_fake_weigher1']
+        self.flags(least_cost_functions=fns)
         hostinfo_list = self._get_all_hosts()
 
         # host1: free_ram_mb=0
@@ -80,24 +134,10 @@ class LeastCostTestCase(test.TestCase):
 
         # [offset, ]=
         # [10512, 11024, 13072, 18192]
+        # adjusted [ 2.0 * x ]=
+        # [21024, 22048, 26144, 36384]
 
         # so, host1 should win:
-        options = {}
-        weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list,
-                                                                    options)
-        self.assertEqual(weighted_host.weight, 10512)
-        self.assertEqual(weighted_host.host_state.host, 'host1')
-
-
-class TestWeightedHost(test.TestCase):
-    def test_dict_conversion_without_host_state(self):
-        host = least_cost.WeightedHost('someweight')
-        expected = {'weight': 'someweight'}
-        self.assertThat(host.to_dict(), matchers.DictMatches(expected))
-
-    def test_dict_conversion_with_host_state(self):
-        host_state = host_manager.HostState('somehost', None)
-        host = least_cost.WeightedHost('someweight', host_state)
-        expected = {'weight': 'someweight',
-                    'host': 'somehost'}
-        self.assertThat(host.to_dict(), matchers.DictMatches(expected))
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, 36384)
+        self.assertEqual(weighed_host.obj.host, 'host4')
diff --git a/nova/tests/scheduler/test_weights.py b/nova/tests/scheduler/test_weights.py
new file mode 100644
index 000000000..8699ed811
--- /dev/null
+++ b/nova/tests/scheduler/test_weights.py
@@ -0,0 +1,117 @@
+# Copyright 2011-2012 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For Scheduler weights.
+"""
+
+from nova import context
+from nova.scheduler import weights
+from nova import test
+from nova.tests import matchers
+from nova.tests.scheduler import fakes
+
+
+class TestWeighedHost(test.TestCase):
+    def test_dict_conversion(self):
+        host_state = fakes.FakeHostState('somehost', None, {})
+        host = weights.WeighedHost(host_state, 'someweight')
+        expected = {'weight': 'someweight',
+                    'host': 'somehost'}
+        self.assertThat(host.to_dict(), matchers.DictMatches(expected))
+
+    def test_all_weighers(self):
+        classes = weights.all_weighers()
+        class_names = [cls.__name__ for cls in classes]
+        self.assertEqual(len(classes), 1)
+        self.assertIn('RAMWeigher', class_names)
+
+    def test_all_weighers_with_deprecated_config1(self):
+        self.flags(compute_fill_first_cost_fn_weight=-1.0)
+        classes = weights.all_weighers()
+        class_names = [cls.__name__ for cls in classes]
+        self.assertEqual(len(classes), 1)
+        self.assertIn('_LeastCostWeigher', class_names)
+
+    def test_all_weighers_with_deprecated_config2(self):
+        self.flags(least_cost_functions=['something'])
+        classes = weights.all_weighers()
+        class_names = [cls.__name__ for cls in classes]
+        self.assertEqual(len(classes), 1)
+        self.assertIn('_LeastCostWeigher', class_names)
+
+
+class RamWeigherTestCase(test.TestCase):
+    def setUp(self):
+        super(RamWeigherTestCase, self).setUp()
+        self.host_manager = fakes.FakeHostManager()
+        self.weight_handler = weights.HostWeightHandler()
+        self.weight_classes = self.weight_handler.get_matching_classes(
+                ['nova.scheduler.weights.ram.RAMWeigher'])
+
+    def _get_weighed_host(self, hosts, weight_properties=None):
+        if weight_properties is None:
+            weight_properties = {}
+        return self.weight_handler.get_weighed_objects(self.weight_classes,
+                hosts, weight_properties)[0]
+
+    def _get_all_hosts(self):
+        ctxt = context.get_admin_context()
+        fakes.mox_host_manager_db_calls(self.mox, ctxt)
+        self.mox.ReplayAll()
+        host_states = self.host_manager.get_all_host_states(ctxt)
+        self.mox.VerifyAll()
+        self.mox.ResetAll()
+        return host_states
+
+    def test_default_of_spreading_first(self):
+        hostinfo_list = self._get_all_hosts()
+
+        # host1: free_ram_mb=512
+        # host2: free_ram_mb=1024
+        # host3: free_ram_mb=3072
+        # host4: free_ram_mb=8192
+
+        # so, host4 should win:
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, 8192)
+        self.assertEqual(weighed_host.obj.host, 'host4')
+
+    def test_ram_filter_multiplier1(self):
+        self.flags(ram_weight_multiplier=-1.0)
+        hostinfo_list = self._get_all_hosts()
+
+        # host1: free_ram_mb=-512
+        # host2: free_ram_mb=-1024
+        # host3: free_ram_mb=-3072
+        # host4: free_ram_mb=-8192
+
+        # so, host1 should win:
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, -512)
+        self.assertEqual(weighed_host.obj.host, 'host1')
+
+    def test_ram_filter_multiplier2(self):
+        self.flags(ram_weight_multiplier=2.0)
+        hostinfo_list = self._get_all_hosts()
+
+        # host1: free_ram_mb=512 * 2
+        # host2: free_ram_mb=1024 * 2
+        # host3: free_ram_mb=3072 * 2
+        # host4: free_ram_mb=8192 * 2
+
+        # so, host4 should win:
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(weighed_host.weight, 8192 * 2)
+        self.assertEqual(weighed_host.obj.host, 'host4')