diff options
Diffstat (limited to 'nova')
| -rw-r--r-- | nova/scheduler/filter_scheduler.py | 94 | ||||
| -rw-r--r-- | nova/scheduler/host_manager.py | 12 | ||||
| -rw-r--r-- | nova/scheduler/least_cost.py | 118 | ||||
| -rw-r--r-- | nova/scheduler/weights/__init__.py | 61 | ||||
| -rw-r--r-- | nova/scheduler/weights/least_cost.py | 126 | ||||
| -rw-r--r-- | nova/scheduler/weights/ram.py | 46 | ||||
| -rw-r--r-- | nova/tests/scheduler/test_filter_scheduler.py | 41 | ||||
| -rw-r--r-- | nova/tests/scheduler/test_least_cost.py | 112 | ||||
| -rw-r--r-- | nova/tests/scheduler/test_weights.py | 117 | ||||
| -rw-r--r-- | nova/weights.py | 71 |
10 files changed, 543 insertions, 255 deletions
diff --git a/nova/scheduler/filter_scheduler.py b/nova/scheduler/filter_scheduler.py index 10dc7e37c..636818e59 100644 --- a/nova/scheduler/filter_scheduler.py +++ b/nova/scheduler/filter_scheduler.py @@ -19,16 +19,11 @@ You can customize this scheduler by specifying your own Host Filters and Weighing Functions. """ -import operator - from nova import config from nova import exception -from nova import flags -from nova.openstack.common import importutils from nova.openstack.common import log as logging from nova.openstack.common.notifier import api as notifier from nova.scheduler import driver -from nova.scheduler import least_cost from nova.scheduler import scheduler_options CONF = config.CONF @@ -61,7 +56,7 @@ class FilterScheduler(driver.Scheduler): notifier.notify(context, notifier.publisher_id("scheduler"), 'scheduler.run_instance.start', notifier.INFO, payload) - weighted_hosts = self._schedule(context, request_spec, + weighed_hosts = self._schedule(context, request_spec, filter_properties, instance_uuids) # NOTE(comstud): Make sure we do not pass this through. It @@ -73,11 +68,11 @@ class FilterScheduler(driver.Scheduler): try: try: - weighted_host = weighted_hosts.pop(0) + weighed_host = weighed_hosts.pop(0) except IndexError: raise exception.NoValidHost(reason="") - self._provision_resource(context, weighted_host, + self._provision_resource(context, weighed_host, request_spec, filter_properties, requested_networks, @@ -107,29 +102,29 @@ class FilterScheduler(driver.Scheduler): the prep_resize operation to it. """ - hosts = self._schedule(context, request_spec, filter_properties, - [instance['uuid']]) - if not hosts: + weighed_hosts = self._schedule(context, request_spec, + filter_properties, [instance['uuid']]) + if not weighed_hosts: raise exception.NoValidHost(reason="") - host = hosts.pop(0) + weighed_host = weighed_hosts.pop(0) self._post_select_populate_filter_properties(filter_properties, - host.host_state) + weighed_host.obj) # context is not serializable filter_properties.pop('context', None) # Forward off to the host self.compute_rpcapi.prep_resize(context, image, instance, - instance_type, host.host_state.host, reservations, + instance_type, weighed_host.obj.host, reservations, request_spec=request_spec, filter_properties=filter_properties) - def _provision_resource(self, context, weighted_host, request_spec, + def _provision_resource(self, context, weighed_host, request_spec, filter_properties, requested_networks, injected_files, admin_password, is_first_time, instance_uuid=None): """Create the requested resource in this Zone.""" payload = dict(request_spec=request_spec, - weighted_host=weighted_host.to_dict(), + weighted_host=weighed_host.to_dict(), instance_id=instance_uuid) notifier.notify(context, notifier.publisher_id("scheduler"), 'scheduler.run_instance.scheduled', notifier.INFO, @@ -137,15 +132,15 @@ class FilterScheduler(driver.Scheduler): # TODO(NTTdocomo): Combine the next two updates into one driver.db_instance_node_set(context, - instance_uuid, weighted_host.host_state.nodename) + instance_uuid, weighed_host.obj.nodename) updated_instance = driver.instance_update_db(context, instance_uuid) self._post_select_populate_filter_properties(filter_properties, - weighted_host.host_state) + weighed_host.obj) self.compute_rpcapi.run_instance(context, instance=updated_instance, - host=weighted_host.host_state.host, + host=weighed_host.obj.host, request_spec=request_spec, filter_properties=filter_properties, requested_networks=requested_networks, injected_files=injected_files, @@ -232,7 +227,6 @@ class FilterScheduler(driver.Scheduler): instance_properties = request_spec['instance_properties'] instance_type = request_spec.get("instance_type", None) - cost_functions = self.get_cost_functions() config_options = self._get_configuration_options() # check retry policy. Rather ugly use of instance_uuids[0]... @@ -276,60 +270,12 @@ class FilterScheduler(driver.Scheduler): LOG.debug(_("Filtered %(hosts)s") % locals()) - # weighted_host = WeightedHost() ... the best - # host for the job. - # TODO(comstud): filter_properties will also be used for - # weighing and I plan fold weighing into the host manager - # in a future patch. I'll address the naming of this - # variable at that time. - weighted_host = least_cost.weighted_sum(cost_functions, - hosts, filter_properties) - LOG.debug(_("Weighted %(weighted_host)s") % locals()) - selected_hosts.append(weighted_host) - + weighed_hosts = self.host_manager.get_weighed_hosts(hosts, + filter_properties) + best_host = weighed_hosts[0] + LOG.debug(_("Choosing host %(best_host)s") % locals()) + selected_hosts.append(best_host) # Now consume the resources so the filter/weights # will change for the next instance. - weighted_host.host_state.consume_from_instance( - instance_properties) - - selected_hosts.sort(key=operator.attrgetter('weight')) + best_host.obj.consume_from_instance(instance_properties) return selected_hosts - - def get_cost_functions(self): - """Returns a list of tuples containing weights and cost functions to - use for weighing hosts - """ - if self.cost_function_cache is not None: - return self.cost_function_cache - - cost_fns = [] - for cost_fn_str in CONF.least_cost_functions: - if '.' in cost_fn_str: - short_name = cost_fn_str.split('.')[-1] - else: - short_name = cost_fn_str - cost_fn_str = "%s.%s.%s" % ( - __name__, self.__class__.__name__, short_name) - if not (short_name.startswith('compute_') or - short_name.startswith('noop')): - continue - - try: - # NOTE: import_class is somewhat misnamed since - # the weighing function can be any non-class callable - # (i.e., no 'self') - cost_fn = importutils.import_class(cost_fn_str) - except ImportError: - raise exception.SchedulerCostFunctionNotFound( - cost_fn_str=cost_fn_str) - - try: - flag_name = "%s_weight" % cost_fn.__name__ - weight = getattr(CONF, flag_name) - except AttributeError: - raise exception.SchedulerWeightFlagNotFound( - flag_name=flag_name) - cost_fns.append((weight, cost_fn)) - - self.cost_function_cache = cost_fns - return cost_fns diff --git a/nova/scheduler/host_manager.py b/nova/scheduler/host_manager.py index e93b529f9..ba4fa3d34 100644 --- a/nova/scheduler/host_manager.py +++ b/nova/scheduler/host_manager.py @@ -28,6 +28,7 @@ from nova.openstack.common import cfg from nova.openstack.common import log as logging from nova.openstack.common import timeutils from nova.scheduler import filters +from nova.scheduler import weights host_manager_opts = [ cfg.MultiStrOpt('scheduler_available_filters', @@ -47,6 +48,9 @@ host_manager_opts = [ ], help='Which filter class names to use for filtering hosts ' 'when not specified in the request.'), + cfg.ListOpt('scheduler_weight_classes', + default=['nova.scheduler.weights.all_weighers'], + help='Which weight class names to use for weighing hosts'), ] CONF = config.CONF @@ -258,6 +262,9 @@ class HostManager(object): self.filter_handler = filters.HostFilterHandler() self.filter_classes = self.filter_handler.get_matching_classes( CONF.scheduler_available_filters) + self.weight_handler = weights.HostWeightHandler() + self.weight_classes = self.weight_handler.get_matching_classes( + CONF.scheduler_weight_classes) def _choose_host_filters(self, filter_cls_names): """Since the caller may specify which filters to use we need @@ -316,6 +323,11 @@ class HostManager(object): return self.filter_handler.get_filtered_objects(filter_classes, hosts, filter_properties) + def get_weighed_hosts(self, hosts, weight_properties): + """Weigh the hosts""" + return self.weight_handler.get_weighed_objects(self.weight_classes, + hosts, weight_properties) + def update_service_capabilities(self, service_name, host, capabilities): """Update the per-service capabilities based on this notification.""" diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py deleted file mode 100644 index d3eaee735..000000000 --- a/nova/scheduler/least_cost.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (c) 2011 OpenStack, LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Least Cost is an algorithm for choosing which host machines to -provision a set of resources to. The input is a WeightedHost object which -is decided upon by a set of objective-functions, called the 'cost-functions'. -The WeightedHost contains a combined weight for each cost-function. - -The cost-function and weights are tabulated, and the host with the least cost -is then selected for provisioning. -""" - -from nova import config -from nova import flags -from nova.openstack.common import cfg -from nova.openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - -least_cost_opts = [ - cfg.ListOpt('least_cost_functions', - default=[ - 'nova.scheduler.least_cost.compute_fill_first_cost_fn' - ], - help='Which cost functions the LeastCostScheduler should use'), - cfg.FloatOpt('noop_cost_fn_weight', - default=1.0, - help='How much weight to give the noop cost function'), - cfg.FloatOpt('compute_fill_first_cost_fn_weight', - default=-1.0, - help='How much weight to give the fill-first cost function. ' - 'A negative value will reverse behavior: ' - 'e.g. spread-first'), - ] - -CONF = config.CONF -CONF.register_opts(least_cost_opts) - -# TODO(sirp): Once we have enough of these rules, we can break them out into a -# cost_functions.py file (perhaps in a least_cost_scheduler directory) - - -class WeightedHost(object): - """Reduced set of information about a host that has been weighed. - This is an attempt to remove some of the ad-hoc dict structures - previously used.""" - - def __init__(self, weight, host_state=None): - self.weight = weight - self.host_state = host_state - - def to_dict(self): - x = dict(weight=self.weight) - if self.host_state: - x['host'] = self.host_state.host - return x - - def __repr__(self): - if self.host_state: - return "WeightedHost host: %s" % self.host_state.host - return "WeightedHost with no host_state" - - -def noop_cost_fn(host_state, weighing_properties): - """Return a pre-weight cost of 1 for each host""" - return 1 - - -def compute_fill_first_cost_fn(host_state, weighing_properties): - """More free ram = higher weight. So servers with less free - ram will be preferred. - - Note: the weight for this function in default configuration - is -1.0. With a -1.0 this function runs in reverse, so systems - with the most free memory will be preferred. - """ - return host_state.free_ram_mb - - -def weighted_sum(weighted_fns, host_states, weighing_properties): - """Use the weighted-sum method to compute a score for an array of objects. - - Normalize the results of the objective-functions so that the weights are - meaningful regardless of objective-function's range. - - :param host_list: ``[(host, HostInfo()), ...]`` - :param weighted_fns: list of weights and functions like:: - - [(weight, objective-functions), ...] - - :param weighing_properties: an arbitrary dict of values that can - influence weights. - - :returns: a single WeightedHost object which represents the best - candidate. - """ - - min_score, best_host = None, None - for host_state in host_states: - score = sum(weight * fn(host_state, weighing_properties) - for weight, fn in weighted_fns) - if min_score is None or score < min_score: - min_score, best_host = score, host_state - - return WeightedHost(min_score, host_state=best_host) diff --git a/nova/scheduler/weights/__init__.py b/nova/scheduler/weights/__init__.py new file mode 100644 index 000000000..55c44b528 --- /dev/null +++ b/nova/scheduler/weights/__init__.py @@ -0,0 +1,61 @@ +# Copyright (c) 2011 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Scheduler host weights +""" + + +from nova import config +from nova.openstack.common import log as logging +from nova.scheduler.weights import least_cost +from nova import weights + +LOG = logging.getLogger(__name__) +CONF = config.CONF + + +class WeighedHost(weights.WeighedObject): + def to_dict(self): + x = dict(weight=self.weight) + x['host'] = self.obj.host + return x + + def __repr__(self): + return "WeighedHost [host: %s, weight: %s]" % ( + self.obj.host, self.weight) + + +class BaseHostWeigher(weights.BaseWeigher): + """Base class for host weights.""" + pass + + +class HostWeightHandler(weights.BaseWeightHandler): + object_class = WeighedHost + + def __init__(self): + super(HostWeightHandler, self).__init__(BaseHostWeigher) + + +def all_weighers(): + """Return a list of weight plugin classes found in this directory.""" + + if (CONF.least_cost_functions is not None or + CONF.compute_fill_first_cost_fn_weight is not None): + LOG.deprecated(_('least_cost has been deprecated in favor of ' + 'the RAM Weigher.')) + return least_cost.get_least_cost_weighers() + return HostWeightHandler().get_all_classes() diff --git a/nova/scheduler/weights/least_cost.py b/nova/scheduler/weights/least_cost.py new file mode 100644 index 000000000..2d886f461 --- /dev/null +++ b/nova/scheduler/weights/least_cost.py @@ -0,0 +1,126 @@ +# Copyright (c) 2011-2012 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Least Cost is an algorithm for choosing which host machines to +provision a set of resources to. The input is a WeightedHost object which +is decided upon by a set of objective-functions, called the 'cost-functions'. +The WeightedHost contains a combined weight for each cost-function. + +The cost-function and weights are tabulated, and the host with the least cost +is then selected for provisioning. + +NOTE(comstud): This is deprecated. One should use the RAMWeigher and/or +create other weight modules. +""" + +from nova import config +from nova import exception +from nova.openstack.common import cfg +from nova.openstack.common import importutils +from nova.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + +least_cost_opts = [ + cfg.ListOpt('least_cost_functions', + default=None, + help='Which cost functions the LeastCostScheduler should use'), + cfg.FloatOpt('noop_cost_fn_weight', + default=1.0, + help='How much weight to give the noop cost function'), + cfg.FloatOpt('compute_fill_first_cost_fn_weight', + default=None, + help='How much weight to give the fill-first cost function. ' + 'A negative value will reverse behavior: ' + 'e.g. spread-first'), + ] + +CONF = config.CONF +CONF.register_opts(least_cost_opts) + + +def noop_cost_fn(host_state, weight_properties): + """Return a pre-weight cost of 1 for each host""" + return 1 + + +def compute_fill_first_cost_fn(host_state, weight_properties): + """Higher weights win, so we should return a lower weight + when there's more free ram available. + + Note: the weight modifier for this function in default configuration + is -1.0. With -1.0 this function runs in reverse, so systems + with the most free memory will be preferred. + """ + return -host_state.free_ram_mb + + +def _get_cost_functions(): + """Returns a list of tuples containing weights and cost functions to + use for weighing hosts + """ + cost_fns_conf = CONF.least_cost_functions + if cost_fns_conf is None: + # The old default. This will get fixed up below. + fn_str = 'nova.scheduler.least_cost.compute_fill_first_cost_fn' + cost_fns_conf = [fn_str] + cost_fns = [] + for cost_fn_str in cost_fns_conf: + short_name = cost_fn_str.split('.')[-1] + if not (short_name.startswith('compute_') or + short_name.startswith('noop')): + continue + # Fix up any old paths to the new paths + if cost_fn_str.startswith('nova.scheduler.least_cost.'): + cost_fn_str = ('nova.scheduler.weights.least_cost' + + cost_fn_str[25:]) + try: + # NOTE: import_class is somewhat misnamed since + # the weighing function can be any non-class callable + # (i.e., no 'self') + cost_fn = importutils.import_class(cost_fn_str) + except ImportError: + raise exception.SchedulerCostFunctionNotFound( + cost_fn_str=cost_fn_str) + + try: + flag_name = "%s_weight" % cost_fn.__name__ + weight = getattr(CONF, flag_name) + except AttributeError: + raise exception.SchedulerWeightFlagNotFound( + flag_name=flag_name) + # Set the original default. + if (flag_name == 'compute_fill_first_cost_fn_weight' and + weight is None): + weight = -1.0 + cost_fns.append((weight, cost_fn)) + return cost_fns + + +def get_least_cost_weighers(): + cost_functions = _get_cost_functions() + + # Unfortunately we need to import this late so we don't have an + # import loop. + from nova.scheduler import weights + + class _LeastCostWeigher(weights.BaseHostWeigher): + def weigh_objects(self, weighted_hosts, weight_properties): + for host in weighted_hosts: + host.weight = sum(weight * fn(host.obj, weight_properties) + for weight, fn in cost_functions) + + return [_LeastCostWeigher] diff --git a/nova/scheduler/weights/ram.py b/nova/scheduler/weights/ram.py new file mode 100644 index 000000000..0fe1911c4 --- /dev/null +++ b/nova/scheduler/weights/ram.py @@ -0,0 +1,46 @@ +# Copyright (c) 2011 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +RAM Weigher. Weigh hosts by their RAM usage. + +The default is to spread instances across all hosts evenly. If you prefer +stacking, you can set the 'ram_weight_multiplier' option to a negative +number and the weighing has the opposite effect of the default. +""" + +from nova import config +from nova.openstack.common import cfg +from nova.scheduler import weights + + +ram_weight_opts = [ + cfg.FloatOpt('ram_weight_multiplier', + default=1.0, + help='Multiplier used for weighing ram. Negative ' + 'numbers mean to stack vs spread.'), +] + +CONF = config.CONF +CONF.register_opts(ram_weight_opts) + + +class RAMWeigher(weights.BaseHostWeigher): + def _weight_multiplier(self): + """Override the weight multiplier.""" + return CONF.ram_weight_multiplier + + def _weigh_object(self, host_state, weight_properties): + """Higher weights win. We want spreading to be the default.""" + return host_state.free_ram_mb diff --git a/nova/tests/scheduler/test_filter_scheduler.py b/nova/tests/scheduler/test_filter_scheduler.py index b8ab45bc2..e9412ba60 100644 --- a/nova/tests/scheduler/test_filter_scheduler.py +++ b/nova/tests/scheduler/test_filter_scheduler.py @@ -27,7 +27,7 @@ from nova import exception from nova.scheduler import driver from nova.scheduler import filter_scheduler from nova.scheduler import host_manager -from nova.scheduler import least_cost +from nova.scheduler import weights from nova.tests.scheduler import fakes from nova.tests.scheduler import test_scheduler @@ -145,11 +145,10 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase): self.next_weight = 1.0 - def _fake_weighted_sum(functions, hosts, options): + def _fake_weigh_objects(_self, functions, hosts, options): self.next_weight += 2.0 host_state = hosts[0] - return least_cost.WeightedHost(self.next_weight, - host_state=host_state) + return [weights.WeighedHost(host_state, self.next_weight)] sched = fakes.FakeFilterScheduler() fake_context = context.RequestContext('user', 'project', @@ -157,7 +156,8 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase): self.stubs.Set(sched.host_manager, 'get_filtered_hosts', fake_get_filtered_hosts) - self.stubs.Set(least_cost, 'weighted_sum', _fake_weighted_sum) + self.stubs.Set(weights.HostWeightHandler, + 'get_weighed_objects', _fake_weigh_objects) fakes.mox_host_manager_db_calls(self.mox, fake_context) request_spec = {'num_instances': 10, @@ -171,10 +171,10 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase): 'vcpus': 1, 'os_type': 'Linux'}} self.mox.ReplayAll() - weighted_hosts = sched._schedule(fake_context, request_spec, {}) - self.assertEquals(len(weighted_hosts), 10) - for weighted_host in weighted_hosts: - self.assertTrue(weighted_host.host_state is not None) + weighed_hosts = sched._schedule(fake_context, request_spec, {}) + self.assertEquals(len(weighed_hosts), 10) + for weighed_host in weighed_hosts: + self.assertTrue(weighed_host.obj is not None) def test_schedule_prep_resize_doesnt_update_host(self): fake_context = context.RequestContext('user', 'project', @@ -184,7 +184,7 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase): def _return_hosts(*args, **kwargs): host_state = host_manager.HostState('host2', 'node2') - return [least_cost.WeightedHost(1.0, host_state=host_state)] + return [weights.WeighedHost(host_state, 1.0)] self.stubs.Set(sched, '_schedule', _return_hosts) @@ -203,19 +203,6 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase): instance, {}, None) self.assertEqual(info['called'], 0) - def test_get_cost_functions(self): - fixture = fakes.FakeFilterScheduler() - fns = fixture.get_cost_functions() - self.assertEquals(len(fns), 1) - weight, fn = fns[0] - self.assertEquals(weight, -1.0) - hostinfo = host_manager.HostState('host', 'node') - hostinfo.update_from_compute_node(dict(memory_mb=1000, - local_gb=0, vcpus=1, disk_available_least=1000, - free_disk_mb=1000, free_ram_mb=872, vcpus_used=0, - local_gb_used=0, updated_at=None)) - self.assertEquals(872, fn(hostinfo, {})) - def test_max_attempts(self): self.flags(scheduler_max_attempts=4) @@ -332,14 +319,14 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase): reservations = None host = fakes.FakeHostState('host', 'node', {}) - weighted_host = least_cost.WeightedHost(1, host) - hosts = [weighted_host] + weighed_host = weights.WeighedHost(host, 1) + weighed_hosts = [weighed_host] self.mox.StubOutWithMock(sched, '_schedule') self.mox.StubOutWithMock(sched.compute_rpcapi, 'prep_resize') - sched._schedule(self.context, request_spec, - filter_properties, [instance['uuid']]).AndReturn(hosts) + sched._schedule(self.context, request_spec, filter_properties, + [instance['uuid']]).AndReturn(weighed_hosts) sched.compute_rpcapi.prep_resize(self.context, image, instance, instance_type, 'host', reservations, request_spec=request_spec, filter_properties=filter_properties) diff --git a/nova/tests/scheduler/test_least_cost.py b/nova/tests/scheduler/test_least_cost.py index 1d180d718..f8ed20b43 100644 --- a/nova/tests/scheduler/test_least_cost.py +++ b/nova/tests/scheduler/test_least_cost.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011-2012 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,27 +15,51 @@ """ Tests For Least Cost functions. """ +from nova import config from nova import context -from nova.scheduler import host_manager -from nova.scheduler import least_cost +from nova.openstack.common import cfg +from nova.scheduler import weights +from nova.scheduler.weights import least_cost from nova import test -from nova.tests import matchers from nova.tests.scheduler import fakes -def offset(hostinfo, options): +test_least_cost_opts = [ + cfg.FloatOpt('compute_fake_weigher1_weight', + default=2.0, + help='How much weight to give the fake_weigher1 function'), + cfg.FloatOpt('compute_fake_weigher2_weight', + default=1.0, + help='How much weight to give the fake_weigher2 function'), + ] + +CONF = config.CONF +CONF.import_opt('least_cost_functions', 'nova.scheduler.weights.least_cost') +CONF.import_opt('compute_fill_first_cost_fn_weight', + 'nova.scheduler.weights.least_cost') +CONF.register_opts(test_least_cost_opts) + + +def compute_fake_weigher1(hostinfo, options): return hostinfo.free_ram_mb + 10000 -def scale(hostinfo, options): +def compute_fake_weigher2(hostinfo, options): return hostinfo.free_ram_mb * 2 class LeastCostTestCase(test.TestCase): def setUp(self): super(LeastCostTestCase, self).setUp() - self.flags(reserved_host_disk_mb=0, reserved_host_memory_mb=0) self.host_manager = fakes.FakeHostManager() + self.weight_handler = weights.HostWeightHandler() + + def _get_weighed_host(self, hosts, weight_properties=None): + weigher_classes = least_cost.get_least_cost_weighers() + if weight_properties is None: + weight_properties = {} + return self.weight_handler.get_weighed_objects(weigher_classes, + hosts, weight_properties)[0] def _get_all_hosts(self): ctxt = context.get_admin_context() @@ -46,8 +70,39 @@ class LeastCostTestCase(test.TestCase): self.mox.ResetAll() return host_states - def test_weighted_sum_happy_day(self): - fn_tuples = [(1.0, offset), (1.0, scale)] + def test_default_of_spread_first(self): + # Default modifier is -1.0, so it turns out that hosts with + # the most free memory win + hostinfo_list = self._get_all_hosts() + + # host1: free_ram_mb=512 + # host2: free_ram_mb=1024 + # host3: free_ram_mb=3072 + # host4: free_ram_mb=8192 + + # so, host1 should win: + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, 8192) + self.assertEqual(weighed_host.obj.host, 'host4') + + def test_filling_first(self): + self.flags(compute_fill_first_cost_fn_weight=1.0) + hostinfo_list = self._get_all_hosts() + + # host1: free_ram_mb=-512 + # host2: free_ram_mb=-1024 + # host3: free_ram_mb=-3072 + # host4: free_ram_mb=-8192 + + # so, host1 should win: + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, -512) + self.assertEqual(weighed_host.obj.host, 'host1') + + def test_weighted_sum_provided_method(self): + fns = ['nova.tests.scheduler.test_least_cost.compute_fake_weigher1', + 'nova.tests.scheduler.test_least_cost.compute_fake_weigher2'] + self.flags(least_cost_functions=fns) hostinfo_list = self._get_all_hosts() # host1: free_ram_mb=512 @@ -59,18 +114,17 @@ class LeastCostTestCase(test.TestCase): # [10512, 11024, 13072, 18192] # [1024, 2048, 6144, 16384] - # adjusted [ 1.0 * x + 1.0 * y] = - # [11536, 13072, 19216, 34576] + # adjusted [ 2.0 * x + 1.0 * y] = + # [22048, 24096, 32288, 52768] # so, host1 should win: - options = {} - weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list, - options) - self.assertEqual(weighted_host.weight, 11536) - self.assertEqual(weighted_host.host_state.host, 'host1') + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, 52768) + self.assertEqual(weighed_host.obj.host, 'host4') def test_weighted_sum_single_function(self): - fn_tuples = [(1.0, offset), ] + fns = ['nova.tests.scheduler.test_least_cost.compute_fake_weigher1'] + self.flags(least_cost_functions=fns) hostinfo_list = self._get_all_hosts() # host1: free_ram_mb=0 @@ -80,24 +134,10 @@ class LeastCostTestCase(test.TestCase): # [offset, ]= # [10512, 11024, 13072, 18192] + # adjusted [ 2.0 * x ]= + # [21024, 22048, 26144, 36384] # so, host1 should win: - options = {} - weighted_host = least_cost.weighted_sum(fn_tuples, hostinfo_list, - options) - self.assertEqual(weighted_host.weight, 10512) - self.assertEqual(weighted_host.host_state.host, 'host1') - - -class TestWeightedHost(test.TestCase): - def test_dict_conversion_without_host_state(self): - host = least_cost.WeightedHost('someweight') - expected = {'weight': 'someweight'} - self.assertThat(host.to_dict(), matchers.DictMatches(expected)) - - def test_dict_conversion_with_host_state(self): - host_state = host_manager.HostState('somehost', None) - host = least_cost.WeightedHost('someweight', host_state) - expected = {'weight': 'someweight', - 'host': 'somehost'} - self.assertThat(host.to_dict(), matchers.DictMatches(expected)) + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, 36384) + self.assertEqual(weighed_host.obj.host, 'host4') diff --git a/nova/tests/scheduler/test_weights.py b/nova/tests/scheduler/test_weights.py new file mode 100644 index 000000000..8699ed811 --- /dev/null +++ b/nova/tests/scheduler/test_weights.py @@ -0,0 +1,117 @@ +# Copyright 2011-2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler weights. +""" + +from nova import context +from nova.scheduler import weights +from nova import test +from nova.tests import matchers +from nova.tests.scheduler import fakes + + +class TestWeighedHost(test.TestCase): + def test_dict_conversion(self): + host_state = fakes.FakeHostState('somehost', None, {}) + host = weights.WeighedHost(host_state, 'someweight') + expected = {'weight': 'someweight', + 'host': 'somehost'} + self.assertThat(host.to_dict(), matchers.DictMatches(expected)) + + def test_all_weighers(self): + classes = weights.all_weighers() + class_names = [cls.__name__ for cls in classes] + self.assertEqual(len(classes), 1) + self.assertIn('RAMWeigher', class_names) + + def test_all_weighers_with_deprecated_config1(self): + self.flags(compute_fill_first_cost_fn_weight=-1.0) + classes = weights.all_weighers() + class_names = [cls.__name__ for cls in classes] + self.assertEqual(len(classes), 1) + self.assertIn('_LeastCostWeigher', class_names) + + def test_all_weighers_with_deprecated_config2(self): + self.flags(least_cost_functions=['something']) + classes = weights.all_weighers() + class_names = [cls.__name__ for cls in classes] + self.assertEqual(len(classes), 1) + self.assertIn('_LeastCostWeigher', class_names) + + +class RamWeigherTestCase(test.TestCase): + def setUp(self): + super(RamWeigherTestCase, self).setUp() + self.host_manager = fakes.FakeHostManager() + self.weight_handler = weights.HostWeightHandler() + self.weight_classes = self.weight_handler.get_matching_classes( + ['nova.scheduler.weights.ram.RAMWeigher']) + + def _get_weighed_host(self, hosts, weight_properties=None): + if weight_properties is None: + weight_properties = {} + return self.weight_handler.get_weighed_objects(self.weight_classes, + hosts, weight_properties)[0] + + def _get_all_hosts(self): + ctxt = context.get_admin_context() + fakes.mox_host_manager_db_calls(self.mox, ctxt) + self.mox.ReplayAll() + host_states = self.host_manager.get_all_host_states(ctxt) + self.mox.VerifyAll() + self.mox.ResetAll() + return host_states + + def test_default_of_spreading_first(self): + hostinfo_list = self._get_all_hosts() + + # host1: free_ram_mb=512 + # host2: free_ram_mb=1024 + # host3: free_ram_mb=3072 + # host4: free_ram_mb=8192 + + # so, host4 should win: + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, 8192) + self.assertEqual(weighed_host.obj.host, 'host4') + + def test_ram_filter_multiplier1(self): + self.flags(ram_weight_multiplier=-1.0) + hostinfo_list = self._get_all_hosts() + + # host1: free_ram_mb=-512 + # host2: free_ram_mb=-1024 + # host3: free_ram_mb=-3072 + # host4: free_ram_mb=-8192 + + # so, host1 should win: + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, -512) + self.assertEqual(weighed_host.obj.host, 'host1') + + def test_ram_filter_multiplier2(self): + self.flags(ram_weight_multiplier=2.0) + hostinfo_list = self._get_all_hosts() + + # host1: free_ram_mb=512 * 2 + # host2: free_ram_mb=1024 * 2 + # host3: free_ram_mb=3072 * 2 + # host4: free_ram_mb=8192 * 2 + + # so, host4 should win: + weighed_host = self._get_weighed_host(hostinfo_list) + self.assertEqual(weighed_host.weight, 8192 * 2) + self.assertEqual(weighed_host.obj.host, 'host4') diff --git a/nova/weights.py b/nova/weights.py new file mode 100644 index 000000000..981171b3e --- /dev/null +++ b/nova/weights.py @@ -0,0 +1,71 @@ +# Copyright (c) 2011-2012 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Pluggable Weighing support +""" + +from nova import loadables + + +class WeighedObject(object): + """Object with weight information.""" + def __init__(self, obj, weight): + self.obj = obj + self.weight = weight + + def __repr__(self): + return "<WeighedObject '%s': %s>" % (self.obj, self.weight) + + +class BaseWeigher(object): + """Base class for pluggable weighers.""" + def _weight_multiplier(self): + """How weighted this weigher should be. Normally this would + be overriden in a subclass based on a config value. + """ + return 1.0 + + def _weigh_object(self, obj, weight_properties): + """Override in a subclass to specify a weight for a specific + object. + """ + return 0.0 + + def weigh_objects(self, weighed_obj_list, weight_properties): + """Weigh multiple objects. Override in a subclass if you need + need access to all objects in order to manipulate weights. + """ + for obj in weighed_obj_list: + obj.weight += (self._weight_multiplier() * + self._weigh_object(obj.obj, weight_properties)) + + +class BaseWeightHandler(loadables.BaseLoader): + object_class = WeighedObject + + def get_weighed_objects(self, weigher_classes, obj_list, + weighing_properties): + """Return a sorted (highest score first) list of WeighedObjects.""" + + if not obj_list: + return [] + + weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list] + for weigher_cls in weigher_classes: + weigher = weigher_cls() + weigher.weigh_objects(weighed_objs, weighing_properties) + + return sorted(weighed_objs, key=lambda x: x.weight, reverse=True) |