From 80e196069fa94edb8981415f9b8d432bbf92888f Mon Sep 17 00:00:00 2001 From: Johannes Erdfelt Date: Fri, 30 Sep 2011 15:42:38 +0000 Subject: Restructure host filtering to be easier to use. The original design for host filtering in the scheduler required the entire filtering process be contained in a single class; contrast this with the design for weighting the hosts, which allowed you to specify a list of functions that would apply various weighting factors to the hosts. This commit modifies the filtering process to resemble the way that the weighting process is designed. Filters can now be small, focused classes, and you specify which filters to apply by setting the 'FLAGS.default_host_filters' flag to a list of the filter classes that match your needs. This is a port of the code from Launchpad, where it was orphaned: https://code.launchpad.net/~ed-leafe/nova/scheduler-multifilter/+merge/72478 Change-Id: I5f3eff6f21409a9f0eddda3392e9ff9d03039ebe --- nova/tests/scheduler/test_abstract_scheduler.py | 3 +- nova/tests/scheduler/test_host_filter.py | 74 +++++++++++++---------- nova/tests/scheduler/test_least_cost_scheduler.py | 40 ++---------- 3 files changed, 47 insertions(+), 70 deletions(-) (limited to 'nova/tests') diff --git a/nova/tests/scheduler/test_abstract_scheduler.py b/nova/tests/scheduler/test_abstract_scheduler.py index da25f1544..08b0b9cde 100644 --- a/nova/tests/scheduler/test_abstract_scheduler.py +++ b/nova/tests/scheduler/test_abstract_scheduler.py @@ -455,8 +455,7 @@ class BaseSchedulerTestCase(test.TestCase): # Call weigh_hosts() num_instances = len(hostlist) * 2 + len(hostlist) / 2 - instlist = sched.weigh_hosts('compute', - dict(num_instances=num_instances), + instlist = sched.weigh_hosts(dict(num_instances=num_instances), hostlist) # Should be enough entries to cover all instances diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 17431fc7e..a21f4c380 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -18,10 +18,10 @@ Tests For Scheduler Host Filters. import json +import nova from nova import exception from nova import test from nova.scheduler import host_filter -from nova.scheduler import filters class FakeZoneManager: @@ -52,12 +52,13 @@ class HostFilterTestCase(test.TestCase): 'disk_total': 1000, 'disk_used': 0, 'host_uuid': 'xxx-%d' % multiplier, - 'host_name-label': 'xs-%s' % multiplier} + 'host_name-label': 'xs-%s' % multiplier, + 'enabled': True} def setUp(self): super(HostFilterTestCase, self).setUp() - default_host_filter = 'AllHostsFilter' - self.flags(default_host_filter=default_host_filter) + default_host_filters = ['AllHostsFilter'] + self.flags(default_host_filters=default_host_filters) self.instance_type = dict(name='tiny', memory_mb=50, vcpus=10, @@ -96,34 +97,41 @@ class HostFilterTestCase(test.TestCase): host09['xpu_arch'] = 'fermi' host09['xpu_info'] = 'Tesla 2150' + def _get_all_hosts(self): + return self.zone_manager.service_states.items() + def test_choose_filter(self): # Test default filter ... - hf = host_filter.choose_host_filter() + hfs = host_filter.choose_host_filters() + hf = hfs[0] self.assertEquals(hf._full_name().split(".")[-1], 'AllHostsFilter') # Test valid filter ... - hf = host_filter.choose_host_filter('InstanceTypeFilter') + hfs = host_filter.choose_host_filters('InstanceTypeFilter') + hf = hfs[0] self.assertEquals(hf._full_name().split(".")[-1], 'InstanceTypeFilter') # Test invalid filter ... try: - host_filter.choose_host_filter('does not exist') + host_filter.choose_host_filters('does not exist') self.fail("Should not find host filter.") except exception.SchedulerHostFilterNotFound: pass def test_all_host_filter(self): - hf = filters.AllHostsFilter() + hfs = host_filter.choose_host_filters('AllHostsFilter') + hf = hfs[0] + all_hosts = self._get_all_hosts() cooked = hf.instance_type_to_filter(self.instance_type) - hosts = hf.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(10, len(hosts)) for host, capabilities in hosts: self.assertTrue(host.startswith('host')) def test_instance_type_filter(self): - hf = filters.InstanceTypeFilter() + hf = nova.scheduler.filters.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals(name.split(".")[-1], 'InstanceTypeFilter') - hosts = hf.filter_hosts(self.zone_manager, cooked) + cooked = hf.instance_type_to_filter(self.instance_type) + all_hosts = self._get_all_hosts() + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() @@ -131,21 +139,21 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host10', just_hosts[5]) def test_instance_type_filter_extra_specs(self): - hf = filters.InstanceTypeFilter() + hf = nova.scheduler.filters.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = hf.instance_type_to_filter(self.gpu_instance_type) - self.assertEquals(name.split(".")[-1], 'InstanceTypeFilter') - hosts = hf.filter_hosts(self.zone_manager, cooked) + cooked = hf.instance_type_to_filter(self.gpu_instance_type) + all_hosts = self._get_all_hosts() + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(1, len(hosts)) just_hosts = [host for host, caps in hosts] self.assertEquals('host07', just_hosts[0]) def test_json_filter(self): - hf = filters.JsonFilter() + hf = nova.scheduler.filters.JsonFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = hf.instance_type_to_filter(self.instance_type) - self.assertEquals(name.split(".")[-1], 'JsonFilter') - hosts = hf.filter_hosts(self.zone_manager, cooked) + cooked = hf.instance_type_to_filter(self.instance_type) + all_hosts = self._get_all_hosts() + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() @@ -165,7 +173,7 @@ class HostFilterTestCase(test.TestCase): ] ] cooked = json.dumps(raw) - hosts = hf.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -177,7 +185,7 @@ class HostFilterTestCase(test.TestCase): ['=', '$compute.host_memory_free', 30], ] cooked = json.dumps(raw) - hosts = hf.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(9, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -187,7 +195,7 @@ class HostFilterTestCase(test.TestCase): raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) - hosts = hf.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(all_hosts, cooked) self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() @@ -198,32 +206,32 @@ class HostFilterTestCase(test.TestCase): raw = ['unknown command', ] cooked = json.dumps(raw) try: - hf.filter_hosts(self.zone_manager, cooked) + hf.filter_hosts(all_hosts, cooked) self.fail("Should give KeyError") except KeyError, e: pass - self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps([]))) - self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps({}))) - self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps( + self.assertTrue(hf.filter_hosts(all_hosts, json.dumps([]))) + self.assertTrue(hf.filter_hosts(all_hosts, json.dumps({}))) + self.assertTrue(hf.filter_hosts(all_hosts, json.dumps( ['not', True, False, True, False], ))) try: - hf.filter_hosts(self.zone_manager, json.dumps( + hf.filter_hosts(all_hosts, json.dumps( 'not', True, False, True, False, )) self.fail("Should give KeyError") except KeyError, e: pass - self.assertFalse(hf.filter_hosts(self.zone_manager, + self.assertFalse(hf.filter_hosts(all_hosts, json.dumps(['=', '$foo', 100]))) - self.assertFalse(hf.filter_hosts(self.zone_manager, + self.assertFalse(hf.filter_hosts(all_hosts, json.dumps(['=', '$.....', 100]))) - self.assertFalse(hf.filter_hosts(self.zone_manager, + self.assertFalse(hf.filter_hosts(all_hosts, json.dumps( ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]]))) - self.assertFalse(hf.filter_hosts(self.zone_manager, + self.assertFalse(hf.filter_hosts(all_hosts, json.dumps(['=', {}, ['>', '$missing....foo']]))) diff --git a/nova/tests/scheduler/test_least_cost_scheduler.py b/nova/tests/scheduler/test_least_cost_scheduler.py index b8847a2bf..589308e38 100644 --- a/nova/tests/scheduler/test_least_cost_scheduler.py +++ b/nova/tests/scheduler/test_least_cost_scheduler.py @@ -82,7 +82,7 @@ class LeastCostSchedulerTestCase(test.TestCase): super(LeastCostSchedulerTestCase, self).tearDown() def assertWeights(self, expected, num, request_spec, hosts): - weighted = self.sched.weigh_hosts("compute", request_spec, hosts) + weighted = self.sched.weigh_hosts(request_spec, hosts) self.assertDictListMatch(weighted, expected, approx_equal=True) def test_no_hosts(self): @@ -97,50 +97,20 @@ class LeastCostSchedulerTestCase(test.TestCase): self.flags(least_cost_scheduler_cost_functions=[ 'nova.scheduler.least_cost.noop_cost_fn'], noop_cost_fn_weight=1) - num = 1 request_spec = {} hosts = self.sched.filter_hosts(num, request_spec) - - expected = [dict(weight=1, hostname=hostname) - for hostname, caps in hosts] + expected = [{"hostname": hostname, "weight": 1, "capabilities": caps} + for hostname, caps in hosts] self.assertWeights(expected, num, request_spec, hosts) def test_cost_fn_weights(self): self.flags(least_cost_scheduler_cost_functions=[ 'nova.scheduler.least_cost.noop_cost_fn'], noop_cost_fn_weight=2) - num = 1 request_spec = {} hosts = self.sched.filter_hosts(num, request_spec) - - expected = [dict(weight=2, hostname=hostname) - for hostname, caps in hosts] - self.assertWeights(expected, num, request_spec, hosts) - - def test_compute_fill_first_cost_fn(self): - self.flags(least_cost_scheduler_cost_functions=[ - 'nova.scheduler.least_cost.compute_fill_first_cost_fn'], - compute_fill_first_cost_fn_weight=1) - num = 1 - instance_type = {'memory_mb': 1024} - request_spec = {'instance_type': instance_type} - svc_states = self.sched.zone_manager.service_states.iteritems() - all_hosts = [(host, services["compute"]) - for host, services in svc_states - if "compute" in services] - hosts = self.sched.filter_hosts('compute', request_spec, all_hosts) - - expected = [] - for idx, (hostname, services) in enumerate(hosts): - caps = copy.deepcopy(services) - # Costs are normalized so over 10 hosts, each host with increasing - # free ram will cost 1/N more. Since the lowest cost host has some - # free ram, we add in the 1/N for the base_cost - weight = 0.1 + (0.1 * idx) - wtd_dict = dict(hostname=hostname, weight=weight, - capabilities=caps) - expected.append(wtd_dict) - + expected = [{"hostname": hostname, "weight": 2, "capabilities": caps} + for hostname, caps in hosts] self.assertWeights(expected, num, request_spec, hosts) -- cgit