From f6ee353e388a52e338a7f1a27f924a9a0c60f9a1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 2 May 2011 12:49:10 -0700 Subject: initial pass --- nova/scheduler/query.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 nova/scheduler/query.py (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py new file mode 100644 index 000000000..5379b3d44 --- /dev/null +++ b/nova/scheduler/query.py @@ -0,0 +1,164 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Query is a plug-in mechanism for requesting instance resources. +Three plug-ins are included: PassThru, Flavor & JSON. PassThru just +returns the full, unfiltered list of hosts. Flavor is a hard coded +matching mechanism based on flavor criteria and JSON is an ad-hoc +query grammar. +""" + +from nova import exception +from nova import utils + +class Query: + """Base class for query plug-ins.""" + + def instance_type_to_query(self, instance_type): + """Convert instance_type into a query for most common use-case.""" + raise exception.Error(_("Query driver not specified.")) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that fulfill the query.""" + raise exception.Error(_("Query driver not specified.")) + + +def load_driver(driver_name): + resource = utils.import_class(driver_name) + if type(resource) != types.ClassType: + continue + cls = resource + if not issubclass(cls, Query): + raise exception.Error(_("Query driver does not derive " + "from nova.scheduler.query.Query.")) + return cls + + +class PassThruQuery: + """NOP query plug-in. Returns all hosts in ZoneManager. + This essentially does what the old Scheduler+Chance used + to give us.""" + + def instance_type_to_query(self, instance_type): + """Return anything to prevent base-class from raising + exception.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts from ZoneManager list.""" + hosts = zone_manager.service_state.get('compute', {}) + return [(host, capabilities) + for host, capabilities in hosts.iteritems()] + + +class FlavorQuery: + """Query plug-in hard-coded to work with flavors.""" + + def instance_type_to_query(self, instance_type): + """Use instance_type to filter hosts.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can create instance_type.""" + hosts = zone_manager.service_state.get('compute', {}) + selected_hosts = [] + instance_type = query + for host, capabilities in hosts.iteritems(): + host_ram_mb = capabilities.get['host_memory']['free'] + disk_bytes = capabilities.get['disk']['available'] + if host_ram_mb >= instance_type['memory_mb'] and \ + disk_bytes >= instance_type['local_gb']: + selected_hosts.append((host, capabilities)) + return selected_hosts + +#host entries (currently) are like: +# {'host_name-description': 'Default install of XenServer', +# 'host_hostname': 'xs-mini', +# 'host_memory': {'total': 8244539392, +# 'overhead': 184225792, +# 'free': 3868327936, +# 'free-computed': 3840843776}, +# 'host_other-config': {}, +# 'host_ip_address': '192.168.1.109', +# 'host_cpu_info': {}, +# 'disk': {'available': 32954957824, +# 'total': 50394562560, +# 'used': 17439604736}, +# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', +# 'host_name-label': 'xs-mini'} + +# instance_type table has: +#name = Column(String(255), unique=True) +#memory_mb = Column(Integer) +#vcpus = Column(Integer) +#local_gb = Column(Integer) +#flavorid = Column(Integer, unique=True) +#swap = Column(Integer, nullable=False, default=0) +#rxtx_quota = Column(Integer, nullable=False, default=0) +#rxtx_cap = Column(Integer, nullable=False, default=0) + +class JsonQuery: + """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" + + def _equals(self, args): + pass + + def _less_than(self, args): + pass + + def _greater_than(self, args): + pass + + def _in(self, args): + pass + + def _less_than_equal(self, args): + pass + + def _greater_than_equal(self, args): + pass + + def _not(self, args): + pass + + def _must(self, args): + pass + + def _or(self, args): + pass + + commands = { + '=': _equals, + '<': _less_than, + '>': _greater_than, + 'in': _in, + '<=': _less_than_equal, + '>=': _greater_than_equal, + 'not': _not, + 'must', _must, + 'or', _or, + } + + def instance_type_to_query(self, instance_type): + """Convert instance_type into JSON query object.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can fulfill query.""" + return [] + + + -- cgit From f50c7260fa0b1dbcfb725bedeb9bb0ed4398f767 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 3 May 2011 15:02:55 -0700 Subject: tests and better driver loading --- nova/exception.py | 8 +++++ nova/scheduler/query.py | 50 ++++++++++++++++----------- nova/tests/test_query.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 nova/tests/test_query.py (limited to 'nova') diff --git a/nova/exception.py b/nova/exception.py index e8444cb14..586ec009b 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -449,6 +449,14 @@ class ZoneNotFound(NotFound): message = _("Zone %(zone_id)s could not be found.") +class SchedulerQueryDriverNotFound(NotFound): + message = _("Scheduler Query Driver %(driver_name)s could not be found.") + + +class BadSchedulerQueryDriver(NotFound): + message = _("Invalid Scheduler Query Driver selected.") + + class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 5379b3d44..4971bc7e4 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -15,39 +15,38 @@ """ Query is a plug-in mechanism for requesting instance resources. -Three plug-ins are included: PassThru, Flavor & JSON. PassThru just +Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. """ from nova import exception +from nova import flags +from nova import log as logging from nova import utils +LOG = logging.getLogger('nova.scheduler.query') + +FLAGS = flags.FLAGS +flags.DEFINE_string('default_query_engine', + 'nova.scheduler.query.AllHostsQuery', + 'Which query engine to use for filtering hosts.') + + class Query: """Base class for query plug-ins.""" def instance_type_to_query(self, instance_type): """Convert instance_type into a query for most common use-case.""" - raise exception.Error(_("Query driver not specified.")) + raise exception.BadSchedulerQueryDriver() def filter_hosts(self, zone_manager, query): """Return a list of hosts that fulfill the query.""" - raise exception.Error(_("Query driver not specified.")) - + raise exception.BadSchedulerQueryDriver() -def load_driver(driver_name): - resource = utils.import_class(driver_name) - if type(resource) != types.ClassType: - continue - cls = resource - if not issubclass(cls, Query): - raise exception.Error(_("Query driver does not derive " - "from nova.scheduler.query.Query.")) - return cls - -class PassThruQuery: +class AllHostsQuery: """NOP query plug-in. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" @@ -59,7 +58,7 @@ class PassThruQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" - hosts = zone_manager.service_state.get('compute', {}) + hosts = zone_manager.service_states.get('compute', {}) return [(host, capabilities) for host, capabilities in hosts.iteritems()] @@ -73,7 +72,7 @@ class FlavorQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" - hosts = zone_manager.service_state.get('compute', {}) + hosts = zone_manager.service_states.get('compute', {}) selected_hosts = [] instance_type = query for host, capabilities in hosts.iteritems(): @@ -148,8 +147,8 @@ class JsonQuery: '<=': _less_than_equal, '>=': _greater_than_equal, 'not': _not, - 'must', _must, - 'or', _or, + 'must': _must, + 'or': _or, } def instance_type_to_query(self, instance_type): @@ -161,4 +160,15 @@ class JsonQuery: return [] - +# Since the caller may specify which driver to use we need +# to have an authoritative list of what is permissible. +DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] + + +def choose_driver(driver_name=None): + if not driver_name: + driver_name = FLAGS.default_query_engine + for driver in DRIVERS: + if str(driver) == driver_name: + return driver + raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py new file mode 100644 index 000000000..0fab91933 --- /dev/null +++ b/nova/tests/test_query.py @@ -0,0 +1,89 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler Query Drivers +""" + +from nova import exception +from nova import flags +from nova import test +from nova.scheduler import query + +FLAGS = flags.FLAGS + +class FakeZoneManager: + pass + +class QueryTestCase(test.TestCase): + """Test case for query drivers.""" + + def _host_caps(self, multiplier): + return {'host_name-description':'XenServer %s' % multiplier, + 'host_hostname':'xs-%s' % multiplier, + 'host_memory':{'total': 100, + 'overhead': 5, + 'free': 5 + multiplier * 5, + 'free-computed': 5 + multiplier * 5}, + 'host_other-config':{}, + 'host_ip_address':'192.168.1.%d' % (100 + multiplier), + 'host_cpu_info':{}, + 'disk':{'available': 100 + multiplier * 100, + 'total': 1000, + 'used': 50 + multiplier * 50}, + 'host_uuid':'xxx-%d' % multiplier, + 'host_name-label':'xs-%s' % multiplier} + + def setUp(self): + self.old_flag = FLAGS.default_query_engine + FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' + self.instance_type = dict(name='tiny', + memory_mb=500, + vcpus=10, + local_gb=50, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) + + hosts = {} + for x in xrange(10): + hosts['host%s' % x] = self._host_caps(x) + + self.zone_manager = FakeZoneManager() + self.zone_manager.service_states = {} + self.zone_manager.service_states['compute'] = hosts + + def tearDown(self): + FLAGS.default_query_engine = self.old_flag + + def test_choose_driver(self): + driver = query.choose_driver() + self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + driver = query.choose_driver('nova.scheduler.query.FlavorQuery') + self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + try: + query.choose_driver('does not exist') + self.fail("Should not find driver") + except exception.SchedulerQueryDriverNotFound: + pass + + def test_all_host_driver(self): + driver = query.AllHostsQuery() + cooked = driver.instance_type_to_query(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(10, len(hosts)) + for host, capabilities in hosts: + self.assertTrue(host.startswith('host')) + -- cgit From ad07b86110b0bcb90f3b71bd423c06a6ff5f922d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 4 May 2011 06:16:33 -0700 Subject: flavor test --- nova/scheduler/query.py | 6 +++--- nova/tests/test_query.py | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 15 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 4971bc7e4..1c593f1ba 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -74,10 +74,10 @@ class FlavorQuery: """Return a list of hosts that can create instance_type.""" hosts = zone_manager.service_states.get('compute', {}) selected_hosts = [] - instance_type = query + query_type, instance_type = query for host, capabilities in hosts.iteritems(): - host_ram_mb = capabilities.get['host_memory']['free'] - disk_bytes = capabilities.get['disk']['available'] + host_ram_mb = capabilities['host_memory']['free'] + disk_bytes = capabilities['disk']['available'] if host_ram_mb >= instance_type['memory_mb'] and \ disk_bytes >= instance_type['local_gb']: selected_hosts.append((host, capabilities)) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 0fab91933..d89183e9f 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -30,32 +30,39 @@ class QueryTestCase(test.TestCase): """Test case for query drivers.""" def _host_caps(self, multiplier): + # Returns host capabilities in the following way: + # host0 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. return {'host_name-description':'XenServer %s' % multiplier, 'host_hostname':'xs-%s' % multiplier, 'host_memory':{'total': 100, - 'overhead': 5, - 'free': 5 + multiplier * 5, - 'free-computed': 5 + multiplier * 5}, + 'overhead': 10, + 'free': 10 + multiplier * 10, + 'free-computed': 10 + multiplier * 10}, 'host_other-config':{}, 'host_ip_address':'192.168.1.%d' % (100 + multiplier), 'host_cpu_info':{}, 'disk':{'available': 100 + multiplier * 100, 'total': 1000, - 'used': 50 + multiplier * 50}, + 'used': 0}, 'host_uuid':'xxx-%d' % multiplier, 'host_name-label':'xs-%s' % multiplier} def setUp(self): self.old_flag = FLAGS.default_query_engine FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name='tiny', - memory_mb=500, - vcpus=10, - local_gb=50, - flavorid=1, - swap=500, - rxtx_quota=30000, - rxtx_cap=200) + self.instance_type = dict(name= 'tiny', + memory_mb= 50, + vcpus= 10, + local_gb= 500, + flavorid= 1, + swap= 500, + rxtx_quota= 30000, + rxtx_cap= 200) hosts = {} for x in xrange(10): @@ -69,10 +76,13 @@ class QueryTestCase(test.TestCase): FLAGS.default_query_engine = self.old_flag def test_choose_driver(self): + # Test default driver ... driver = query.choose_driver() self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + # Test valid driver ... driver = query.choose_driver('nova.scheduler.query.FlavorQuery') self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + # Test invalid driver ... try: query.choose_driver('does not exist') self.fail("Should not find driver") @@ -87,3 +97,13 @@ class QueryTestCase(test.TestCase): for host, capabilities in hosts: self.assertTrue(host.startswith('host')) + def test_flavor_driver(self): + driver = query.FlavorQuery() + # filter all hosts that can support 50 ram and 500 disk + cooked = driver.instance_type_to_query(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host4', just_hosts[0]) + self.assertEquals('host9', just_hosts[5]) -- cgit From 772fc58644871f7ee0bb880074c89d79871e197a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 4 May 2011 14:48:23 -0700 Subject: json parser --- nova/scheduler/query.py | 123 ++++++++++++++++++++++++++++++++++++++++------- nova/tests/test_query.py | 25 +++++++--- 2 files changed, 124 insertions(+), 24 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1c593f1ba..1b84e6b1d 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -21,6 +21,8 @@ matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. """ +import json + from nova import exception from nova import flags from nova import log as logging @@ -58,9 +60,8 @@ class AllHostsQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" - hosts = zone_manager.service_states.get('compute', {}) - return [(host, capabilities) - for host, capabilities in hosts.iteritems()] + return [(host, services) + for host, services in zone_manager.service_state.iteritems()] class FlavorQuery: @@ -72,10 +73,10 @@ class FlavorQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" - hosts = zone_manager.service_states.get('compute', {}) + instance_type = query selected_hosts = [] - query_type, instance_type = query - for host, capabilities in hosts.iteritems(): + for host, services in zone_manager.service_states.iteritems(): + capabilities = services.get('compute', {}) host_ram_mb = capabilities['host_memory']['free'] disk_bytes = capabilities['disk']['available'] if host_ram_mb >= instance_type['memory_mb'] and \ @@ -113,31 +114,74 @@ class JsonQuery: """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" def _equals(self, args): - pass + """First term is == all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs != rhs: + return False + return True def _less_than(self, args): - pass + """First term is < all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs >= rhs: + return False + return True def _greater_than(self, args): - pass + """First term is > all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs <= rhs: + return False + return True def _in(self, args): - pass + """First term is in set of remaining terms""" + if len(args) < 2: + return False + return args[0] in args[1:] def _less_than_equal(self, args): - pass + """First term is <= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs > rhs: + return False + return True def _greater_than_equal(self, args): - pass + """First term is >= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs < rhs: + return False + return True def _not(self, args): - pass + if len(args) == 0: + return False + return not args[0] def _must(self, args): - pass + return True def _or(self, args): - pass + return True in args + + def _and(self, args): + return False not in args commands = { '=': _equals, @@ -149,15 +193,60 @@ class JsonQuery: 'not': _not, 'must': _must, 'or': _or, + 'and': _and, } def instance_type_to_query(self, instance_type): """Convert instance_type into JSON query object.""" - return (str(self.__class__), instance_type) + required_ram = instance_type['memory_mb'] + required_disk = instance_type['local_gb'] + query = ['and', + ['>=', '$compute.host_memory.free', required_ram], + ['>=', '$compute.disk.available', required_disk] + ] + return (str(self.__class__), json.dumps(query)) + + def _parse_string(self, string, host, services): + """Strings prefixed with $ are capability lookups in the + form '$service.capability[.subcap*]'""" + if not string: + return None + if string[0] != '$': + return string + + path = string[1:].split('.') + for item in path: + services = services.get(item, None) + if not services: + return None + return services + + def _process_query(self, zone_manager, query, host, services): + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_query(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + print "*** %s %s = %s" % (cmd, cooked_args, result) + return result def filter_hosts(self, zone_manager, query): """Return a list of hosts that can fulfill query.""" - return [] + expanded = json.loads(query) + hosts = [] + for host, services in zone_manager.service_states.iteritems(): + print "-----" + if self._process_query(zone_manager, expanded, host, services): + hosts.append((host, services)) + return hosts # Since the caller may specify which driver to use we need diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index d89183e9f..7a036a4d8 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -64,13 +64,11 @@ class QueryTestCase(test.TestCase): rxtx_quota= 30000, rxtx_cap= 200) - hosts = {} - for x in xrange(10): - hosts['host%s' % x] = self._host_caps(x) - self.zone_manager = FakeZoneManager() - self.zone_manager.service_states = {} - self.zone_manager.service_states['compute'] = hosts + states = {} + for x in xrange(10): + states['host%s' % x] = {'compute': self._host_caps(x)} + self.zone_manager.service_states = states def tearDown(self): FLAGS.default_query_engine = self.old_flag @@ -100,7 +98,20 @@ class QueryTestCase(test.TestCase): def test_flavor_driver(self): driver = query.FlavorQuery() # filter all hosts that can support 50 ram and 500 disk - cooked = driver.instance_type_to_query(self.instance_type) + name, cooked = driver.instance_type_to_query(self.instance_type) + self.assertEquals('nova.scheduler.query.FlavorQuery', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host4', just_hosts[0]) + self.assertEquals('host9', just_hosts[5]) + + def test_json_driver(self): + driver = query.JsonQuery() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_query(self.instance_type) + self.assertEquals('nova.scheduler.query.JsonQuery', name) hosts = driver.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] -- cgit From cc18ff47ff41ddefd7a31db5b772d55b2e312e8c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 04:57:25 -0700 Subject: and or test --- nova/scheduler/query.py | 10 +++++----- nova/tests/test_query.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1b84e6b1d..3233cc0d8 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -19,6 +19,10 @@ Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. + +Note: These are hard filters. All capabilities used must be present +or the host will excluded. If you want soft filters use the weighting +mechanism which is intended for the more touchy-feely capabilities. """ import json @@ -61,7 +65,7 @@ class AllHostsQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" return [(host, services) - for host, services in zone_manager.service_state.iteritems()] + for host, services in zone_manager.service_states.iteritems()] class FlavorQuery: @@ -174,9 +178,6 @@ class JsonQuery: return False return not args[0] - def _must(self, args): - return True - def _or(self, args): return True in args @@ -191,7 +192,6 @@ class JsonQuery: '<=': _less_than_equal, '>=': _greater_than_equal, 'not': _not, - 'must': _must, 'or': _or, 'and': _and, } diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 7a036a4d8..896b2364d 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -16,6 +16,8 @@ Tests For Scheduler Query Drivers """ +import json + from nova import exception from nova import flags from nova import test @@ -118,3 +120,27 @@ class QueryTestCase(test.TestCase): just_hosts.sort() self.assertEquals('host4', just_hosts[0]) self.assertEquals('host9', just_hosts[5]) + + # Try some custom queries + + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host0', just_hosts[0]) + self.assertEquals('host1', just_hosts[1]) + self.assertEquals('host7', just_hosts[2]) + self.assertEquals('host8', just_hosts[3]) + self.assertEquals('host9', just_hosts[4]) -- cgit From 5f4fc98c9648fd3f124819e0f4a26cb1d2d7f0e8 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 04:59:26 -0700 Subject: and or test --- nova/tests/test_query.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 896b2364d..90ae80dc0 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -139,8 +139,5 @@ class QueryTestCase(test.TestCase): self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host0', just_hosts[0]) - self.assertEquals('host1', just_hosts[1]) - self.assertEquals('host7', just_hosts[2]) - self.assertEquals('host8', just_hosts[3]) - self.assertEquals('host9', just_hosts[4]) + for index, host in zip([0, 1, 7, 8, 9], just_hosts): + self.assertEquals('host%d' % index, host) -- cgit From 5a066cf5c2b952371eea753dcd0f95f917d08744 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 05:29:31 -0700 Subject: not = --- nova/scheduler/query.py | 7 +++++-- nova/tests/test_query.py | 29 +++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 3233cc0d8..0279efc9e 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -176,7 +176,7 @@ class JsonQuery: def _not(self, args): if len(args) == 0: return False - return not args[0] + return [not arg for arg in args] def _or(self, args): return True in args @@ -244,7 +244,10 @@ class JsonQuery: hosts = [] for host, services in zone_manager.service_states.iteritems(): print "-----" - if self._process_query(zone_manager, expanded, host, services): + r = self._process_query(zone_manager, expanded, host, services) + if isinstance(r, list): + r = True in r + if r: hosts.append((host, services)) return hosts diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 90ae80dc0..9bfc83b75 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -33,7 +33,7 @@ class QueryTestCase(test.TestCase): def _host_caps(self, multiplier): # Returns host capabilities in the following way: - # host0 = memory:free 10 (100max) + # host1 = memory:free 10 (100max) # disk:available 100 (1000max) # hostN = memory:free 10 + 10N # disk:available 100 + 100N @@ -69,7 +69,7 @@ class QueryTestCase(test.TestCase): self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): - states['host%s' % x] = {'compute': self._host_caps(x)} + states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states def tearDown(self): @@ -106,8 +106,8 @@ class QueryTestCase(test.TestCase): self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host4', just_hosts[0]) - self.assertEquals('host9', just_hosts[5]) + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) def test_json_driver(self): driver = query.JsonQuery() @@ -118,8 +118,8 @@ class QueryTestCase(test.TestCase): self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host4', just_hosts[0]) - self.assertEquals('host9', just_hosts[5]) + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) # Try some custom queries @@ -139,5 +139,18 @@ class QueryTestCase(test.TestCase): self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - for index, host in zip([0, 1, 7, 8, 9], just_hosts): - self.assertEquals('host%d' % index, host) + for index, host in zip([1, 2, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(9, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + -- cgit From 4b03036214cff2fcaad079f84605737d7f9dc711 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 05:30:58 -0700 Subject: not = --- nova/tests/test_query.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'nova') diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 9bfc83b75..78e8ee9de 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -153,4 +153,14 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100]] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([2, 4, 6, 8, 10], just_hosts): + self.assertEquals('host%02d' % index, host) -- cgit From 31c7b40b89c076a32d8105219d623320c58b8166 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 06:01:56 -0700 Subject: pep8 --- nova/scheduler/query.py | 43 +++++++++++++++-------------- nova/tests/test_query.py | 71 ++++++++++++++++++++++++------------------------ 2 files changed, 58 insertions(+), 56 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 0279efc9e..4332f43b0 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -114,6 +114,7 @@ class FlavorQuery: #rxtx_quota = Column(Integer, nullable=False, default=0) #rxtx_cap = Column(Integer, nullable=False, default=0) + class JsonQuery: """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" @@ -162,7 +163,7 @@ class JsonQuery: if lhs > rhs: return False return True - + def _greater_than_equal(self, args): """First term is >= all the other terms.""" if len(args) < 2: @@ -180,10 +181,10 @@ class JsonQuery: def _or(self, args): return True in args - + def _and(self, args): return False not in args - + commands = { '=': _equals, '<': _less_than, @@ -200,7 +201,7 @@ class JsonQuery: """Convert instance_type into JSON query object.""" required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] - query = ['and', + query = ['and', ['>=', '$compute.host_memory.free', required_ram], ['>=', '$compute.disk.available', required_disk] ] @@ -219,24 +220,24 @@ class JsonQuery: services = services.get(item, None) if not services: return None - return services + return services def _process_query(self, zone_manager, query, host, services): - if len(query) == 0: - return True - cmd = query[0] - method = self.commands[cmd] # Let exception fly. - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_query(zone_manager, arg, host, services) - elif isinstance(arg, basestring): - arg = self._parse_string(arg, host, services) - if arg != None: - cooked_args.append(arg) - result = method(self, cooked_args) - print "*** %s %s = %s" % (cmd, cooked_args, result) - return result + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_query(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + print "*** %s %s = %s" % (cmd, cooked_args, result) + return result def filter_hosts(self, zone_manager, query): """Return a list of hosts that can fulfill query.""" @@ -253,7 +254,7 @@ class JsonQuery: # Since the caller may specify which driver to use we need -# to have an authoritative list of what is permissible. +# to have an authoritative list of what is permissible. DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 78e8ee9de..8b894a4e6 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -25,9 +25,11 @@ from nova.scheduler import query FLAGS = flags.FLAGS + class FakeZoneManager: pass + class QueryTestCase(test.TestCase): """Test case for query drivers.""" @@ -39,32 +41,32 @@ class QueryTestCase(test.TestCase): # disk:available 100 + 100N # in other words: hostN has more resources than host0 # which means ... don't go above 10 hosts. - return {'host_name-description':'XenServer %s' % multiplier, - 'host_hostname':'xs-%s' % multiplier, - 'host_memory':{'total': 100, + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory': {'total': 100, 'overhead': 10, 'free': 10 + multiplier * 10, 'free-computed': 10 + multiplier * 10}, - 'host_other-config':{}, - 'host_ip_address':'192.168.1.%d' % (100 + multiplier), - 'host_cpu_info':{}, - 'disk':{'available': 100 + multiplier * 100, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk': {'available': 100 + multiplier * 100, 'total': 1000, 'used': 0}, - 'host_uuid':'xxx-%d' % multiplier, - 'host_name-label':'xs-%s' % multiplier} + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} def setUp(self): self.old_flag = FLAGS.default_query_engine FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name= 'tiny', - memory_mb= 50, - vcpus= 10, - local_gb= 500, - flavorid= 1, - swap= 500, - rxtx_quota= 30000, - rxtx_cap= 200) + self.instance_type = dict(name='tiny', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) self.zone_manager = FakeZoneManager() states = {} @@ -123,16 +125,16 @@ class QueryTestCase(test.TestCase): # Try some custom queries - raw = ['or', - ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] - ], - ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] - ] - ] + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -141,10 +143,10 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - - raw = ['not', - ['=', '$compute.host_memory.free', 30], - ] + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -153,8 +155,8 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100]] + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -163,4 +165,3 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([2, 4, 6, 8, 10], just_hosts): self.assertEquals('host%02d' % index, host) - -- cgit From 37954e665b874ac2358921d175b30617f456c007 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 07:35:44 -0700 Subject: flipped service_state in ZoneManager and fixed tests --- nova/scheduler/api.py | 8 +++----- nova/scheduler/manager.py | 7 +++---- nova/scheduler/zone_manager.py | 20 +++++++++----------- nova/tests/api/openstack/test_zones.py | 2 +- nova/tests/test_zones.py | 18 ++++++------------ 5 files changed, 22 insertions(+), 33 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6bb3bf3cd..816ae5513 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -76,11 +76,9 @@ def zone_update(context, zone_id, data): return db.zone_update(context, zone_id, data) -def get_zone_capabilities(context, service=None): - """Returns a dict of key, value capabilities for this zone, - or for a particular class of services running in this zone.""" - return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) +def get_zone_capabilities(context): + """Returns a dict of key, value capabilities for this zone.""" + return _call_scheduler('get_zone_capabilities', context=context) def update_service_capabilities(context, service_name, host, capabilities): diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 7d62cfc4e..55cd7208b 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -60,10 +60,9 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() - def get_zone_capabilities(self, context=None, service=None): - """Get the normalized set of capabilites for this zone, - or for a particular service.""" - return self.zone_manager.get_zone_capabilities(context, service) + def get_zone_capabilities(self, context=None): + """Get the normalized set of capabilites for this zone.""" + return self.zone_manager.get_zone_capabilities(context) def update_service_capabilities(self, context=None, service_name=None, host=None, capabilities={}): diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 198f9d4cc..3ddf6f3c3 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -106,28 +106,26 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} # { : ZoneState } - self.service_states = {} # { : { : { cap k : v }}} + self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" return [zone.to_dict() for zone in self.zone_states.values()] - def get_zone_capabilities(self, context, service=None): + def get_zone_capabilities(self, context): """Roll up all the individual host info to generic 'service' capabilities. Each capability is aggregated into _min and _max values.""" - service_dict = self.service_states - if service: - service_dict = {service: self.service_states.get(service, {})} + hosts_dict = self.service_states # TODO(sandy) - be smarter about fabricating this structure. # But it's likely to change once we understand what the Best-Match # code will need better. combined = {} # { _ : (min, max), ... } - for service_name, host_dict in service_dict.iteritems(): - for host, caps_dict in host_dict.iteritems(): - for cap, value in caps_dict.iteritems(): + for host, host_dict in hosts_dict.iteritems(): + for service_name, service_dict in host_dict.iteritems(): + for cap, value in service_dict.iteritems(): key = "%s_%s" % (service_name, cap) min_value, max_value = combined.get(key, (value, value)) min_value = min(min_value, value) @@ -171,6 +169,6 @@ class ZoneManager(object): """Update the per-service capabilities based on this notification.""" logging.debug(_("Received %(service_name)s service update from " "%(host)s: %(capabilities)s") % locals()) - service_caps = self.service_states.get(service_name, {}) - service_caps[host] = capabilities - self.service_states[service_name] = service_caps + service_caps = self.service_states.get(host, {}) + service_caps[service_name] = capabilities + self.service_states[host] = service_caps diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index a3f191aaa..5d5799b59 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -75,7 +75,7 @@ def zone_get_all_db(context): ] -def zone_capabilities(method, context, params): +def zone_capabilities(method, context): return dict() diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 688dc704d..e132809dc 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -78,38 +78,32 @@ class ZoneManagerTestCase(test.TestCase): def test_service_capabilities(self): zm = zone_manager.ZoneManager() - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, {}) zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2))) zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc10_a=(99, 99), svc10_b=(99, 99))) zm.update_service_capabilities("svc1", "host3", dict(c=5)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) - caps = zm.get_zone_capabilities(self, 'svc1') - self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), - svc1_c=(5, 5))) - caps = zm.get_zone_capabilities(self, 'svc10') - self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) - def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 27f99a14a381062f87fbbb65d5aaa07914aa82c0 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 08:13:22 -0700 Subject: print statements removed --- nova/scheduler/query.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 4332f43b0..1a6705b42 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -236,7 +236,6 @@ class JsonQuery: if arg != None: cooked_args.append(arg) result = method(self, cooked_args) - print "*** %s %s = %s" % (cmd, cooked_args, result) return result def filter_hosts(self, zone_manager, query): @@ -244,7 +243,6 @@ class JsonQuery: expanded = json.loads(query) hosts = [] for host, services in zone_manager.service_states.iteritems(): - print "-----" r = self._process_query(zone_manager, expanded, host, services) if isinstance(r, list): r = True in r -- cgit From 9797bda1baacf5d90d6ce678c20396bacb513e37 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 10:50:59 -0700 Subject: merge prop fixes --- nova/exception.py | 4 ---- nova/scheduler/query.py | 37 ++++++++++++++++++++++++------------- nova/tests/test_query.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 19 deletions(-) (limited to 'nova') diff --git a/nova/exception.py b/nova/exception.py index eeeed7952..50f50de9d 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -461,10 +461,6 @@ class SchedulerQueryDriverNotFound(NotFound): message = _("Scheduler Query Driver %(driver_name)s could not be found.") -class BadSchedulerQueryDriver(NotFound): - message = _("Invalid Scheduler Query Driver selected.") - - class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1a6705b42..1e294b595 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -40,19 +40,23 @@ flags.DEFINE_string('default_query_engine', 'Which query engine to use for filtering hosts.') -class Query: +class Query(object): """Base class for query plug-ins.""" def instance_type_to_query(self, instance_type): """Convert instance_type into a query for most common use-case.""" - raise exception.BadSchedulerQueryDriver() + raise NotImplementedError() def filter_hosts(self, zone_manager, query): """Return a list of hosts that fulfill the query.""" - raise exception.BadSchedulerQueryDriver() + raise NotImplementedError() + def _full_name(self): + """module.classname of the Query object""" + return "%s.%s" % (self.__module__, self.__class__.__name__) -class AllHostsQuery: + +class AllHostsQuery(Query): """NOP query plug-in. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" @@ -60,7 +64,7 @@ class AllHostsQuery: def instance_type_to_query(self, instance_type): """Return anything to prevent base-class from raising exception.""" - return (str(self.__class__), instance_type) + return (self._full_name(), instance_type) def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" @@ -68,12 +72,12 @@ class AllHostsQuery: for host, services in zone_manager.service_states.iteritems()] -class FlavorQuery: +class FlavorQuery(Query): """Query plug-in hard-coded to work with flavors.""" def instance_type_to_query(self, instance_type): """Use instance_type to filter hosts.""" - return (str(self.__class__), instance_type) + return (self._full_name(), instance_type) def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" @@ -115,7 +119,7 @@ class FlavorQuery: #rxtx_cap = Column(Integer, nullable=False, default=0) -class JsonQuery: +class JsonQuery(Query): """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" def _equals(self, args): @@ -175,14 +179,17 @@ class JsonQuery: return True def _not(self, args): + """Flip each of the arguments.""" if len(args) == 0: return False return [not arg for arg in args] def _or(self, args): + """True if any arg is True.""" return True in args def _and(self, args): + """True if all args are True.""" return False not in args commands = { @@ -205,7 +212,7 @@ class JsonQuery: ['>=', '$compute.host_memory.free', required_ram], ['>=', '$compute.disk.available', required_disk] ] - return (str(self.__class__), json.dumps(query)) + return (self._full_name(), json.dumps(query)) def _parse_string(self, string, host, services): """Strings prefixed with $ are capability lookups in the @@ -223,6 +230,7 @@ class JsonQuery: return services def _process_query(self, zone_manager, query, host, services): + """Recursively parse the query structure.""" if len(query) == 0: return True cmd = query[0] @@ -251,15 +259,18 @@ class JsonQuery: return hosts -# Since the caller may specify which driver to use we need -# to have an authoritative list of what is permissible. DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] def choose_driver(driver_name=None): + """Since the caller may specify which driver to use we need + to have an authoritative list of what is permissible. This + function checks the driver name against a predefined set + of acceptable drivers.""" + if not driver_name: driver_name = FLAGS.default_query_engine for driver in DRIVERS: - if str(driver) == driver_name: - return driver + if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: + return driver() raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 8b894a4e6..9497a8c96 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -80,10 +80,12 @@ class QueryTestCase(test.TestCase): def test_choose_driver(self): # Test default driver ... driver = query.choose_driver() - self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + self.assertEquals(driver._full_name(), + 'nova.scheduler.query.AllHostsQuery') # Test valid driver ... driver = query.choose_driver('nova.scheduler.query.FlavorQuery') - self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + self.assertEquals(driver._full_name(), + 'nova.scheduler.query.FlavorQuery') # Test invalid driver ... try: query.choose_driver('does not exist') @@ -165,3 +167,40 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([2, 4, 6, 8, 10], just_hosts): self.assertEquals('host%02d' % index, host) + + # Try some bogus input ... + raw = ['unknown command', ] + cooked = json.dumps(raw) + try: + driver.filter_hosts(self.zone_manager, cooked) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False] + ))) + + try: + driver.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False + )) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$foo', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$.....', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] + ))) + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', {}, ['>', '$missing....foo']] + ))) -- cgit From a5e7d039ec9ee9528186fa011021da00d809e683 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 18:09:11 -0700 Subject: terminology: no more plug-ins or queries. They are host filters and drivers. --- nova/exception.py | 5 +- nova/scheduler/host_filter.py | 286 +++++++++++++++++++++++++++++++++++++++++ nova/scheduler/query.py | 276 --------------------------------------- nova/tests/test_host_filter.py | 208 ++++++++++++++++++++++++++++++ nova/tests/test_query.py | 206 ----------------------------- 5 files changed, 497 insertions(+), 484 deletions(-) create mode 100644 nova/scheduler/host_filter.py delete mode 100644 nova/scheduler/query.py create mode 100644 nova/tests/test_host_filter.py delete mode 100644 nova/tests/test_query.py (limited to 'nova') diff --git a/nova/exception.py b/nova/exception.py index 50f50de9d..9905fb19b 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -457,8 +457,9 @@ class ZoneNotFound(NotFound): message = _("Zone %(zone_id)s could not be found.") -class SchedulerQueryDriverNotFound(NotFound): - message = _("Scheduler Query Driver %(driver_name)s could not be found.") +class SchedulerHostFilterDriverNotFound(NotFound): + message = _("Scheduler Host Filter Driver %(driver_name)s could" + " not be found.") class InstanceMetadataNotFound(NotFound): diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py new file mode 100644 index 000000000..aa6101c93 --- /dev/null +++ b/nova/scheduler/host_filter.py @@ -0,0 +1,286 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Host Filter is a driver mechanism for requesting instance resources. +Three drivers are included: AllHosts, Flavor & JSON. AllHosts just +returns the full, unfiltered list of hosts. Flavor is a hard coded +matching mechanism based on flavor criteria and JSON is an ad-hoc +filter grammar. + +Why JSON? The requests for instances may come in through the +REST interface from a user or a parent Zone. +Currently Flavors and/or InstanceTypes are used for +specifing the type of instance desired. Specific Nova users have +noted a need for a more expressive way of specifying instances. +Since we don't want to get into building full DSL this is a simple +form as an example of how this could be done. In reality, most +consumers will use the more rigid filters such as FlavorFilter. + +Note: These are hard filters. All capabilities used must be present +or the host will be excluded. If you want soft filters use the weighting +mechanism which is intended for the more touchy-feely capabilities. +""" + +import json + +from nova import exception +from nova import flags +from nova import log as logging +from nova import utils + +LOG = logging.getLogger('nova.scheduler.host_filter') + +FLAGS = flags.FLAGS +flags.DEFINE_string('default_host_filter_driver', + 'nova.scheduler.host_filter.AllHostsFilter', + 'Which driver to use for filtering hosts.') + + +class HostFilter(object): + """Base class for host filter drivers.""" + + def instance_type_to_filter(self, instance_type): + """Convert instance_type into a filter for most common use-case.""" + raise NotImplementedError() + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that fulfill the filter.""" + raise NotImplementedError() + + def _full_name(self): + """module.classname of the filter driver""" + return "%s.%s" % (self.__module__, self.__class__.__name__) + + +class AllHostsFilter(HostFilter): + """NOP host filter driver. Returns all hosts in ZoneManager. + This essentially does what the old Scheduler+Chance used + to give us.""" + + def instance_type_to_filter(self, instance_type): + """Return anything to prevent base-class from raising + exception.""" + return (self._full_name(), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts from ZoneManager list.""" + return [(host, services) + for host, services in zone_manager.service_states.iteritems()] + + +class FlavorFilter(HostFilter): + """HostFilter driver hard-coded to work with flavors.""" + + def instance_type_to_filter(self, instance_type): + """Use instance_type to filter hosts.""" + return (self._full_name(), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can create instance_type.""" + instance_type = query + selected_hosts = [] + for host, services in zone_manager.service_states.iteritems(): + capabilities = services.get('compute', {}) + host_ram_mb = capabilities['host_memory']['free'] + disk_bytes = capabilities['disk']['available'] + if host_ram_mb >= instance_type['memory_mb'] and \ + disk_bytes >= instance_type['local_gb']: + selected_hosts.append((host, capabilities)) + return selected_hosts + +#host entries (currently) are like: +# {'host_name-description': 'Default install of XenServer', +# 'host_hostname': 'xs-mini', +# 'host_memory': {'total': 8244539392, +# 'overhead': 184225792, +# 'free': 3868327936, +# 'free-computed': 3840843776}, +# 'host_other-config': {}, +# 'host_ip_address': '192.168.1.109', +# 'host_cpu_info': {}, +# 'disk': {'available': 32954957824, +# 'total': 50394562560, +# 'used': 17439604736}, +# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', +# 'host_name-label': 'xs-mini'} + +# instance_type table has: +#name = Column(String(255), unique=True) +#memory_mb = Column(Integer) +#vcpus = Column(Integer) +#local_gb = Column(Integer) +#flavorid = Column(Integer, unique=True) +#swap = Column(Integer, nullable=False, default=0) +#rxtx_quota = Column(Integer, nullable=False, default=0) +#rxtx_cap = Column(Integer, nullable=False, default=0) + + +class JsonFilter(HostFilter): + """Host Filter driver to allow simple JSON-based grammar for + selecting hosts.""" + + def _equals(self, args): + """First term is == all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs != rhs: + return False + return True + + def _less_than(self, args): + """First term is < all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs >= rhs: + return False + return True + + def _greater_than(self, args): + """First term is > all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs <= rhs: + return False + return True + + def _in(self, args): + """First term is in set of remaining terms""" + if len(args) < 2: + return False + return args[0] in args[1:] + + def _less_than_equal(self, args): + """First term is <= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs > rhs: + return False + return True + + def _greater_than_equal(self, args): + """First term is >= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs < rhs: + return False + return True + + def _not(self, args): + """Flip each of the arguments.""" + if len(args) == 0: + return False + return [not arg for arg in args] + + def _or(self, args): + """True if any arg is True.""" + return True in args + + def _and(self, args): + """True if all args are True.""" + return False not in args + + commands = { + '=': _equals, + '<': _less_than, + '>': _greater_than, + 'in': _in, + '<=': _less_than_equal, + '>=': _greater_than_equal, + 'not': _not, + 'or': _or, + 'and': _and, + } + + def instance_type_to_filter(self, instance_type): + """Convert instance_type into JSON filter object.""" + required_ram = instance_type['memory_mb'] + required_disk = instance_type['local_gb'] + query = ['and', + ['>=', '$compute.host_memory.free', required_ram], + ['>=', '$compute.disk.available', required_disk] + ] + return (self._full_name(), json.dumps(query)) + + def _parse_string(self, string, host, services): + """Strings prefixed with $ are capability lookups in the + form '$service.capability[.subcap*]'""" + if not string: + return None + if string[0] != '$': + return string + + path = string[1:].split('.') + for item in path: + services = services.get(item, None) + if not services: + return None + return services + + def _process_filter(self, zone_manager, query, host, services): + """Recursively parse the query structure.""" + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_filter(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + return result + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can fulfill filter.""" + expanded = json.loads(query) + hosts = [] + for host, services in zone_manager.service_states.iteritems(): + r = self._process_filter(zone_manager, expanded, host, services) + if isinstance(r, list): + r = True in r + if r: + hosts.append((host, services)) + return hosts + + +DRIVERS = [AllHostsFilter, FlavorFilter, JsonFilter] + + +def choose_driver(driver_name=None): + """Since the caller may specify which driver to use we need + to have an authoritative list of what is permissible. This + function checks the driver name against a predefined set + of acceptable drivers.""" + + if not driver_name: + driver_name = FLAGS.default_host_filter_driver + for driver in DRIVERS: + if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: + return driver() + raise exception.SchedulerHostFilterDriverNotFound(driver_name=driver_name) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py deleted file mode 100644 index 1e294b595..000000000 --- a/nova/scheduler/query.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright (c) 2011 Openstack, LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Query is a plug-in mechanism for requesting instance resources. -Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just -returns the full, unfiltered list of hosts. Flavor is a hard coded -matching mechanism based on flavor criteria and JSON is an ad-hoc -query grammar. - -Note: These are hard filters. All capabilities used must be present -or the host will excluded. If you want soft filters use the weighting -mechanism which is intended for the more touchy-feely capabilities. -""" - -import json - -from nova import exception -from nova import flags -from nova import log as logging -from nova import utils - -LOG = logging.getLogger('nova.scheduler.query') - -FLAGS = flags.FLAGS -flags.DEFINE_string('default_query_engine', - 'nova.scheduler.query.AllHostsQuery', - 'Which query engine to use for filtering hosts.') - - -class Query(object): - """Base class for query plug-ins.""" - - def instance_type_to_query(self, instance_type): - """Convert instance_type into a query for most common use-case.""" - raise NotImplementedError() - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that fulfill the query.""" - raise NotImplementedError() - - def _full_name(self): - """module.classname of the Query object""" - return "%s.%s" % (self.__module__, self.__class__.__name__) - - -class AllHostsQuery(Query): - """NOP query plug-in. Returns all hosts in ZoneManager. - This essentially does what the old Scheduler+Chance used - to give us.""" - - def instance_type_to_query(self, instance_type): - """Return anything to prevent base-class from raising - exception.""" - return (self._full_name(), instance_type) - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts from ZoneManager list.""" - return [(host, services) - for host, services in zone_manager.service_states.iteritems()] - - -class FlavorQuery(Query): - """Query plug-in hard-coded to work with flavors.""" - - def instance_type_to_query(self, instance_type): - """Use instance_type to filter hosts.""" - return (self._full_name(), instance_type) - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can create instance_type.""" - instance_type = query - selected_hosts = [] - for host, services in zone_manager.service_states.iteritems(): - capabilities = services.get('compute', {}) - host_ram_mb = capabilities['host_memory']['free'] - disk_bytes = capabilities['disk']['available'] - if host_ram_mb >= instance_type['memory_mb'] and \ - disk_bytes >= instance_type['local_gb']: - selected_hosts.append((host, capabilities)) - return selected_hosts - -#host entries (currently) are like: -# {'host_name-description': 'Default install of XenServer', -# 'host_hostname': 'xs-mini', -# 'host_memory': {'total': 8244539392, -# 'overhead': 184225792, -# 'free': 3868327936, -# 'free-computed': 3840843776}, -# 'host_other-config': {}, -# 'host_ip_address': '192.168.1.109', -# 'host_cpu_info': {}, -# 'disk': {'available': 32954957824, -# 'total': 50394562560, -# 'used': 17439604736}, -# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', -# 'host_name-label': 'xs-mini'} - -# instance_type table has: -#name = Column(String(255), unique=True) -#memory_mb = Column(Integer) -#vcpus = Column(Integer) -#local_gb = Column(Integer) -#flavorid = Column(Integer, unique=True) -#swap = Column(Integer, nullable=False, default=0) -#rxtx_quota = Column(Integer, nullable=False, default=0) -#rxtx_cap = Column(Integer, nullable=False, default=0) - - -class JsonQuery(Query): - """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" - - def _equals(self, args): - """First term is == all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs != rhs: - return False - return True - - def _less_than(self, args): - """First term is < all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs >= rhs: - return False - return True - - def _greater_than(self, args): - """First term is > all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs <= rhs: - return False - return True - - def _in(self, args): - """First term is in set of remaining terms""" - if len(args) < 2: - return False - return args[0] in args[1:] - - def _less_than_equal(self, args): - """First term is <= all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs > rhs: - return False - return True - - def _greater_than_equal(self, args): - """First term is >= all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs < rhs: - return False - return True - - def _not(self, args): - """Flip each of the arguments.""" - if len(args) == 0: - return False - return [not arg for arg in args] - - def _or(self, args): - """True if any arg is True.""" - return True in args - - def _and(self, args): - """True if all args are True.""" - return False not in args - - commands = { - '=': _equals, - '<': _less_than, - '>': _greater_than, - 'in': _in, - '<=': _less_than_equal, - '>=': _greater_than_equal, - 'not': _not, - 'or': _or, - 'and': _and, - } - - def instance_type_to_query(self, instance_type): - """Convert instance_type into JSON query object.""" - required_ram = instance_type['memory_mb'] - required_disk = instance_type['local_gb'] - query = ['and', - ['>=', '$compute.host_memory.free', required_ram], - ['>=', '$compute.disk.available', required_disk] - ] - return (self._full_name(), json.dumps(query)) - - def _parse_string(self, string, host, services): - """Strings prefixed with $ are capability lookups in the - form '$service.capability[.subcap*]'""" - if not string: - return None - if string[0] != '$': - return string - - path = string[1:].split('.') - for item in path: - services = services.get(item, None) - if not services: - return None - return services - - def _process_query(self, zone_manager, query, host, services): - """Recursively parse the query structure.""" - if len(query) == 0: - return True - cmd = query[0] - method = self.commands[cmd] # Let exception fly. - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_query(zone_manager, arg, host, services) - elif isinstance(arg, basestring): - arg = self._parse_string(arg, host, services) - if arg != None: - cooked_args.append(arg) - result = method(self, cooked_args) - return result - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can fulfill query.""" - expanded = json.loads(query) - hosts = [] - for host, services in zone_manager.service_states.iteritems(): - r = self._process_query(zone_manager, expanded, host, services) - if isinstance(r, list): - r = True in r - if r: - hosts.append((host, services)) - return hosts - - -DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] - - -def choose_driver(driver_name=None): - """Since the caller may specify which driver to use we need - to have an authoritative list of what is permissible. This - function checks the driver name against a predefined set - of acceptable drivers.""" - - if not driver_name: - driver_name = FLAGS.default_query_engine - for driver in DRIVERS: - if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: - return driver() - raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py new file mode 100644 index 000000000..31e40ae1d --- /dev/null +++ b/nova/tests/test_host_filter.py @@ -0,0 +1,208 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler Host Filter Drivers. +""" + +import json + +from nova import exception +from nova import flags +from nova import test +from nova.scheduler import host_filter + +FLAGS = flags.FLAGS + + +class FakeZoneManager: + pass + + +class HostFilterTestCase(test.TestCase): + """Test case for host filter drivers.""" + + def _host_caps(self, multiplier): + # Returns host capabilities in the following way: + # host1 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory': {'total': 100, + 'overhead': 10, + 'free': 10 + multiplier * 10, + 'free-computed': 10 + multiplier * 10}, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk': {'available': 100 + multiplier * 100, + 'total': 1000, + 'used': 0}, + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} + + def setUp(self): + self.old_flag = FLAGS.default_host_filter_driver + FLAGS.default_host_filter_driver = \ + 'nova.scheduler.host_filter.AllHostsFilter' + self.instance_type = dict(name='tiny', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) + + self.zone_manager = FakeZoneManager() + states = {} + for x in xrange(10): + states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} + self.zone_manager.service_states = states + + def tearDown(self): + FLAGS.default_host_filter_driver = self.old_flag + + def test_choose_driver(self): + # Test default driver ... + driver = host_filter.choose_driver() + self.assertEquals(driver._full_name(), + 'nova.scheduler.host_filter.AllHostsFilter') + # Test valid driver ... + driver = host_filter.choose_driver( + 'nova.scheduler.host_filter.FlavorFilter') + self.assertEquals(driver._full_name(), + 'nova.scheduler.host_filter.FlavorFilter') + # Test invalid driver ... + try: + host_filter.choose_driver('does not exist') + self.fail("Should not find driver") + except exception.SchedulerHostFilterDriverNotFound: + pass + + def test_all_host_driver(self): + driver = host_filter.AllHostsFilter() + cooked = driver.instance_type_to_filter(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(10, len(hosts)) + for host, capabilities in hosts: + self.assertTrue(host.startswith('host')) + + def test_flavor_driver(self): + driver = host_filter.FlavorFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.FlavorFilter', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) + + def test_json_driver(self): + driver = host_filter.JsonFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) + + # Try some custom queries + + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(9, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([2, 4, 6, 8, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + # Try some bogus input ... + raw = ['unknown command', ] + cooked = json.dumps(raw) + try: + driver.filter_hosts(self.zone_manager, cooked) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False] + ))) + + try: + driver.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False + )) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$foo', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$.....', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] + ))) + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', {}, ['>', '$missing....foo']] + ))) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py deleted file mode 100644 index 9497a8c96..000000000 --- a/nova/tests/test_query.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Tests For Scheduler Query Drivers -""" - -import json - -from nova import exception -from nova import flags -from nova import test -from nova.scheduler import query - -FLAGS = flags.FLAGS - - -class FakeZoneManager: - pass - - -class QueryTestCase(test.TestCase): - """Test case for query drivers.""" - - def _host_caps(self, multiplier): - # Returns host capabilities in the following way: - # host1 = memory:free 10 (100max) - # disk:available 100 (1000max) - # hostN = memory:free 10 + 10N - # disk:available 100 + 100N - # in other words: hostN has more resources than host0 - # which means ... don't go above 10 hosts. - return {'host_name-description': 'XenServer %s' % multiplier, - 'host_hostname': 'xs-%s' % multiplier, - 'host_memory': {'total': 100, - 'overhead': 10, - 'free': 10 + multiplier * 10, - 'free-computed': 10 + multiplier * 10}, - 'host_other-config': {}, - 'host_ip_address': '192.168.1.%d' % (100 + multiplier), - 'host_cpu_info': {}, - 'disk': {'available': 100 + multiplier * 100, - 'total': 1000, - 'used': 0}, - 'host_uuid': 'xxx-%d' % multiplier, - 'host_name-label': 'xs-%s' % multiplier} - - def setUp(self): - self.old_flag = FLAGS.default_query_engine - FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name='tiny', - memory_mb=50, - vcpus=10, - local_gb=500, - flavorid=1, - swap=500, - rxtx_quota=30000, - rxtx_cap=200) - - self.zone_manager = FakeZoneManager() - states = {} - for x in xrange(10): - states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} - self.zone_manager.service_states = states - - def tearDown(self): - FLAGS.default_query_engine = self.old_flag - - def test_choose_driver(self): - # Test default driver ... - driver = query.choose_driver() - self.assertEquals(driver._full_name(), - 'nova.scheduler.query.AllHostsQuery') - # Test valid driver ... - driver = query.choose_driver('nova.scheduler.query.FlavorQuery') - self.assertEquals(driver._full_name(), - 'nova.scheduler.query.FlavorQuery') - # Test invalid driver ... - try: - query.choose_driver('does not exist') - self.fail("Should not find driver") - except exception.SchedulerQueryDriverNotFound: - pass - - def test_all_host_driver(self): - driver = query.AllHostsQuery() - cooked = driver.instance_type_to_query(self.instance_type) - hosts = driver.filter_hosts(self.zone_manager, cooked) - self.assertEquals(10, len(hosts)) - for host, capabilities in hosts: - self.assertTrue(host.startswith('host')) - - def test_flavor_driver(self): - driver = query.FlavorQuery() - # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_query(self.instance_type) - self.assertEquals('nova.scheduler.query.FlavorQuery', name) - hosts = driver.filter_hosts(self.zone_manager, cooked) - self.assertEquals(6, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - self.assertEquals('host05', just_hosts[0]) - self.assertEquals('host10', just_hosts[5]) - - def test_json_driver(self): - driver = query.JsonQuery() - # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_query(self.instance_type) - self.assertEquals('nova.scheduler.query.JsonQuery', name) - hosts = driver.filter_hosts(self.zone_manager, cooked) - self.assertEquals(6, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - self.assertEquals('host05', just_hosts[0]) - self.assertEquals('host10', just_hosts[5]) - - # Try some custom queries - - raw = ['or', - ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] - ], - ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] - ] - ] - cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(5, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([1, 2, 8, 9, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - raw = ['not', - ['=', '$compute.host_memory.free', 30], - ] - cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(9, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] - cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(5, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([2, 4, 6, 8, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - # Try some bogus input ... - raw = ['unknown command', ] - cooked = json.dumps(raw) - try: - driver.filter_hosts(self.zone_manager, cooked) - self.fail("Should give KeyError") - except KeyError, e: - pass - - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( - ['not', True, False, True, False] - ))) - - try: - driver.filter_hosts(self.zone_manager, json.dumps( - 'not', True, False, True, False - )) - self.fail("Should give KeyError") - except KeyError, e: - pass - - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', '$foo', 100] - ))) - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', '$.....', 100] - ))) - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] - ))) - - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', {}, ['>', '$missing....foo']] - ))) -- cgit From a845a769671066a684e38b82c34ded7ded55eed4 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 6 May 2011 09:06:46 -0700 Subject: revised file docs --- nova/scheduler/host_filter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index aa6101c93..ed8e65c77 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -29,9 +29,11 @@ Since we don't want to get into building full DSL this is a simple form as an example of how this could be done. In reality, most consumers will use the more rigid filters such as FlavorFilter. -Note: These are hard filters. All capabilities used must be present -or the host will be excluded. If you want soft filters use the weighting -mechanism which is intended for the more touchy-feely capabilities. +Note: These are "required" capability filters. These capabilities +used must be present or the host will be excluded. The hosts +returned are then weighed by the Weighted Scheduler. Weights +can take the more esoteric factors into consideration (such as +server affinity and customer separation). """ import json -- cgit