From 01051c71c8f405e9f40a088509d09f171aea1c7d Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 7 Apr 2011 02:07:19 +0900 Subject: Enable RightAWS style signing on server_string without port number portion. --- nova/auth/manager.py | 9 +++++++++ nova/utils.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 486845399..b51cc7f4b 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -315,6 +315,15 @@ class AuthManager(object): LOG.debug('expected_signature: %s', expected_signature) LOG.debug('signature: %s', signature) if signature != expected_signature: + secondary = utils.get_secondary_server_string(server_string) + if secondary is not '': + secondary_signature = signer.Signer( + user.secret.encode()).generate(params, verb, + secondary, path) + LOG.debug('secondary_signature: %s', secondary_signature) + if signature == secondary_signature: + return (user, project) + # NOTE(itoumsn): RightAWS success case. LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.NotAuthorized(_('Signature does not match')) return (user, project) diff --git a/nova/utils.py b/nova/utils.py index 3f6f9fc8a..8fd464452 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -714,3 +714,24 @@ def check_isinstance(obj, cls): raise Exception(_("Expected object of type: %s") % (str(cls))) # TODO(justinsb): Can we make this better?? return cls() # Ugly PyLint hack + +def get_secondary_server_string(str): + """Returns host part only of the given server_string if it's a combination + of host part and port. Otherwise, return null string.""" + + # First of all, exclude pure IPv6 address (w/o port). + if netaddr.valid_ipv6(str): + return '' + + # Next, check if this is IPv6 address with port number combination. + if str.find("]:") != -1: + [address, sep, port] = str.replace('[', '', 1).partition(']:') + return address + + # Third, check if this is a combination of general address and port + if str.find(':') == -1: + return '' + + # This must be a combination of host part and port + [address, sep, port] = str.partition(':') + return address -- cgit From 8decff01ad8ee7e8f7d4103a727321b162280cbe Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 7 Apr 2011 11:42:02 +0900 Subject: pep8 cleanup. --- nova/auth/manager.py | 2 +- nova/utils.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index b51cc7f4b..f1d4a1e39 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -315,7 +315,7 @@ class AuthManager(object): LOG.debug('expected_signature: %s', expected_signature) LOG.debug('signature: %s', signature) if signature != expected_signature: - secondary = utils.get_secondary_server_string(server_string) + secondary = utils.get_secondary_server_string(server_string) if secondary is not '': secondary_signature = signer.Signer( user.secret.encode()).generate(params, verb, diff --git a/nova/utils.py b/nova/utils.py index 8fd464452..3e938247f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -715,6 +715,7 @@ def check_isinstance(obj, cls): # TODO(justinsb): Can we make this better?? return cls() # Ugly PyLint hack + def get_secondary_server_string(str): """Returns host part only of the given server_string if it's a combination of host part and port. Otherwise, return null string.""" @@ -724,14 +725,14 @@ def get_secondary_server_string(str): return '' # Next, check if this is IPv6 address with port number combination. - if str.find("]:") != -1: + if str.find("]:") != -1: [address, sep, port] = str.replace('[', '', 1).partition(']:') - return address + return address # Third, check if this is a combination of general address and port - if str.find(':') == -1: - return '' + if str.find(':') == -1: + return '' # This must be a combination of host part and port [address, sep, port] = str.partition(':') - return address + return address -- cgit From 8c4fa0f16ac170662e113edfdc0f8d3c8863f082 Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 7 Apr 2011 23:48:00 +0900 Subject: Blush up a bit. --- nova/auth/manager.py | 14 +++++++------- nova/utils.py | 8 +++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index f1d4a1e39..c8a3a46a2 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -315,15 +315,15 @@ class AuthManager(object): LOG.debug('expected_signature: %s', expected_signature) LOG.debug('signature: %s', signature) if signature != expected_signature: - secondary = utils.get_secondary_server_string(server_string) - if secondary is not '': - secondary_signature = signer.Signer( + host_only = utils.get_host_only_server_string(server_string) + # If the given server_string contains port num, try without it. + if host_only is not '': + host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, - secondary, path) - LOG.debug('secondary_signature: %s', secondary_signature) - if signature == secondary_signature: + host_only, path) + LOG.debug('host_only_signature: %s', host_only_signature) + if signature == host_only_signature: return (user, project) - # NOTE(itoumsn): RightAWS success case. LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.NotAuthorized(_('Signature does not match')) return (user, project) diff --git a/nova/utils.py b/nova/utils.py index 3e938247f..8b7cbf30c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -716,9 +716,11 @@ def check_isinstance(obj, cls): return cls() # Ugly PyLint hack -def get_secondary_server_string(str): - """Returns host part only of the given server_string if it's a combination - of host part and port. Otherwise, return null string.""" +def get_host_only_server_string(str): + """ + Returns host part only of the given server_string if it's a combination + of host part and port. Otherwise, return null string. + """ # First of all, exclude pure IPv6 address (w/o port). if netaddr.valid_ipv6(str): -- cgit From 32d081f8f0a50b87f7b5d3f5bab4cf4ba92b1b4d Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Wed, 13 Apr 2011 02:11:36 +0900 Subject: Blushed up a little bit. --- nova/auth/manager.py | 2 +- nova/utils.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index c8a3a46a2..01aa87e31 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -317,7 +317,7 @@ class AuthManager(object): if signature != expected_signature: host_only = utils.get_host_only_server_string(server_string) # If the given server_string contains port num, try without it. - if host_only is not '': + if host_only != '': host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, host_only, path) diff --git a/nova/utils.py b/nova/utils.py index 8b7cbf30c..369b5265c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -716,25 +716,25 @@ def check_isinstance(obj, cls): return cls() # Ugly PyLint hack -def get_host_only_server_string(str): +def get_host_only_server_string(server_str): """ Returns host part only of the given server_string if it's a combination of host part and port. Otherwise, return null string. """ # First of all, exclude pure IPv6 address (w/o port). - if netaddr.valid_ipv6(str): + if netaddr.valid_ipv6(server_str): return '' # Next, check if this is IPv6 address with port number combination. - if str.find("]:") != -1: - [address, sep, port] = str.replace('[', '', 1).partition(']:') + if server_str.find("]:") != -1: + [address, sep, port] = server_str.replace('[', '', 1).partition(']:') return address # Third, check if this is a combination of general address and port - if str.find(':') == -1: + if server_str.find(':') == -1: return '' # This must be a combination of host part and port - [address, sep, port] = str.partition(':') + [address, sep, port] = server_str.partition(':') return address -- cgit From b501eb0748ba629a4a742431a42af591f94b6b4c Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Thu, 14 Apr 2011 10:47:37 +0900 Subject: Updated following to RIck's comments. --- nova/auth/manager.py | 15 ++++++++------- nova/utils.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 01aa87e31..dc37ae063 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -300,9 +300,9 @@ class AuthManager(object): if check_type == 's3': sign = signer.Signer(user.secret.encode()) expected_signature = sign.s3_authorization(headers, verb, path) - LOG.debug('user.secret: %s', user.secret) - LOG.debug('expected_signature: %s', expected_signature) - LOG.debug('signature: %s', signature) + LOG.debug(_('user.secret: %s'), user.secret) + LOG.debug(_('expected_signature: %s'), expected_signature) + LOG.debug(_('signature: %s'), signature) if signature != expected_signature: LOG.audit(_("Invalid signature for user %s"), user.name) raise exception.NotAuthorized(_('Signature does not match')) @@ -311,9 +311,9 @@ class AuthManager(object): # secret isn't unicode expected_signature = signer.Signer(user.secret.encode()).generate( params, verb, server_string, path) - LOG.debug('user.secret: %s', user.secret) - LOG.debug('expected_signature: %s', expected_signature) - LOG.debug('signature: %s', signature) + LOG.debug(_('user.secret: %s'), user.secret) + LOG.debug(_('expected_signature: %s'), expected_signature) + LOG.debug(_('signature: %s'), signature) if signature != expected_signature: host_only = utils.get_host_only_server_string(server_string) # If the given server_string contains port num, try without it. @@ -321,7 +321,8 @@ class AuthManager(object): host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, host_only, path) - LOG.debug('host_only_signature: %s', host_only_signature) + LOG.debug(_('host_only_signature: %s'), + host_only_signature) if signature == host_only_signature: return (user, project) LOG.audit(_("Invalid signature for user %s"), user.name) diff --git a/nova/utils.py b/nova/utils.py index 369b5265c..b0f961b90 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -736,5 +736,5 @@ def get_host_only_server_string(server_str): return '' # This must be a combination of host part and port - [address, sep, port] = server_str.partition(':') + (address, port) = server_str.split(':') return address -- cgit From 891eb82afacc10795e4ac05a0c8f817645db85c2 Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Fri, 22 Apr 2011 01:26:59 +0900 Subject: Utility method reworked, etc. --- nova/auth/authutils.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ nova/auth/manager.py | 4 +++- nova/tests/test_auth.py | 24 ++++++++++++++++++++++++ nova/utils.py | 24 ------------------------ 4 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 nova/auth/authutils.py diff --git a/nova/auth/authutils.py b/nova/auth/authutils.py new file mode 100644 index 000000000..429e86ef9 --- /dev/null +++ b/nova/auth/authutils.py @@ -0,0 +1,48 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 NTT DATA CORPORATION. +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Auth module specific utilities and helper functions. +""" + +import netaddr +import string + + +def get_host_only_server_string(server_str): + """ + Returns host part only of the given server_string if it's a combination + of host part and port. Otherwise, return null string. + """ + + # First of all, exclude pure IPv6 address (w/o port). + if netaddr.valid_ipv6(server_str): + return '' + + # Next, check if this is IPv6 address with port number combination. + if server_str.find("]:") != -1: + [address, sep, port] = server_str.replace('[', '', 1).partition(']:') + return address + + # Third, check if this is a combination of general address and port + if server_str.find(':') == -1: + return '' + + # This must be a combination of host part and port + (address, port) = server_str.split(':') + return address diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 06def220a..775b38af1 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -35,6 +35,7 @@ from nova import flags from nova import log as logging from nova import utils from nova.auth import signer +from nova.auth import authutils FLAGS = flags.FLAGS @@ -315,7 +316,8 @@ class AuthManager(object): LOG.debug(_('expected_signature: %s'), expected_signature) LOG.debug(_('signature: %s'), signature) if signature != expected_signature: - host_only = utils.get_host_only_server_string(server_string) + host_only = authutils.get_host_only_server_string( + server_string) # If the given server_string contains port num, try without it. if host_only != '': host_only_signature = signer.Signer( diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index f8a1b1564..3886e9e6b 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -25,6 +25,7 @@ from nova import log as logging from nova import test from nova.auth import manager from nova.api.ec2 import cloud +from nova.auth import authutils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.auth_unittest') @@ -339,6 +340,29 @@ class AuthManagerDbTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' +class AuthManagerUtilTestCase(test.TestCase): + def test_get_host_only_server_string(self): + result = authutils.get_host_only_server_string('::1') + self.assertEqual('', result) + result = authutils.get_host_only_server_string('[::1]:8773') + self.assertEqual('::1', result) + result = authutils.get_host_only_server_string('2001:db8::192.168.1.1') + self.assertEqual('', result) + result = authutils.get_host_only_server_string( + '[2001:db8::192.168.1.1]:8773') + self.assertEqual('2001:db8::192.168.1.1', result) + result = authutils.get_host_only_server_string('192.168.1.1') + self.assertEqual('', result) + result = authutils.get_host_only_server_string('192.168.1.2:8773') + self.assertEqual('192.168.1.2', result) + result = authutils.get_host_only_server_string('192.168.1.3') + self.assertEqual('', result) + result = authutils.get_host_only_server_string('www.example.com:8443') + self.assertEqual('www.example.com', result) + result = authutils.get_host_only_server_string('www.example.com') + self.assertEqual('', result) + + if __name__ == "__main__": # TODO: Implement use_fake as an option unittest.main() diff --git a/nova/utils.py b/nova/utils.py index 5060b1ef6..76cba1a08 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -714,27 +714,3 @@ def check_isinstance(obj, cls): raise Exception(_("Expected object of type: %s") % (str(cls))) # TODO(justinsb): Can we make this better?? return cls() # Ugly PyLint hack - - -def get_host_only_server_string(server_str): - """ - Returns host part only of the given server_string if it's a combination - of host part and port. Otherwise, return null string. - """ - - # First of all, exclude pure IPv6 address (w/o port). - if netaddr.valid_ipv6(server_str): - return '' - - # Next, check if this is IPv6 address with port number combination. - if server_str.find("]:") != -1: - [address, sep, port] = server_str.replace('[', '', 1).partition(']:') - return address - - # Third, check if this is a combination of general address and port - if server_str.find(':') == -1: - return '' - - # This must be a combination of host part and port - (address, port) = server_str.split(':') - return address -- cgit From edc63f9734a4b053a3b57fd6febe94824c83815f Mon Sep 17 00:00:00 2001 From: Masanori Itoh Date: Fri, 22 Apr 2011 21:35:54 +0900 Subject: Rework completed. Added test cases, changed helper method name, etc. --- nova/auth/authutils.py | 48 ------------------------------------ nova/auth/manager.py | 8 +++--- nova/tests/test_auth.py | 64 ++++++++++++++++++++++++++++-------------------- nova/tests/test_utils.py | 25 +++++++++++++++++++ nova/utils.py | 30 +++++++++++++++++++++++ 5 files changed, 95 insertions(+), 80 deletions(-) delete mode 100644 nova/auth/authutils.py diff --git a/nova/auth/authutils.py b/nova/auth/authutils.py deleted file mode 100644 index 429e86ef9..000000000 --- a/nova/auth/authutils.py +++ /dev/null @@ -1,48 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 NTT DATA CORPORATION. -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Auth module specific utilities and helper functions. -""" - -import netaddr -import string - - -def get_host_only_server_string(server_str): - """ - Returns host part only of the given server_string if it's a combination - of host part and port. Otherwise, return null string. - """ - - # First of all, exclude pure IPv6 address (w/o port). - if netaddr.valid_ipv6(server_str): - return '' - - # Next, check if this is IPv6 address with port number combination. - if server_str.find("]:") != -1: - [address, sep, port] = server_str.replace('[', '', 1).partition(']:') - return address - - # Third, check if this is a combination of general address and port - if server_str.find(':') == -1: - return '' - - # This must be a combination of host part and port - (address, port) = server_str.split(':') - return address diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 775b38af1..d42594c84 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -35,7 +35,6 @@ from nova import flags from nova import log as logging from nova import utils from nova.auth import signer -from nova.auth import authutils FLAGS = flags.FLAGS @@ -316,13 +315,12 @@ class AuthManager(object): LOG.debug(_('expected_signature: %s'), expected_signature) LOG.debug(_('signature: %s'), signature) if signature != expected_signature: - host_only = authutils.get_host_only_server_string( - server_string) + (addr_str, port_str) = utils.parse_server_string(server_string) # If the given server_string contains port num, try without it. - if host_only != '': + if port_str != '': host_only_signature = signer.Signer( user.secret.encode()).generate(params, verb, - host_only, path) + addr_str, path) LOG.debug(_('host_only_signature: %s'), host_only_signature) if signature == host_only_signature: diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 3886e9e6b..f02dd94b7 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -25,7 +25,6 @@ from nova import log as logging from nova import test from nova.auth import manager from nova.api.ec2 import cloud -from nova.auth import authutils FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.auth_unittest') @@ -102,9 +101,43 @@ class _AuthManagerBaseTestCase(test.TestCase): self.assertEqual('private-party', u.access) def test_004_signature_is_valid(self): - #self.assertTrue(self.manager.authenticate(**boto.generate_url ...? )) - pass - #raise NotImplementedError + with user_generator(self.manager, name='admin', secret='admin', + access='admin'): + with project_generator(self.manager, name="admin", + manager_user='admin'): + accesskey = 'admin:admin' + expected_result = (self.manager.get_user('admin'), + self.manager.get_project('admin')) + # captured sig and query string using boto 1.9b/euca2ools 1.2 + sig = 'd67Wzd9Bwz8xid9QU+lzWXcF2Y3tRicYABPJgrqfrwM=' + auth_params = {'AWSAccessKeyId': 'admin:admin', + 'Action': 'DescribeAvailabilityZones', + 'SignatureMethod': 'HmacSHA256', + 'SignatureVersion': '2', + 'Timestamp': '2011-04-22T11:29:29', + 'Version': '2009-11-30'} + self.assertTrue(expected_result, self.manager.authenticate( + accesskey, + sig, + auth_params, + 'GET', + '127.0.0.1:8773', + '/services/Cloud/')) + # captured sig and query string using RightAWS 1.10.0 + sig = 'ECYLU6xdFG0ZqRVhQybPJQNJ5W4B9n8fGs6+/fuGD2c=' + auth_params = {'AWSAccessKeyId': 'admin:admin', + 'Action': 'DescribeAvailabilityZones', + 'SignatureMethod': 'HmacSHA256', + 'SignatureVersion': '2', + 'Timestamp': '2011-04-22T11:29:49.000Z', + 'Version': '2008-12-01'} + self.assertTrue(expected_result, self.manager.authenticate( + accesskey, + sig, + auth_params, + 'GET', + '127.0.0.1', + '/services/Cloud')) def test_005_can_get_credentials(self): return @@ -340,29 +373,6 @@ class AuthManagerDbTestCase(_AuthManagerBaseTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' -class AuthManagerUtilTestCase(test.TestCase): - def test_get_host_only_server_string(self): - result = authutils.get_host_only_server_string('::1') - self.assertEqual('', result) - result = authutils.get_host_only_server_string('[::1]:8773') - self.assertEqual('::1', result) - result = authutils.get_host_only_server_string('2001:db8::192.168.1.1') - self.assertEqual('', result) - result = authutils.get_host_only_server_string( - '[2001:db8::192.168.1.1]:8773') - self.assertEqual('2001:db8::192.168.1.1', result) - result = authutils.get_host_only_server_string('192.168.1.1') - self.assertEqual('', result) - result = authutils.get_host_only_server_string('192.168.1.2:8773') - self.assertEqual('192.168.1.2', result) - result = authutils.get_host_only_server_string('192.168.1.3') - self.assertEqual('', result) - result = authutils.get_host_only_server_string('www.example.com:8443') - self.assertEqual('www.example.com', result) - result = authutils.get_host_only_server_string('www.example.com') - self.assertEqual('', result) - - if __name__ == "__main__": # TODO: Implement use_fake as an option unittest.main() diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index e08d229b0..e7b5c826e 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -250,3 +250,28 @@ class GetFromPathTestCase(test.TestCase): input = {'a': [1, 2, {'b': 'b_1'}]} self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) self.assertEquals(['b_1'], f(input, "a/b")) + + +class GenericUtilsTestCase(test.TestCase): + def test_parse_server_string(self): + result = utils.parse_server_string('::1') + self.assertEqual(('::1', ''), result) + result = utils.parse_server_string('[::1]:8773') + self.assertEqual(('::1', '8773'), result) + result = utils.parse_server_string('2001:db8::192.168.1.1') + self.assertEqual(('2001:db8::192.168.1.1', ''), result) + result = utils.parse_server_string('[2001:db8::192.168.1.1]:8773') + self.assertEqual(('2001:db8::192.168.1.1', '8773'), result) + result = utils.parse_server_string('192.168.1.1') + self.assertEqual(('192.168.1.1', ''), result) + result = utils.parse_server_string('192.168.1.2:8773') + self.assertEqual(('192.168.1.2', '8773'), result) + result = utils.parse_server_string('192.168.1.3') + self.assertEqual(('192.168.1.3', ''), result) + result = utils.parse_server_string('www.example.com:8443') + self.assertEqual(('www.example.com', '8443'), result) + result = utils.parse_server_string('www.example.com') + self.assertEqual(('www.example.com', ''), result) + # error case + result = utils.parse_server_string('www.exa:mple.com:8443') + self.assertEqual(('', ''), result) diff --git a/nova/utils.py b/nova/utils.py index b783f6c14..82ab3fc9e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -709,3 +709,33 @@ def check_isinstance(obj, cls): raise Exception(_('Expected object of type: %s') % (str(cls))) # TODO(justinsb): Can we make this better?? return cls() # Ugly PyLint hack + + +def parse_server_string(server_str): + """ + Parses the given server_string and returns a list of host and port. + If it's not a combination of host part and port, the port element + is a null string. If the input is invalid expression, return a null + list. + """ + try: + # First of all, exclude pure IPv6 address (w/o port). + if netaddr.valid_ipv6(server_str): + return (server_str, '') + + # Next, check if this is IPv6 address with a port number combination. + if server_str.find("]:") != -1: + (address, port) = server_str.replace('[', '', 1).split(']:') + return (address, port) + + # Third, check if this is a combination of an address and a port + if server_str.find(':') == -1: + return (server_str, '') + + # This must be a combination of an address and a port + (address, port) = server_str.split(':') + return (address, port) + + except: + LOG.debug(_('Invalid server_string: %s' % server_str)) + return ('', '') -- cgit From f6ee353e388a52e338a7f1a27f924a9a0c60f9a1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 2 May 2011 12:49:10 -0700 Subject: initial pass --- nova/scheduler/query.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 nova/scheduler/query.py diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py new file mode 100644 index 000000000..5379b3d44 --- /dev/null +++ b/nova/scheduler/query.py @@ -0,0 +1,164 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Query is a plug-in mechanism for requesting instance resources. +Three plug-ins are included: PassThru, Flavor & JSON. PassThru just +returns the full, unfiltered list of hosts. Flavor is a hard coded +matching mechanism based on flavor criteria and JSON is an ad-hoc +query grammar. +""" + +from nova import exception +from nova import utils + +class Query: + """Base class for query plug-ins.""" + + def instance_type_to_query(self, instance_type): + """Convert instance_type into a query for most common use-case.""" + raise exception.Error(_("Query driver not specified.")) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that fulfill the query.""" + raise exception.Error(_("Query driver not specified.")) + + +def load_driver(driver_name): + resource = utils.import_class(driver_name) + if type(resource) != types.ClassType: + continue + cls = resource + if not issubclass(cls, Query): + raise exception.Error(_("Query driver does not derive " + "from nova.scheduler.query.Query.")) + return cls + + +class PassThruQuery: + """NOP query plug-in. Returns all hosts in ZoneManager. + This essentially does what the old Scheduler+Chance used + to give us.""" + + def instance_type_to_query(self, instance_type): + """Return anything to prevent base-class from raising + exception.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts from ZoneManager list.""" + hosts = zone_manager.service_state.get('compute', {}) + return [(host, capabilities) + for host, capabilities in hosts.iteritems()] + + +class FlavorQuery: + """Query plug-in hard-coded to work with flavors.""" + + def instance_type_to_query(self, instance_type): + """Use instance_type to filter hosts.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can create instance_type.""" + hosts = zone_manager.service_state.get('compute', {}) + selected_hosts = [] + instance_type = query + for host, capabilities in hosts.iteritems(): + host_ram_mb = capabilities.get['host_memory']['free'] + disk_bytes = capabilities.get['disk']['available'] + if host_ram_mb >= instance_type['memory_mb'] and \ + disk_bytes >= instance_type['local_gb']: + selected_hosts.append((host, capabilities)) + return selected_hosts + +#host entries (currently) are like: +# {'host_name-description': 'Default install of XenServer', +# 'host_hostname': 'xs-mini', +# 'host_memory': {'total': 8244539392, +# 'overhead': 184225792, +# 'free': 3868327936, +# 'free-computed': 3840843776}, +# 'host_other-config': {}, +# 'host_ip_address': '192.168.1.109', +# 'host_cpu_info': {}, +# 'disk': {'available': 32954957824, +# 'total': 50394562560, +# 'used': 17439604736}, +# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', +# 'host_name-label': 'xs-mini'} + +# instance_type table has: +#name = Column(String(255), unique=True) +#memory_mb = Column(Integer) +#vcpus = Column(Integer) +#local_gb = Column(Integer) +#flavorid = Column(Integer, unique=True) +#swap = Column(Integer, nullable=False, default=0) +#rxtx_quota = Column(Integer, nullable=False, default=0) +#rxtx_cap = Column(Integer, nullable=False, default=0) + +class JsonQuery: + """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" + + def _equals(self, args): + pass + + def _less_than(self, args): + pass + + def _greater_than(self, args): + pass + + def _in(self, args): + pass + + def _less_than_equal(self, args): + pass + + def _greater_than_equal(self, args): + pass + + def _not(self, args): + pass + + def _must(self, args): + pass + + def _or(self, args): + pass + + commands = { + '=': _equals, + '<': _less_than, + '>': _greater_than, + 'in': _in, + '<=': _less_than_equal, + '>=': _greater_than_equal, + 'not': _not, + 'must', _must, + 'or', _or, + } + + def instance_type_to_query(self, instance_type): + """Convert instance_type into JSON query object.""" + return (str(self.__class__), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can fulfill query.""" + return [] + + + -- cgit From f50c7260fa0b1dbcfb725bedeb9bb0ed4398f767 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 3 May 2011 15:02:55 -0700 Subject: tests and better driver loading --- nova/exception.py | 8 +++++ nova/scheduler/query.py | 50 ++++++++++++++++----------- nova/tests/test_query.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 nova/tests/test_query.py diff --git a/nova/exception.py b/nova/exception.py index e8444cb14..586ec009b 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -449,6 +449,14 @@ class ZoneNotFound(NotFound): message = _("Zone %(zone_id)s could not be found.") +class SchedulerQueryDriverNotFound(NotFound): + message = _("Scheduler Query Driver %(driver_name)s could not be found.") + + +class BadSchedulerQueryDriver(NotFound): + message = _("Invalid Scheduler Query Driver selected.") + + class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 5379b3d44..4971bc7e4 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -15,39 +15,38 @@ """ Query is a plug-in mechanism for requesting instance resources. -Three plug-ins are included: PassThru, Flavor & JSON. PassThru just +Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. """ from nova import exception +from nova import flags +from nova import log as logging from nova import utils +LOG = logging.getLogger('nova.scheduler.query') + +FLAGS = flags.FLAGS +flags.DEFINE_string('default_query_engine', + 'nova.scheduler.query.AllHostsQuery', + 'Which query engine to use for filtering hosts.') + + class Query: """Base class for query plug-ins.""" def instance_type_to_query(self, instance_type): """Convert instance_type into a query for most common use-case.""" - raise exception.Error(_("Query driver not specified.")) + raise exception.BadSchedulerQueryDriver() def filter_hosts(self, zone_manager, query): """Return a list of hosts that fulfill the query.""" - raise exception.Error(_("Query driver not specified.")) - + raise exception.BadSchedulerQueryDriver() -def load_driver(driver_name): - resource = utils.import_class(driver_name) - if type(resource) != types.ClassType: - continue - cls = resource - if not issubclass(cls, Query): - raise exception.Error(_("Query driver does not derive " - "from nova.scheduler.query.Query.")) - return cls - -class PassThruQuery: +class AllHostsQuery: """NOP query plug-in. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" @@ -59,7 +58,7 @@ class PassThruQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" - hosts = zone_manager.service_state.get('compute', {}) + hosts = zone_manager.service_states.get('compute', {}) return [(host, capabilities) for host, capabilities in hosts.iteritems()] @@ -73,7 +72,7 @@ class FlavorQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" - hosts = zone_manager.service_state.get('compute', {}) + hosts = zone_manager.service_states.get('compute', {}) selected_hosts = [] instance_type = query for host, capabilities in hosts.iteritems(): @@ -148,8 +147,8 @@ class JsonQuery: '<=': _less_than_equal, '>=': _greater_than_equal, 'not': _not, - 'must', _must, - 'or', _or, + 'must': _must, + 'or': _or, } def instance_type_to_query(self, instance_type): @@ -161,4 +160,15 @@ class JsonQuery: return [] - +# Since the caller may specify which driver to use we need +# to have an authoritative list of what is permissible. +DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] + + +def choose_driver(driver_name=None): + if not driver_name: + driver_name = FLAGS.default_query_engine + for driver in DRIVERS: + if str(driver) == driver_name: + return driver + raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py new file mode 100644 index 000000000..0fab91933 --- /dev/null +++ b/nova/tests/test_query.py @@ -0,0 +1,89 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler Query Drivers +""" + +from nova import exception +from nova import flags +from nova import test +from nova.scheduler import query + +FLAGS = flags.FLAGS + +class FakeZoneManager: + pass + +class QueryTestCase(test.TestCase): + """Test case for query drivers.""" + + def _host_caps(self, multiplier): + return {'host_name-description':'XenServer %s' % multiplier, + 'host_hostname':'xs-%s' % multiplier, + 'host_memory':{'total': 100, + 'overhead': 5, + 'free': 5 + multiplier * 5, + 'free-computed': 5 + multiplier * 5}, + 'host_other-config':{}, + 'host_ip_address':'192.168.1.%d' % (100 + multiplier), + 'host_cpu_info':{}, + 'disk':{'available': 100 + multiplier * 100, + 'total': 1000, + 'used': 50 + multiplier * 50}, + 'host_uuid':'xxx-%d' % multiplier, + 'host_name-label':'xs-%s' % multiplier} + + def setUp(self): + self.old_flag = FLAGS.default_query_engine + FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' + self.instance_type = dict(name='tiny', + memory_mb=500, + vcpus=10, + local_gb=50, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) + + hosts = {} + for x in xrange(10): + hosts['host%s' % x] = self._host_caps(x) + + self.zone_manager = FakeZoneManager() + self.zone_manager.service_states = {} + self.zone_manager.service_states['compute'] = hosts + + def tearDown(self): + FLAGS.default_query_engine = self.old_flag + + def test_choose_driver(self): + driver = query.choose_driver() + self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + driver = query.choose_driver('nova.scheduler.query.FlavorQuery') + self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + try: + query.choose_driver('does not exist') + self.fail("Should not find driver") + except exception.SchedulerQueryDriverNotFound: + pass + + def test_all_host_driver(self): + driver = query.AllHostsQuery() + cooked = driver.instance_type_to_query(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(10, len(hosts)) + for host, capabilities in hosts: + self.assertTrue(host.startswith('host')) + -- cgit From ad07b86110b0bcb90f3b71bd423c06a6ff5f922d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 4 May 2011 06:16:33 -0700 Subject: flavor test --- nova/scheduler/query.py | 6 +++--- nova/tests/test_query.py | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 4971bc7e4..1c593f1ba 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -74,10 +74,10 @@ class FlavorQuery: """Return a list of hosts that can create instance_type.""" hosts = zone_manager.service_states.get('compute', {}) selected_hosts = [] - instance_type = query + query_type, instance_type = query for host, capabilities in hosts.iteritems(): - host_ram_mb = capabilities.get['host_memory']['free'] - disk_bytes = capabilities.get['disk']['available'] + host_ram_mb = capabilities['host_memory']['free'] + disk_bytes = capabilities['disk']['available'] if host_ram_mb >= instance_type['memory_mb'] and \ disk_bytes >= instance_type['local_gb']: selected_hosts.append((host, capabilities)) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 0fab91933..d89183e9f 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -30,32 +30,39 @@ class QueryTestCase(test.TestCase): """Test case for query drivers.""" def _host_caps(self, multiplier): + # Returns host capabilities in the following way: + # host0 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. return {'host_name-description':'XenServer %s' % multiplier, 'host_hostname':'xs-%s' % multiplier, 'host_memory':{'total': 100, - 'overhead': 5, - 'free': 5 + multiplier * 5, - 'free-computed': 5 + multiplier * 5}, + 'overhead': 10, + 'free': 10 + multiplier * 10, + 'free-computed': 10 + multiplier * 10}, 'host_other-config':{}, 'host_ip_address':'192.168.1.%d' % (100 + multiplier), 'host_cpu_info':{}, 'disk':{'available': 100 + multiplier * 100, 'total': 1000, - 'used': 50 + multiplier * 50}, + 'used': 0}, 'host_uuid':'xxx-%d' % multiplier, 'host_name-label':'xs-%s' % multiplier} def setUp(self): self.old_flag = FLAGS.default_query_engine FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name='tiny', - memory_mb=500, - vcpus=10, - local_gb=50, - flavorid=1, - swap=500, - rxtx_quota=30000, - rxtx_cap=200) + self.instance_type = dict(name= 'tiny', + memory_mb= 50, + vcpus= 10, + local_gb= 500, + flavorid= 1, + swap= 500, + rxtx_quota= 30000, + rxtx_cap= 200) hosts = {} for x in xrange(10): @@ -69,10 +76,13 @@ class QueryTestCase(test.TestCase): FLAGS.default_query_engine = self.old_flag def test_choose_driver(self): + # Test default driver ... driver = query.choose_driver() self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + # Test valid driver ... driver = query.choose_driver('nova.scheduler.query.FlavorQuery') self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + # Test invalid driver ... try: query.choose_driver('does not exist') self.fail("Should not find driver") @@ -87,3 +97,13 @@ class QueryTestCase(test.TestCase): for host, capabilities in hosts: self.assertTrue(host.startswith('host')) + def test_flavor_driver(self): + driver = query.FlavorQuery() + # filter all hosts that can support 50 ram and 500 disk + cooked = driver.instance_type_to_query(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host4', just_hosts[0]) + self.assertEquals('host9', just_hosts[5]) -- cgit From 772fc58644871f7ee0bb880074c89d79871e197a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 4 May 2011 14:48:23 -0700 Subject: json parser --- nova/scheduler/query.py | 123 ++++++++++++++++++++++++++++++++++++++++------- nova/tests/test_query.py | 25 +++++++--- 2 files changed, 124 insertions(+), 24 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1c593f1ba..1b84e6b1d 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -21,6 +21,8 @@ matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. """ +import json + from nova import exception from nova import flags from nova import log as logging @@ -58,9 +60,8 @@ class AllHostsQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" - hosts = zone_manager.service_states.get('compute', {}) - return [(host, capabilities) - for host, capabilities in hosts.iteritems()] + return [(host, services) + for host, services in zone_manager.service_state.iteritems()] class FlavorQuery: @@ -72,10 +73,10 @@ class FlavorQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" - hosts = zone_manager.service_states.get('compute', {}) + instance_type = query selected_hosts = [] - query_type, instance_type = query - for host, capabilities in hosts.iteritems(): + for host, services in zone_manager.service_states.iteritems(): + capabilities = services.get('compute', {}) host_ram_mb = capabilities['host_memory']['free'] disk_bytes = capabilities['disk']['available'] if host_ram_mb >= instance_type['memory_mb'] and \ @@ -113,31 +114,74 @@ class JsonQuery: """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" def _equals(self, args): - pass + """First term is == all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs != rhs: + return False + return True def _less_than(self, args): - pass + """First term is < all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs >= rhs: + return False + return True def _greater_than(self, args): - pass + """First term is > all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs <= rhs: + return False + return True def _in(self, args): - pass + """First term is in set of remaining terms""" + if len(args) < 2: + return False + return args[0] in args[1:] def _less_than_equal(self, args): - pass + """First term is <= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs > rhs: + return False + return True def _greater_than_equal(self, args): - pass + """First term is >= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs < rhs: + return False + return True def _not(self, args): - pass + if len(args) == 0: + return False + return not args[0] def _must(self, args): - pass + return True def _or(self, args): - pass + return True in args + + def _and(self, args): + return False not in args commands = { '=': _equals, @@ -149,15 +193,60 @@ class JsonQuery: 'not': _not, 'must': _must, 'or': _or, + 'and': _and, } def instance_type_to_query(self, instance_type): """Convert instance_type into JSON query object.""" - return (str(self.__class__), instance_type) + required_ram = instance_type['memory_mb'] + required_disk = instance_type['local_gb'] + query = ['and', + ['>=', '$compute.host_memory.free', required_ram], + ['>=', '$compute.disk.available', required_disk] + ] + return (str(self.__class__), json.dumps(query)) + + def _parse_string(self, string, host, services): + """Strings prefixed with $ are capability lookups in the + form '$service.capability[.subcap*]'""" + if not string: + return None + if string[0] != '$': + return string + + path = string[1:].split('.') + for item in path: + services = services.get(item, None) + if not services: + return None + return services + + def _process_query(self, zone_manager, query, host, services): + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_query(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + print "*** %s %s = %s" % (cmd, cooked_args, result) + return result def filter_hosts(self, zone_manager, query): """Return a list of hosts that can fulfill query.""" - return [] + expanded = json.loads(query) + hosts = [] + for host, services in zone_manager.service_states.iteritems(): + print "-----" + if self._process_query(zone_manager, expanded, host, services): + hosts.append((host, services)) + return hosts # Since the caller may specify which driver to use we need diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index d89183e9f..7a036a4d8 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -64,13 +64,11 @@ class QueryTestCase(test.TestCase): rxtx_quota= 30000, rxtx_cap= 200) - hosts = {} - for x in xrange(10): - hosts['host%s' % x] = self._host_caps(x) - self.zone_manager = FakeZoneManager() - self.zone_manager.service_states = {} - self.zone_manager.service_states['compute'] = hosts + states = {} + for x in xrange(10): + states['host%s' % x] = {'compute': self._host_caps(x)} + self.zone_manager.service_states = states def tearDown(self): FLAGS.default_query_engine = self.old_flag @@ -100,7 +98,20 @@ class QueryTestCase(test.TestCase): def test_flavor_driver(self): driver = query.FlavorQuery() # filter all hosts that can support 50 ram and 500 disk - cooked = driver.instance_type_to_query(self.instance_type) + name, cooked = driver.instance_type_to_query(self.instance_type) + self.assertEquals('nova.scheduler.query.FlavorQuery', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host4', just_hosts[0]) + self.assertEquals('host9', just_hosts[5]) + + def test_json_driver(self): + driver = query.JsonQuery() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_query(self.instance_type) + self.assertEquals('nova.scheduler.query.JsonQuery', name) hosts = driver.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] -- cgit From cc18ff47ff41ddefd7a31db5b772d55b2e312e8c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 04:57:25 -0700 Subject: and or test --- nova/scheduler/query.py | 10 +++++----- nova/tests/test_query.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1b84e6b1d..3233cc0d8 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -19,6 +19,10 @@ Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc query grammar. + +Note: These are hard filters. All capabilities used must be present +or the host will excluded. If you want soft filters use the weighting +mechanism which is intended for the more touchy-feely capabilities. """ import json @@ -61,7 +65,7 @@ class AllHostsQuery: def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" return [(host, services) - for host, services in zone_manager.service_state.iteritems()] + for host, services in zone_manager.service_states.iteritems()] class FlavorQuery: @@ -174,9 +178,6 @@ class JsonQuery: return False return not args[0] - def _must(self, args): - return True - def _or(self, args): return True in args @@ -191,7 +192,6 @@ class JsonQuery: '<=': _less_than_equal, '>=': _greater_than_equal, 'not': _not, - 'must': _must, 'or': _or, 'and': _and, } diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 7a036a4d8..896b2364d 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -16,6 +16,8 @@ Tests For Scheduler Query Drivers """ +import json + from nova import exception from nova import flags from nova import test @@ -118,3 +120,27 @@ class QueryTestCase(test.TestCase): just_hosts.sort() self.assertEquals('host4', just_hosts[0]) self.assertEquals('host9', just_hosts[5]) + + # Try some custom queries + + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host0', just_hosts[0]) + self.assertEquals('host1', just_hosts[1]) + self.assertEquals('host7', just_hosts[2]) + self.assertEquals('host8', just_hosts[3]) + self.assertEquals('host9', just_hosts[4]) -- cgit From 5f4fc98c9648fd3f124819e0f4a26cb1d2d7f0e8 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 04:59:26 -0700 Subject: and or test --- nova/tests/test_query.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 896b2364d..90ae80dc0 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -139,8 +139,5 @@ class QueryTestCase(test.TestCase): self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host0', just_hosts[0]) - self.assertEquals('host1', just_hosts[1]) - self.assertEquals('host7', just_hosts[2]) - self.assertEquals('host8', just_hosts[3]) - self.assertEquals('host9', just_hosts[4]) + for index, host in zip([0, 1, 7, 8, 9], just_hosts): + self.assertEquals('host%d' % index, host) -- cgit From 5a066cf5c2b952371eea753dcd0f95f917d08744 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 05:29:31 -0700 Subject: not = --- nova/scheduler/query.py | 7 +++++-- nova/tests/test_query.py | 29 +++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 3233cc0d8..0279efc9e 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -176,7 +176,7 @@ class JsonQuery: def _not(self, args): if len(args) == 0: return False - return not args[0] + return [not arg for arg in args] def _or(self, args): return True in args @@ -244,7 +244,10 @@ class JsonQuery: hosts = [] for host, services in zone_manager.service_states.iteritems(): print "-----" - if self._process_query(zone_manager, expanded, host, services): + r = self._process_query(zone_manager, expanded, host, services) + if isinstance(r, list): + r = True in r + if r: hosts.append((host, services)) return hosts diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 90ae80dc0..9bfc83b75 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -33,7 +33,7 @@ class QueryTestCase(test.TestCase): def _host_caps(self, multiplier): # Returns host capabilities in the following way: - # host0 = memory:free 10 (100max) + # host1 = memory:free 10 (100max) # disk:available 100 (1000max) # hostN = memory:free 10 + 10N # disk:available 100 + 100N @@ -69,7 +69,7 @@ class QueryTestCase(test.TestCase): self.zone_manager = FakeZoneManager() states = {} for x in xrange(10): - states['host%s' % x] = {'compute': self._host_caps(x)} + states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} self.zone_manager.service_states = states def tearDown(self): @@ -106,8 +106,8 @@ class QueryTestCase(test.TestCase): self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host4', just_hosts[0]) - self.assertEquals('host9', just_hosts[5]) + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) def test_json_driver(self): driver = query.JsonQuery() @@ -118,8 +118,8 @@ class QueryTestCase(test.TestCase): self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - self.assertEquals('host4', just_hosts[0]) - self.assertEquals('host9', just_hosts[5]) + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) # Try some custom queries @@ -139,5 +139,18 @@ class QueryTestCase(test.TestCase): self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() - for index, host in zip([0, 1, 7, 8, 9], just_hosts): - self.assertEquals('host%d' % index, host) + for index, host in zip([1, 2, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(9, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + -- cgit From 4b03036214cff2fcaad079f84605737d7f9dc711 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 05:30:58 -0700 Subject: not = --- nova/tests/test_query.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 9bfc83b75..78e8ee9de 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -153,4 +153,14 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100]] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([2, 4, 6, 8, 10], just_hosts): + self.assertEquals('host%02d' % index, host) -- cgit From 31c7b40b89c076a32d8105219d623320c58b8166 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 06:01:56 -0700 Subject: pep8 --- nova/scheduler/query.py | 43 +++++++++++++++-------------- nova/tests/test_query.py | 71 ++++++++++++++++++++++++------------------------ 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 0279efc9e..4332f43b0 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -114,6 +114,7 @@ class FlavorQuery: #rxtx_quota = Column(Integer, nullable=False, default=0) #rxtx_cap = Column(Integer, nullable=False, default=0) + class JsonQuery: """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" @@ -162,7 +163,7 @@ class JsonQuery: if lhs > rhs: return False return True - + def _greater_than_equal(self, args): """First term is >= all the other terms.""" if len(args) < 2: @@ -180,10 +181,10 @@ class JsonQuery: def _or(self, args): return True in args - + def _and(self, args): return False not in args - + commands = { '=': _equals, '<': _less_than, @@ -200,7 +201,7 @@ class JsonQuery: """Convert instance_type into JSON query object.""" required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] - query = ['and', + query = ['and', ['>=', '$compute.host_memory.free', required_ram], ['>=', '$compute.disk.available', required_disk] ] @@ -219,24 +220,24 @@ class JsonQuery: services = services.get(item, None) if not services: return None - return services + return services def _process_query(self, zone_manager, query, host, services): - if len(query) == 0: - return True - cmd = query[0] - method = self.commands[cmd] # Let exception fly. - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_query(zone_manager, arg, host, services) - elif isinstance(arg, basestring): - arg = self._parse_string(arg, host, services) - if arg != None: - cooked_args.append(arg) - result = method(self, cooked_args) - print "*** %s %s = %s" % (cmd, cooked_args, result) - return result + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_query(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + print "*** %s %s = %s" % (cmd, cooked_args, result) + return result def filter_hosts(self, zone_manager, query): """Return a list of hosts that can fulfill query.""" @@ -253,7 +254,7 @@ class JsonQuery: # Since the caller may specify which driver to use we need -# to have an authoritative list of what is permissible. +# to have an authoritative list of what is permissible. DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 78e8ee9de..8b894a4e6 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -1,4 +1,4 @@ -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -25,9 +25,11 @@ from nova.scheduler import query FLAGS = flags.FLAGS + class FakeZoneManager: pass + class QueryTestCase(test.TestCase): """Test case for query drivers.""" @@ -39,32 +41,32 @@ class QueryTestCase(test.TestCase): # disk:available 100 + 100N # in other words: hostN has more resources than host0 # which means ... don't go above 10 hosts. - return {'host_name-description':'XenServer %s' % multiplier, - 'host_hostname':'xs-%s' % multiplier, - 'host_memory':{'total': 100, + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory': {'total': 100, 'overhead': 10, 'free': 10 + multiplier * 10, 'free-computed': 10 + multiplier * 10}, - 'host_other-config':{}, - 'host_ip_address':'192.168.1.%d' % (100 + multiplier), - 'host_cpu_info':{}, - 'disk':{'available': 100 + multiplier * 100, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk': {'available': 100 + multiplier * 100, 'total': 1000, 'used': 0}, - 'host_uuid':'xxx-%d' % multiplier, - 'host_name-label':'xs-%s' % multiplier} + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} def setUp(self): self.old_flag = FLAGS.default_query_engine FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name= 'tiny', - memory_mb= 50, - vcpus= 10, - local_gb= 500, - flavorid= 1, - swap= 500, - rxtx_quota= 30000, - rxtx_cap= 200) + self.instance_type = dict(name='tiny', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) self.zone_manager = FakeZoneManager() states = {} @@ -123,16 +125,16 @@ class QueryTestCase(test.TestCase): # Try some custom queries - raw = ['or', - ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] - ], - ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] - ] - ] + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -141,10 +143,10 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - - raw = ['not', - ['=', '$compute.host_memory.free', 30], - ] + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -153,8 +155,8 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100]] + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -163,4 +165,3 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([2, 4, 6, 8, 10], just_hosts): self.assertEquals('host%02d' % index, host) - -- cgit From 37954e665b874ac2358921d175b30617f456c007 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 07:35:44 -0700 Subject: flipped service_state in ZoneManager and fixed tests --- nova/scheduler/api.py | 8 +++----- nova/scheduler/manager.py | 7 +++---- nova/scheduler/zone_manager.py | 20 +++++++++----------- nova/tests/api/openstack/test_zones.py | 2 +- nova/tests/test_zones.py | 18 ++++++------------ 5 files changed, 22 insertions(+), 33 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6bb3bf3cd..816ae5513 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -76,11 +76,9 @@ def zone_update(context, zone_id, data): return db.zone_update(context, zone_id, data) -def get_zone_capabilities(context, service=None): - """Returns a dict of key, value capabilities for this zone, - or for a particular class of services running in this zone.""" - return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) +def get_zone_capabilities(context): + """Returns a dict of key, value capabilities for this zone.""" + return _call_scheduler('get_zone_capabilities', context=context) def update_service_capabilities(context, service_name, host, capabilities): diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 7d62cfc4e..55cd7208b 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -60,10 +60,9 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() - def get_zone_capabilities(self, context=None, service=None): - """Get the normalized set of capabilites for this zone, - or for a particular service.""" - return self.zone_manager.get_zone_capabilities(context, service) + def get_zone_capabilities(self, context=None): + """Get the normalized set of capabilites for this zone.""" + return self.zone_manager.get_zone_capabilities(context) def update_service_capabilities(self, context=None, service_name=None, host=None, capabilities={}): diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 198f9d4cc..3ddf6f3c3 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -106,28 +106,26 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} # { : ZoneState } - self.service_states = {} # { : { : { cap k : v }}} + self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" return [zone.to_dict() for zone in self.zone_states.values()] - def get_zone_capabilities(self, context, service=None): + def get_zone_capabilities(self, context): """Roll up all the individual host info to generic 'service' capabilities. Each capability is aggregated into _min and _max values.""" - service_dict = self.service_states - if service: - service_dict = {service: self.service_states.get(service, {})} + hosts_dict = self.service_states # TODO(sandy) - be smarter about fabricating this structure. # But it's likely to change once we understand what the Best-Match # code will need better. combined = {} # { _ : (min, max), ... } - for service_name, host_dict in service_dict.iteritems(): - for host, caps_dict in host_dict.iteritems(): - for cap, value in caps_dict.iteritems(): + for host, host_dict in hosts_dict.iteritems(): + for service_name, service_dict in host_dict.iteritems(): + for cap, value in service_dict.iteritems(): key = "%s_%s" % (service_name, cap) min_value, max_value = combined.get(key, (value, value)) min_value = min(min_value, value) @@ -171,6 +169,6 @@ class ZoneManager(object): """Update the per-service capabilities based on this notification.""" logging.debug(_("Received %(service_name)s service update from " "%(host)s: %(capabilities)s") % locals()) - service_caps = self.service_states.get(service_name, {}) - service_caps[host] = capabilities - self.service_states[service_name] = service_caps + service_caps = self.service_states.get(host, {}) + service_caps[service_name] = capabilities + self.service_states[host] = service_caps diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index a3f191aaa..5d5799b59 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -75,7 +75,7 @@ def zone_get_all_db(context): ] -def zone_capabilities(method, context, params): +def zone_capabilities(method, context): return dict() diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 688dc704d..e132809dc 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -78,38 +78,32 @@ class ZoneManagerTestCase(test.TestCase): def test_service_capabilities(self): zm = zone_manager.ZoneManager() - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, {}) zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2))) zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc10_a=(99, 99), svc10_b=(99, 99))) zm.update_service_capabilities("svc1", "host3", dict(c=5)) - caps = zm.get_zone_capabilities(self, None) + caps = zm.get_zone_capabilities(None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) - caps = zm.get_zone_capabilities(self, 'svc1') - self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), - svc1_c=(5, 5))) - caps = zm.get_zone_capabilities(self, 'svc10') - self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) - def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 27f99a14a381062f87fbbb65d5aaa07914aa82c0 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 08:13:22 -0700 Subject: print statements removed --- nova/scheduler/query.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 4332f43b0..1a6705b42 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -236,7 +236,6 @@ class JsonQuery: if arg != None: cooked_args.append(arg) result = method(self, cooked_args) - print "*** %s %s = %s" % (cmd, cooked_args, result) return result def filter_hosts(self, zone_manager, query): @@ -244,7 +243,6 @@ class JsonQuery: expanded = json.loads(query) hosts = [] for host, services in zone_manager.service_states.iteritems(): - print "-----" r = self._process_query(zone_manager, expanded, host, services) if isinstance(r, list): r = True in r -- cgit From 9797bda1baacf5d90d6ce678c20396bacb513e37 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 10:50:59 -0700 Subject: merge prop fixes --- nova/exception.py | 4 ---- nova/scheduler/query.py | 37 ++++++++++++++++++++++++------------- nova/tests/test_query.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index eeeed7952..50f50de9d 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -461,10 +461,6 @@ class SchedulerQueryDriverNotFound(NotFound): message = _("Scheduler Query Driver %(driver_name)s could not be found.") -class BadSchedulerQueryDriver(NotFound): - message = _("Invalid Scheduler Query Driver selected.") - - class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py index 1a6705b42..1e294b595 100644 --- a/nova/scheduler/query.py +++ b/nova/scheduler/query.py @@ -40,19 +40,23 @@ flags.DEFINE_string('default_query_engine', 'Which query engine to use for filtering hosts.') -class Query: +class Query(object): """Base class for query plug-ins.""" def instance_type_to_query(self, instance_type): """Convert instance_type into a query for most common use-case.""" - raise exception.BadSchedulerQueryDriver() + raise NotImplementedError() def filter_hosts(self, zone_manager, query): """Return a list of hosts that fulfill the query.""" - raise exception.BadSchedulerQueryDriver() + raise NotImplementedError() + def _full_name(self): + """module.classname of the Query object""" + return "%s.%s" % (self.__module__, self.__class__.__name__) -class AllHostsQuery: + +class AllHostsQuery(Query): """NOP query plug-in. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used to give us.""" @@ -60,7 +64,7 @@ class AllHostsQuery: def instance_type_to_query(self, instance_type): """Return anything to prevent base-class from raising exception.""" - return (str(self.__class__), instance_type) + return (self._full_name(), instance_type) def filter_hosts(self, zone_manager, query): """Return a list of hosts from ZoneManager list.""" @@ -68,12 +72,12 @@ class AllHostsQuery: for host, services in zone_manager.service_states.iteritems()] -class FlavorQuery: +class FlavorQuery(Query): """Query plug-in hard-coded to work with flavors.""" def instance_type_to_query(self, instance_type): """Use instance_type to filter hosts.""" - return (str(self.__class__), instance_type) + return (self._full_name(), instance_type) def filter_hosts(self, zone_manager, query): """Return a list of hosts that can create instance_type.""" @@ -115,7 +119,7 @@ class FlavorQuery: #rxtx_cap = Column(Integer, nullable=False, default=0) -class JsonQuery: +class JsonQuery(Query): """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" def _equals(self, args): @@ -175,14 +179,17 @@ class JsonQuery: return True def _not(self, args): + """Flip each of the arguments.""" if len(args) == 0: return False return [not arg for arg in args] def _or(self, args): + """True if any arg is True.""" return True in args def _and(self, args): + """True if all args are True.""" return False not in args commands = { @@ -205,7 +212,7 @@ class JsonQuery: ['>=', '$compute.host_memory.free', required_ram], ['>=', '$compute.disk.available', required_disk] ] - return (str(self.__class__), json.dumps(query)) + return (self._full_name(), json.dumps(query)) def _parse_string(self, string, host, services): """Strings prefixed with $ are capability lookups in the @@ -223,6 +230,7 @@ class JsonQuery: return services def _process_query(self, zone_manager, query, host, services): + """Recursively parse the query structure.""" if len(query) == 0: return True cmd = query[0] @@ -251,15 +259,18 @@ class JsonQuery: return hosts -# Since the caller may specify which driver to use we need -# to have an authoritative list of what is permissible. DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] def choose_driver(driver_name=None): + """Since the caller may specify which driver to use we need + to have an authoritative list of what is permissible. This + function checks the driver name against a predefined set + of acceptable drivers.""" + if not driver_name: driver_name = FLAGS.default_query_engine for driver in DRIVERS: - if str(driver) == driver_name: - return driver + if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: + return driver() raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py index 8b894a4e6..9497a8c96 100644 --- a/nova/tests/test_query.py +++ b/nova/tests/test_query.py @@ -80,10 +80,12 @@ class QueryTestCase(test.TestCase): def test_choose_driver(self): # Test default driver ... driver = query.choose_driver() - self.assertEquals(str(driver), 'nova.scheduler.query.AllHostsQuery') + self.assertEquals(driver._full_name(), + 'nova.scheduler.query.AllHostsQuery') # Test valid driver ... driver = query.choose_driver('nova.scheduler.query.FlavorQuery') - self.assertEquals(str(driver), 'nova.scheduler.query.FlavorQuery') + self.assertEquals(driver._full_name(), + 'nova.scheduler.query.FlavorQuery') # Test invalid driver ... try: query.choose_driver('does not exist') @@ -165,3 +167,40 @@ class QueryTestCase(test.TestCase): just_hosts.sort() for index, host in zip([2, 4, 6, 8, 10], just_hosts): self.assertEquals('host%02d' % index, host) + + # Try some bogus input ... + raw = ['unknown command', ] + cooked = json.dumps(raw) + try: + driver.filter_hosts(self.zone_manager, cooked) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False] + ))) + + try: + driver.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False + )) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$foo', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$.....', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] + ))) + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', {}, ['>', '$missing....foo']] + ))) -- cgit From 2cfa0ab48e981e31484287936fbc738d6073f473 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 5 May 2011 15:35:34 -0400 Subject: removing rogue TimeoutException --- nova/api/openstack/servers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 3cf78e32c..547310613 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -594,10 +594,7 @@ class ControllerV10(Controller): def _parse_update(self, context, server_id, inst_dict, update_dict): if 'adminPass' in inst_dict['server']: update_dict['admin_pass'] = inst_dict['server']['adminPass'] - try: - self.compute_api.set_admin_password(context, server_id) - except exception.TimeoutException: - return exc.HTTPRequestTimeout() + self.compute_api.set_admin_password(context, server_id) def _action_rebuild(self, info, request, instance_id): context = request.environ['nova.context'] -- cgit From a5e7d039ec9ee9528186fa011021da00d809e683 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 5 May 2011 18:09:11 -0700 Subject: terminology: no more plug-ins or queries. They are host filters and drivers. --- nova/exception.py | 5 +- nova/scheduler/host_filter.py | 286 +++++++++++++++++++++++++++++++++++++++++ nova/scheduler/query.py | 276 --------------------------------------- nova/tests/test_host_filter.py | 208 ++++++++++++++++++++++++++++++ nova/tests/test_query.py | 206 ----------------------------- 5 files changed, 497 insertions(+), 484 deletions(-) create mode 100644 nova/scheduler/host_filter.py delete mode 100644 nova/scheduler/query.py create mode 100644 nova/tests/test_host_filter.py delete mode 100644 nova/tests/test_query.py diff --git a/nova/exception.py b/nova/exception.py index 50f50de9d..9905fb19b 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -457,8 +457,9 @@ class ZoneNotFound(NotFound): message = _("Zone %(zone_id)s could not be found.") -class SchedulerQueryDriverNotFound(NotFound): - message = _("Scheduler Query Driver %(driver_name)s could not be found.") +class SchedulerHostFilterDriverNotFound(NotFound): + message = _("Scheduler Host Filter Driver %(driver_name)s could" + " not be found.") class InstanceMetadataNotFound(NotFound): diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py new file mode 100644 index 000000000..aa6101c93 --- /dev/null +++ b/nova/scheduler/host_filter.py @@ -0,0 +1,286 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Host Filter is a driver mechanism for requesting instance resources. +Three drivers are included: AllHosts, Flavor & JSON. AllHosts just +returns the full, unfiltered list of hosts. Flavor is a hard coded +matching mechanism based on flavor criteria and JSON is an ad-hoc +filter grammar. + +Why JSON? The requests for instances may come in through the +REST interface from a user or a parent Zone. +Currently Flavors and/or InstanceTypes are used for +specifing the type of instance desired. Specific Nova users have +noted a need for a more expressive way of specifying instances. +Since we don't want to get into building full DSL this is a simple +form as an example of how this could be done. In reality, most +consumers will use the more rigid filters such as FlavorFilter. + +Note: These are hard filters. All capabilities used must be present +or the host will be excluded. If you want soft filters use the weighting +mechanism which is intended for the more touchy-feely capabilities. +""" + +import json + +from nova import exception +from nova import flags +from nova import log as logging +from nova import utils + +LOG = logging.getLogger('nova.scheduler.host_filter') + +FLAGS = flags.FLAGS +flags.DEFINE_string('default_host_filter_driver', + 'nova.scheduler.host_filter.AllHostsFilter', + 'Which driver to use for filtering hosts.') + + +class HostFilter(object): + """Base class for host filter drivers.""" + + def instance_type_to_filter(self, instance_type): + """Convert instance_type into a filter for most common use-case.""" + raise NotImplementedError() + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that fulfill the filter.""" + raise NotImplementedError() + + def _full_name(self): + """module.classname of the filter driver""" + return "%s.%s" % (self.__module__, self.__class__.__name__) + + +class AllHostsFilter(HostFilter): + """NOP host filter driver. Returns all hosts in ZoneManager. + This essentially does what the old Scheduler+Chance used + to give us.""" + + def instance_type_to_filter(self, instance_type): + """Return anything to prevent base-class from raising + exception.""" + return (self._full_name(), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts from ZoneManager list.""" + return [(host, services) + for host, services in zone_manager.service_states.iteritems()] + + +class FlavorFilter(HostFilter): + """HostFilter driver hard-coded to work with flavors.""" + + def instance_type_to_filter(self, instance_type): + """Use instance_type to filter hosts.""" + return (self._full_name(), instance_type) + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can create instance_type.""" + instance_type = query + selected_hosts = [] + for host, services in zone_manager.service_states.iteritems(): + capabilities = services.get('compute', {}) + host_ram_mb = capabilities['host_memory']['free'] + disk_bytes = capabilities['disk']['available'] + if host_ram_mb >= instance_type['memory_mb'] and \ + disk_bytes >= instance_type['local_gb']: + selected_hosts.append((host, capabilities)) + return selected_hosts + +#host entries (currently) are like: +# {'host_name-description': 'Default install of XenServer', +# 'host_hostname': 'xs-mini', +# 'host_memory': {'total': 8244539392, +# 'overhead': 184225792, +# 'free': 3868327936, +# 'free-computed': 3840843776}, +# 'host_other-config': {}, +# 'host_ip_address': '192.168.1.109', +# 'host_cpu_info': {}, +# 'disk': {'available': 32954957824, +# 'total': 50394562560, +# 'used': 17439604736}, +# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', +# 'host_name-label': 'xs-mini'} + +# instance_type table has: +#name = Column(String(255), unique=True) +#memory_mb = Column(Integer) +#vcpus = Column(Integer) +#local_gb = Column(Integer) +#flavorid = Column(Integer, unique=True) +#swap = Column(Integer, nullable=False, default=0) +#rxtx_quota = Column(Integer, nullable=False, default=0) +#rxtx_cap = Column(Integer, nullable=False, default=0) + + +class JsonFilter(HostFilter): + """Host Filter driver to allow simple JSON-based grammar for + selecting hosts.""" + + def _equals(self, args): + """First term is == all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs != rhs: + return False + return True + + def _less_than(self, args): + """First term is < all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs >= rhs: + return False + return True + + def _greater_than(self, args): + """First term is > all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs <= rhs: + return False + return True + + def _in(self, args): + """First term is in set of remaining terms""" + if len(args) < 2: + return False + return args[0] in args[1:] + + def _less_than_equal(self, args): + """First term is <= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs > rhs: + return False + return True + + def _greater_than_equal(self, args): + """First term is >= all the other terms.""" + if len(args) < 2: + return False + lhs = args[0] + for rhs in args[1:]: + if lhs < rhs: + return False + return True + + def _not(self, args): + """Flip each of the arguments.""" + if len(args) == 0: + return False + return [not arg for arg in args] + + def _or(self, args): + """True if any arg is True.""" + return True in args + + def _and(self, args): + """True if all args are True.""" + return False not in args + + commands = { + '=': _equals, + '<': _less_than, + '>': _greater_than, + 'in': _in, + '<=': _less_than_equal, + '>=': _greater_than_equal, + 'not': _not, + 'or': _or, + 'and': _and, + } + + def instance_type_to_filter(self, instance_type): + """Convert instance_type into JSON filter object.""" + required_ram = instance_type['memory_mb'] + required_disk = instance_type['local_gb'] + query = ['and', + ['>=', '$compute.host_memory.free', required_ram], + ['>=', '$compute.disk.available', required_disk] + ] + return (self._full_name(), json.dumps(query)) + + def _parse_string(self, string, host, services): + """Strings prefixed with $ are capability lookups in the + form '$service.capability[.subcap*]'""" + if not string: + return None + if string[0] != '$': + return string + + path = string[1:].split('.') + for item in path: + services = services.get(item, None) + if not services: + return None + return services + + def _process_filter(self, zone_manager, query, host, services): + """Recursively parse the query structure.""" + if len(query) == 0: + return True + cmd = query[0] + method = self.commands[cmd] # Let exception fly. + cooked_args = [] + for arg in query[1:]: + if isinstance(arg, list): + arg = self._process_filter(zone_manager, arg, host, services) + elif isinstance(arg, basestring): + arg = self._parse_string(arg, host, services) + if arg != None: + cooked_args.append(arg) + result = method(self, cooked_args) + return result + + def filter_hosts(self, zone_manager, query): + """Return a list of hosts that can fulfill filter.""" + expanded = json.loads(query) + hosts = [] + for host, services in zone_manager.service_states.iteritems(): + r = self._process_filter(zone_manager, expanded, host, services) + if isinstance(r, list): + r = True in r + if r: + hosts.append((host, services)) + return hosts + + +DRIVERS = [AllHostsFilter, FlavorFilter, JsonFilter] + + +def choose_driver(driver_name=None): + """Since the caller may specify which driver to use we need + to have an authoritative list of what is permissible. This + function checks the driver name against a predefined set + of acceptable drivers.""" + + if not driver_name: + driver_name = FLAGS.default_host_filter_driver + for driver in DRIVERS: + if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: + return driver() + raise exception.SchedulerHostFilterDriverNotFound(driver_name=driver_name) diff --git a/nova/scheduler/query.py b/nova/scheduler/query.py deleted file mode 100644 index 1e294b595..000000000 --- a/nova/scheduler/query.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright (c) 2011 Openstack, LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Query is a plug-in mechanism for requesting instance resources. -Three plug-ins are included: AllHosts, Flavor & JSON. AllHosts just -returns the full, unfiltered list of hosts. Flavor is a hard coded -matching mechanism based on flavor criteria and JSON is an ad-hoc -query grammar. - -Note: These are hard filters. All capabilities used must be present -or the host will excluded. If you want soft filters use the weighting -mechanism which is intended for the more touchy-feely capabilities. -""" - -import json - -from nova import exception -from nova import flags -from nova import log as logging -from nova import utils - -LOG = logging.getLogger('nova.scheduler.query') - -FLAGS = flags.FLAGS -flags.DEFINE_string('default_query_engine', - 'nova.scheduler.query.AllHostsQuery', - 'Which query engine to use for filtering hosts.') - - -class Query(object): - """Base class for query plug-ins.""" - - def instance_type_to_query(self, instance_type): - """Convert instance_type into a query for most common use-case.""" - raise NotImplementedError() - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that fulfill the query.""" - raise NotImplementedError() - - def _full_name(self): - """module.classname of the Query object""" - return "%s.%s" % (self.__module__, self.__class__.__name__) - - -class AllHostsQuery(Query): - """NOP query plug-in. Returns all hosts in ZoneManager. - This essentially does what the old Scheduler+Chance used - to give us.""" - - def instance_type_to_query(self, instance_type): - """Return anything to prevent base-class from raising - exception.""" - return (self._full_name(), instance_type) - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts from ZoneManager list.""" - return [(host, services) - for host, services in zone_manager.service_states.iteritems()] - - -class FlavorQuery(Query): - """Query plug-in hard-coded to work with flavors.""" - - def instance_type_to_query(self, instance_type): - """Use instance_type to filter hosts.""" - return (self._full_name(), instance_type) - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can create instance_type.""" - instance_type = query - selected_hosts = [] - for host, services in zone_manager.service_states.iteritems(): - capabilities = services.get('compute', {}) - host_ram_mb = capabilities['host_memory']['free'] - disk_bytes = capabilities['disk']['available'] - if host_ram_mb >= instance_type['memory_mb'] and \ - disk_bytes >= instance_type['local_gb']: - selected_hosts.append((host, capabilities)) - return selected_hosts - -#host entries (currently) are like: -# {'host_name-description': 'Default install of XenServer', -# 'host_hostname': 'xs-mini', -# 'host_memory': {'total': 8244539392, -# 'overhead': 184225792, -# 'free': 3868327936, -# 'free-computed': 3840843776}, -# 'host_other-config': {}, -# 'host_ip_address': '192.168.1.109', -# 'host_cpu_info': {}, -# 'disk': {'available': 32954957824, -# 'total': 50394562560, -# 'used': 17439604736}, -# 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', -# 'host_name-label': 'xs-mini'} - -# instance_type table has: -#name = Column(String(255), unique=True) -#memory_mb = Column(Integer) -#vcpus = Column(Integer) -#local_gb = Column(Integer) -#flavorid = Column(Integer, unique=True) -#swap = Column(Integer, nullable=False, default=0) -#rxtx_quota = Column(Integer, nullable=False, default=0) -#rxtx_cap = Column(Integer, nullable=False, default=0) - - -class JsonQuery(Query): - """Query plug-in to allow simple JSON-based grammar for selecting hosts.""" - - def _equals(self, args): - """First term is == all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs != rhs: - return False - return True - - def _less_than(self, args): - """First term is < all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs >= rhs: - return False - return True - - def _greater_than(self, args): - """First term is > all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs <= rhs: - return False - return True - - def _in(self, args): - """First term is in set of remaining terms""" - if len(args) < 2: - return False - return args[0] in args[1:] - - def _less_than_equal(self, args): - """First term is <= all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs > rhs: - return False - return True - - def _greater_than_equal(self, args): - """First term is >= all the other terms.""" - if len(args) < 2: - return False - lhs = args[0] - for rhs in args[1:]: - if lhs < rhs: - return False - return True - - def _not(self, args): - """Flip each of the arguments.""" - if len(args) == 0: - return False - return [not arg for arg in args] - - def _or(self, args): - """True if any arg is True.""" - return True in args - - def _and(self, args): - """True if all args are True.""" - return False not in args - - commands = { - '=': _equals, - '<': _less_than, - '>': _greater_than, - 'in': _in, - '<=': _less_than_equal, - '>=': _greater_than_equal, - 'not': _not, - 'or': _or, - 'and': _and, - } - - def instance_type_to_query(self, instance_type): - """Convert instance_type into JSON query object.""" - required_ram = instance_type['memory_mb'] - required_disk = instance_type['local_gb'] - query = ['and', - ['>=', '$compute.host_memory.free', required_ram], - ['>=', '$compute.disk.available', required_disk] - ] - return (self._full_name(), json.dumps(query)) - - def _parse_string(self, string, host, services): - """Strings prefixed with $ are capability lookups in the - form '$service.capability[.subcap*]'""" - if not string: - return None - if string[0] != '$': - return string - - path = string[1:].split('.') - for item in path: - services = services.get(item, None) - if not services: - return None - return services - - def _process_query(self, zone_manager, query, host, services): - """Recursively parse the query structure.""" - if len(query) == 0: - return True - cmd = query[0] - method = self.commands[cmd] # Let exception fly. - cooked_args = [] - for arg in query[1:]: - if isinstance(arg, list): - arg = self._process_query(zone_manager, arg, host, services) - elif isinstance(arg, basestring): - arg = self._parse_string(arg, host, services) - if arg != None: - cooked_args.append(arg) - result = method(self, cooked_args) - return result - - def filter_hosts(self, zone_manager, query): - """Return a list of hosts that can fulfill query.""" - expanded = json.loads(query) - hosts = [] - for host, services in zone_manager.service_states.iteritems(): - r = self._process_query(zone_manager, expanded, host, services) - if isinstance(r, list): - r = True in r - if r: - hosts.append((host, services)) - return hosts - - -DRIVERS = [AllHostsQuery, FlavorQuery, JsonQuery] - - -def choose_driver(driver_name=None): - """Since the caller may specify which driver to use we need - to have an authoritative list of what is permissible. This - function checks the driver name against a predefined set - of acceptable drivers.""" - - if not driver_name: - driver_name = FLAGS.default_query_engine - for driver in DRIVERS: - if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: - return driver() - raise exception.SchedulerQueryDriverNotFound(driver_name=driver_name) diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py new file mode 100644 index 000000000..31e40ae1d --- /dev/null +++ b/nova/tests/test_host_filter.py @@ -0,0 +1,208 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler Host Filter Drivers. +""" + +import json + +from nova import exception +from nova import flags +from nova import test +from nova.scheduler import host_filter + +FLAGS = flags.FLAGS + + +class FakeZoneManager: + pass + + +class HostFilterTestCase(test.TestCase): + """Test case for host filter drivers.""" + + def _host_caps(self, multiplier): + # Returns host capabilities in the following way: + # host1 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory': {'total': 100, + 'overhead': 10, + 'free': 10 + multiplier * 10, + 'free-computed': 10 + multiplier * 10}, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk': {'available': 100 + multiplier * 100, + 'total': 1000, + 'used': 0}, + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} + + def setUp(self): + self.old_flag = FLAGS.default_host_filter_driver + FLAGS.default_host_filter_driver = \ + 'nova.scheduler.host_filter.AllHostsFilter' + self.instance_type = dict(name='tiny', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) + + self.zone_manager = FakeZoneManager() + states = {} + for x in xrange(10): + states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} + self.zone_manager.service_states = states + + def tearDown(self): + FLAGS.default_host_filter_driver = self.old_flag + + def test_choose_driver(self): + # Test default driver ... + driver = host_filter.choose_driver() + self.assertEquals(driver._full_name(), + 'nova.scheduler.host_filter.AllHostsFilter') + # Test valid driver ... + driver = host_filter.choose_driver( + 'nova.scheduler.host_filter.FlavorFilter') + self.assertEquals(driver._full_name(), + 'nova.scheduler.host_filter.FlavorFilter') + # Test invalid driver ... + try: + host_filter.choose_driver('does not exist') + self.fail("Should not find driver") + except exception.SchedulerHostFilterDriverNotFound: + pass + + def test_all_host_driver(self): + driver = host_filter.AllHostsFilter() + cooked = driver.instance_type_to_filter(self.instance_type) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(10, len(hosts)) + for host, capabilities in hosts: + self.assertTrue(host.startswith('host')) + + def test_flavor_driver(self): + driver = host_filter.FlavorFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.FlavorFilter', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) + + def test_json_driver(self): + driver = host_filter.JsonFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = driver.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) + hosts = driver.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) + + # Try some custom queries + + raw = ['or', + ['and', + ['<', '$compute.host_memory.free', 30], + ['<', '$compute.disk.available', 300] + ], + ['and', + ['>', '$compute.host_memory.free', 70], + ['>', '$compute.disk.available', 700] + ] + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['not', + ['=', '$compute.host_memory.free', 30], + ] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(9, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] + cooked = json.dumps(raw) + hosts = driver.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([2, 4, 6, 8, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + # Try some bogus input ... + raw = ['unknown command', ] + cooked = json.dumps(raw) + try: + driver.filter_hosts(self.zone_manager, cooked) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False] + ))) + + try: + driver.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False + )) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$foo', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', '$.....', 100] + ))) + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] + ))) + + self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( + ['=', {}, ['>', '$missing....foo']] + ))) diff --git a/nova/tests/test_query.py b/nova/tests/test_query.py deleted file mode 100644 index 9497a8c96..000000000 --- a/nova/tests/test_query.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Tests For Scheduler Query Drivers -""" - -import json - -from nova import exception -from nova import flags -from nova import test -from nova.scheduler import query - -FLAGS = flags.FLAGS - - -class FakeZoneManager: - pass - - -class QueryTestCase(test.TestCase): - """Test case for query drivers.""" - - def _host_caps(self, multiplier): - # Returns host capabilities in the following way: - # host1 = memory:free 10 (100max) - # disk:available 100 (1000max) - # hostN = memory:free 10 + 10N - # disk:available 100 + 100N - # in other words: hostN has more resources than host0 - # which means ... don't go above 10 hosts. - return {'host_name-description': 'XenServer %s' % multiplier, - 'host_hostname': 'xs-%s' % multiplier, - 'host_memory': {'total': 100, - 'overhead': 10, - 'free': 10 + multiplier * 10, - 'free-computed': 10 + multiplier * 10}, - 'host_other-config': {}, - 'host_ip_address': '192.168.1.%d' % (100 + multiplier), - 'host_cpu_info': {}, - 'disk': {'available': 100 + multiplier * 100, - 'total': 1000, - 'used': 0}, - 'host_uuid': 'xxx-%d' % multiplier, - 'host_name-label': 'xs-%s' % multiplier} - - def setUp(self): - self.old_flag = FLAGS.default_query_engine - FLAGS.default_query_engine = 'nova.scheduler.query.AllHostsQuery' - self.instance_type = dict(name='tiny', - memory_mb=50, - vcpus=10, - local_gb=500, - flavorid=1, - swap=500, - rxtx_quota=30000, - rxtx_cap=200) - - self.zone_manager = FakeZoneManager() - states = {} - for x in xrange(10): - states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} - self.zone_manager.service_states = states - - def tearDown(self): - FLAGS.default_query_engine = self.old_flag - - def test_choose_driver(self): - # Test default driver ... - driver = query.choose_driver() - self.assertEquals(driver._full_name(), - 'nova.scheduler.query.AllHostsQuery') - # Test valid driver ... - driver = query.choose_driver('nova.scheduler.query.FlavorQuery') - self.assertEquals(driver._full_name(), - 'nova.scheduler.query.FlavorQuery') - # Test invalid driver ... - try: - query.choose_driver('does not exist') - self.fail("Should not find driver") - except exception.SchedulerQueryDriverNotFound: - pass - - def test_all_host_driver(self): - driver = query.AllHostsQuery() - cooked = driver.instance_type_to_query(self.instance_type) - hosts = driver.filter_hosts(self.zone_manager, cooked) - self.assertEquals(10, len(hosts)) - for host, capabilities in hosts: - self.assertTrue(host.startswith('host')) - - def test_flavor_driver(self): - driver = query.FlavorQuery() - # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_query(self.instance_type) - self.assertEquals('nova.scheduler.query.FlavorQuery', name) - hosts = driver.filter_hosts(self.zone_manager, cooked) - self.assertEquals(6, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - self.assertEquals('host05', just_hosts[0]) - self.assertEquals('host10', just_hosts[5]) - - def test_json_driver(self): - driver = query.JsonQuery() - # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_query(self.instance_type) - self.assertEquals('nova.scheduler.query.JsonQuery', name) - hosts = driver.filter_hosts(self.zone_manager, cooked) - self.assertEquals(6, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - self.assertEquals('host05', just_hosts[0]) - self.assertEquals('host10', just_hosts[5]) - - # Try some custom queries - - raw = ['or', - ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] - ], - ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] - ] - ] - cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(5, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([1, 2, 8, 9, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - raw = ['not', - ['=', '$compute.host_memory.free', 30], - ] - cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(9, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] - cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) - - self.assertEquals(5, len(hosts)) - just_hosts = [host for host, caps in hosts] - just_hosts.sort() - for index, host in zip([2, 4, 6, 8, 10], just_hosts): - self.assertEquals('host%02d' % index, host) - - # Try some bogus input ... - raw = ['unknown command', ] - cooked = json.dumps(raw) - try: - driver.filter_hosts(self.zone_manager, cooked) - self.fail("Should give KeyError") - except KeyError, e: - pass - - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( - ['not', True, False, True, False] - ))) - - try: - driver.filter_hosts(self.zone_manager, json.dumps( - 'not', True, False, True, False - )) - self.fail("Should give KeyError") - except KeyError, e: - pass - - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', '$foo', 100] - ))) - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', '$.....', 100] - ))) - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] - ))) - - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', {}, ['>', '$missing....foo']] - ))) -- cgit From fa9eeb65533d897f6e81067986dc614582fb310a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 6 May 2011 07:19:57 -0700 Subject: grabbed from dist-sched branch --- nova/compute/manager.py | 17 ++++++++++++ nova/virt/hyperv.py | 9 +++++++ nova/virt/libvirt_conn.py | 8 ++++++ nova/virt/xenapi_conn.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 1ff78007b..473f93170 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -1082,6 +1082,13 @@ class ComputeManager(manager.SchedulerDependentManager): unicode(ex)) error_list.append(ex) + try: + self._report_driver_status() + except Exception as ex: + LOG.warning(_("Error during report_driver_status(): %s"), + unicode(ex)) + error_list.append(ex) + try: self._poll_instance_states(context) except Exception as ex: @@ -1091,6 +1098,16 @@ class ComputeManager(manager.SchedulerDependentManager): return error_list + def _report_driver_status(self): + curr_time = time.time() + if curr_time - self._last_host_check > FLAGS.host_state_interval: + self._last_host_check = curr_time + LOG.info(_("Updating host status")) + # This will grab info about the host and queue it + # to be sent to the Schedulers. + self.update_service_capabilities( + self.driver.get_host_stats(refresh=True)) + def _poll_instance_states(self, context): vm_instances = self.driver.list_instances_detail() vm_instances = dict((vm.name, vm) for vm in vm_instances) diff --git a/nova/virt/hyperv.py b/nova/virt/hyperv.py index 9026e737e..573e5130e 100644 --- a/nova/virt/hyperv.py +++ b/nova/virt/hyperv.py @@ -486,3 +486,12 @@ class HyperVConnection(driver.ComputeDriver): def update_available_resource(self, ctxt, host): """This method is supported only by libvirt.""" return + + def update_host_status(self): + """See xenapi_conn.py implementation.""" + pass + + def get_host_stats(self, refresh=False): + """See xenapi_conn.py implementation.""" + pass + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9780c69a6..555e44ce2 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1582,6 +1582,14 @@ class LibvirtConnection(driver.ComputeDriver): """See comments of same method in firewall_driver.""" self.firewall_driver.unfilter_instance(instance_ref) + def update_host_status(self): + """See xenapi_conn.py implementation.""" + pass + + def get_host_stats(self, refresh=False): + """See xenapi_conn.py implementation.""" + pass + class FirewallDriver(object): def prepare_instance_filter(self, instance, network_info=None): diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 0cabccf08..63a53af2e 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -168,6 +168,13 @@ class XenAPIConnection(driver.ComputeDriver): session = XenAPISession(url, user, pw) self._vmops = VMOps(session) self._volumeops = VolumeOps(session) + self._host_state = None + + @property + def HostState(self): + if not self._host_state: + self._host_state = HostState(self.session) + return self._host_state def init_host(self, host): #FIXME(armando): implement this @@ -315,6 +322,16 @@ class XenAPIConnection(driver.ComputeDriver): """This method is supported only by libvirt.""" raise NotImplementedError('This method is supported only by libvirt.') + def update_host_status(self): + """Update the status info of the host, and return those values + to the calling program.""" + return self.HostState.update_status() + + def get_host_stats(self, refresh=False): + """Return the current state of the host. If 'refresh' is + True, run the update first.""" + return self.HostState.get_host_stats(refresh=refresh) + class XenAPISession(object): """The session to invoke XenAPI SDK calls""" @@ -436,6 +453,58 @@ class XenAPISession(object): raise +class HostState(object): + """Manages information about the XenServer host this compute + node is running on. + """ + def __init__(self, session): + super(HostState, self).__init__() + self._session = session + self._stats = {} + self.update_status() + + def get_host_stats(self, refresh=False): + """Return the current state of the host. If 'refresh' is + True, run the update first. + """ + if refresh: + self.update_status() + return self._stats + + def update_status(self): + """Since under Xenserver, a compute node runs on a given host, + we can get host status information using xenapi. + """ + LOG.debug(_("Updating host stats")) + # Make it something unlikely to match any actual instance ID + task_id = random.randint(-80000, -70000) + task = self._session.async_call_plugin("xenhost", "host_data", {}) + task_result = self._session.wait_for_task(task, task_id) + if not task_result: + task_result = json.dumps("") + try: + data = json.loads(task_result) + except ValueError as e: + # Invalid JSON object + LOG.error(_("Unable to get updated status: %s") % e) + return + # Get the SR usage + try: + sr_ref = vm_utils.safe_find_sr(self._session) + except exception.NotFound as e: + # No SR configured + LOG.error(_("Unable to get SR for this host: %s") % e) + return + sr_rec = self._session.get_xenapi().SR.get_record(sr_ref) + total = int(sr_rec["virtual_allocation"]) + used = int(sr_rec["physical_utilisation"]) + data["disk"] = dd = {} + dd["total"] = total + dd["used"] = used + dd["available"] = total - used + self._stats = data + + def _parse_xmlrpc_value(val): """Parse the given value as if it were an XML-RPC value. This is sometimes used as the format for the task.result field.""" -- cgit From a845a769671066a684e38b82c34ded7ded55eed4 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 6 May 2011 09:06:46 -0700 Subject: revised file docs --- nova/scheduler/host_filter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index aa6101c93..ed8e65c77 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -29,9 +29,11 @@ Since we don't want to get into building full DSL this is a simple form as an example of how this could be done. In reality, most consumers will use the more rigid filters such as FlavorFilter. -Note: These are hard filters. All capabilities used must be present -or the host will be excluded. If you want soft filters use the weighting -mechanism which is intended for the more touchy-feely capabilities. +Note: These are "required" capability filters. These capabilities +used must be present or the host will be excluded. The hosts +returned are then weighed by the Weighted Scheduler. Weights +can take the more esoteric factors into consideration (such as +server affinity and customer separation). """ import json -- cgit From 595f742763336bb4edeb55f7556ce618bf85481e Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 6 May 2011 11:04:00 -0700 Subject: tests pass again --- nova/tests/test_compute.py | 9 +++++++++ nova/virt/hyperv.py | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 393110791..55e7ae0c4 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -21,6 +21,7 @@ Tests For Compute import datetime import mox +import stubout from nova import compute from nova import context @@ -52,6 +53,10 @@ class FakeTime(object): self.counter += t +def nop_report_driver_status(self): + pass + + class ComputeTestCase(test.TestCase): """Test case for compute""" def setUp(self): @@ -649,6 +654,10 @@ class ComputeTestCase(test.TestCase): def test_run_kill_vm(self): """Detect when a vm is terminated behind the scenes""" + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(compute_manager.ComputeManager, + '_report_driver_status', nop_report_driver_status) + instance_id = self._create_instance() self.compute.run_instance(self.context, instance_id) diff --git a/nova/virt/hyperv.py b/nova/virt/hyperv.py index 573e5130e..1142e97a4 100644 --- a/nova/virt/hyperv.py +++ b/nova/virt/hyperv.py @@ -494,4 +494,3 @@ class HyperVConnection(driver.ComputeDriver): def get_host_stats(self, refresh=False): """See xenapi_conn.py implementation.""" pass - -- cgit From 417de9b7a695b088c2525470f971455b727c8c38 Mon Sep 17 00:00:00 2001 From: Lorin Hochstein Date: Fri, 6 May 2011 14:23:51 -0400 Subject: Added Python packages needed for coverage reports to virtualenv packages --- tools/pip-requires | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/pip-requires b/tools/pip-requires index 2f4136732..e438c2a41 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -31,3 +31,5 @@ sphinx glance nova-adminclient suds==0.4 +coverage +nosexcover -- cgit From 791f8dc895b97caa9395a52113823bde37ae6cfa Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 6 May 2011 13:47:47 -0500 Subject: Set root password upon XenServer instance creation. --- nova/compute/api.py | 22 ++++++++++++-- nova/compute/manager.py | 34 ++++++++++++++-------- nova/tests/api/openstack/test_servers.py | 6 ++++ nova/virt/xenapi/vmops.py | 16 +++++----- plugins/xenserver/xenapi/etc/xapi.d/plugins/agent | 1 - .../xenapi/etc/xapi.d/plugins/xenstore.py | 6 ++-- 6 files changed, 60 insertions(+), 25 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index be26d8ca3..e8adf6ba5 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -482,6 +482,20 @@ class API(base.Base): """Generic handler for RPC calls to the scheduler.""" rpc.cast(context, FLAGS.scheduler_topic, args) + def _find_host(self, context, instance_id): + """Find the host associated with an instance.""" + host = None + for count in xrange(10): + instance = self.get(context, instance_id) + host = instance["host"] + if host: + return host + elif count >= 10: + raise exception.Error(_("Unable to find host for Instance %s") + % instance_id) + else: + time.sleep(1) + def snapshot(self, context, instance_id, name): """Snapshot the given instance. @@ -635,8 +649,12 @@ class API(base.Base): def set_admin_password(self, context, instance_id, password=None): """Set the root/admin password for the given instance.""" - self._cast_compute_message( - 'set_admin_password', context, instance_id, password) + host = self._find_host(context, instance_id) + + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "set_admin_password", + "args": {"instance_id": instance_id, "new_pass": password}}) def inject_file(self, context, instance_id): """Write a file to the given instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 1ff78007b..d92f7719a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -40,6 +40,7 @@ import os import socket import sys import tempfile +import time import functools from eventlet import greenthread @@ -404,21 +405,30 @@ class ComputeManager(manager.SchedulerDependentManager): def set_admin_password(self, context, instance_id, new_pass=None): """Set the root/admin password for an instance on this host.""" context = context.elevated() - instance_ref = self.db.instance_get(context, instance_id) - instance_id = instance_ref['id'] - instance_state = instance_ref['state'] - expected_state = power_state.RUNNING - if instance_state != expected_state: - LOG.warn(_('trying to reset the password on a non-running ' - 'instance: %(instance_id)s (state: %(instance_state)s ' - 'expected: %(expected_state)s)') % locals()) - LOG.audit(_('instance %s: setting admin password'), - instance_ref['name']) + if new_pass is None: # Generate a random password new_pass = utils.generate_password(FLAGS.password_length) - self.driver.set_admin_password(instance_ref, new_pass) - self._update_state(context, instance_id) + + while True: + instance_ref = self.db.instance_get(context, instance_id) + instance_id = instance_ref["id"] + instance_state = instance_ref["state"] + expected_state = power_state.RUNNING + + if instance_state != expected_state: + time.sleep(5) + continue + else: + try: + LOG.audit(_("Instance %s: Setting root password"), + instance_ref["name"]) + self.driver.set_admin_password(instance_ref, new_pass) + break + except Exception, e: + # Catch all here because this could be anything. + LOG.exception(e) + continue @exception.wrap_exception @checks_instance_lock diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 5c643fcef..89edece42 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -134,6 +134,10 @@ def fake_compute_api(cls, req, id): return True +def find_host(self, context, instance_id): + return "nova" + + class ServersTest(test.TestCase): def setUp(self): @@ -473,6 +477,7 @@ class ServersTest(test.TestCase): "_get_kernel_ramdisk_from_image", kernel_ramdisk_mapping) self.stubs.Set(nova.api.openstack.common, "get_image_id_from_image_hash", image_id_from_hash) + self.stubs.Set(nova.compute.api.API, "_find_host", find_host) def _test_create_instance_helper(self): self._setup_for_create_instance() @@ -767,6 +772,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) + self.stubs.Set(nova.compute.api.API, "_find_host", find_host) req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 30f31517d..a13febdab 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -428,15 +428,16 @@ class VMOps(object): """ # Need to uniquely identify this request. - transaction_id = str(uuid.uuid4()) + key_init_transaction_id = str(uuid.uuid4()) # The simple Diffie-Hellman class is used to manage key exchange. dh = SimpleDH() - args = {'id': transaction_id, 'pub': str(dh.get_public())} - resp = self._make_agent_call('key_init', instance, '', args) + key_init_args = {'id': key_init_transaction_id, + 'pub': str(dh.get_public())} + resp = self._make_agent_call('key_init', instance, '', key_init_args) if resp is None: # No response from the agent return - resp_dict = json.loads(resp) + resp_dict = json.loads(json.loads(resp)) # Successful return code from key_init is 'D0' if resp_dict['returncode'] != 'D0': # There was some sort of error; the message will contain @@ -446,12 +447,13 @@ class VMOps(object): dh.compute_shared(agent_pub) enc_pass = dh.encrypt(new_pass) # Send the encrypted password - args['enc_pass'] = enc_pass - resp = self._make_agent_call('password', instance, '', args) + password_transaction_id = str(uuid.uuid4()) + password_args = {'id': password_transaction_id, 'enc_pass': enc_pass} + resp = self._make_agent_call('password', instance, '', password_args) if resp is None: # No response from the agent return - resp_dict = json.loads(resp) + resp_dict = json.loads(json.loads(resp)) # Successful return code from password is '0' if resp_dict['returncode'] != '0': raise RuntimeError(resp_dict['message']) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 5496a6bd5..0bb377dd3 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -80,7 +80,6 @@ def password(self, arg_dict): previous call to key_init. The encrypted password value should be passed as the value for the 'enc_pass' key in arg_dict. """ - pub = int(arg_dict["pub"]) enc_pass = arg_dict["enc_pass"] arg_dict["value"] = json.dumps({"name": "password", "value": enc_pass}) request_id = arg_dict["id"] diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py index d33c7346b..5b45a9845 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py @@ -59,12 +59,12 @@ def read_record(self, arg_dict): cmd = ["xenstore-read", "/local/domain/%(dom_id)s/%(path)s" % arg_dict] try: ret, result = _run_command(cmd) - return result.rstrip("\n") + return result.strip() except pluginlib.PluginError, e: if arg_dict.get("ignore_missing_path", False): cmd = ["xenstore-exists", "/local/domain/%(dom_id)s/%(path)s" % arg_dict] - ret, result = _run_command(cmd).strip() + ret, result = _run_command(cmd) # If the path exists, the cmd should return "0" if ret != 0: # No such path, so ignore the error and return the @@ -180,7 +180,7 @@ def _run_command(cmd): err = proc.stderr.read() if err: raise pluginlib.PluginError(err) - return proc.stdout.read() + return (ret, proc.stdout.read()) if __name__ == "__main__": -- cgit From 311c774e1109d6ce0449f0d06346078020ffa4e0 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 6 May 2011 14:48:54 -0500 Subject: Review feedback --- nova/compute/api.py | 12 ++++++------ nova/compute/manager.py | 4 ++-- nova/virt/xenapi/vmops.py | 4 ++-- plugins/xenserver/xenapi/etc/xapi.d/plugins/agent | 2 -- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index e8adf6ba5..a7276ad2d 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -485,16 +485,16 @@ class API(base.Base): def _find_host(self, context, instance_id): """Find the host associated with an instance.""" host = None - for count in xrange(10): + attempts = 10 + while attempts: instance = self.get(context, instance_id) host = instance["host"] if host: return host - elif count >= 10: - raise exception.Error(_("Unable to find host for Instance %s") - % instance_id) - else: - time.sleep(1) + attempts -= 1 + time.sleep(1) + raise exception.Error(_("Unable to find host for Instance %s") + % instance_id) def snapshot(self, context, instance_id, name): """Snapshot the given instance. diff --git a/nova/compute/manager.py b/nova/compute/manager.py index e75371741..ae5b50ef3 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -421,9 +421,9 @@ class ComputeManager(manager.SchedulerDependentManager): continue else: try: - LOG.audit(_("Instance %s: Setting root password"), - instance_ref["name"]) self.driver.set_admin_password(instance_ref, new_pass) + LOG.audit(_("Instance %s: Root password set"), + instance_ref["name"]) break except Exception, e: # Catch all here because this could be anything. diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index a13febdab..fe9a74dd6 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -437,7 +437,7 @@ class VMOps(object): if resp is None: # No response from the agent return - resp_dict = json.loads(json.loads(resp)) + resp_dict = json.loads(resp) # Successful return code from key_init is 'D0' if resp_dict['returncode'] != 'D0': # There was some sort of error; the message will contain @@ -453,7 +453,7 @@ class VMOps(object): if resp is None: # No response from the agent return - resp_dict = json.loads(json.loads(resp)) + resp_dict = json.loads(resp) # Successful return code from password is '0' if resp_dict['returncode'] != '0': raise RuntimeError(resp_dict['message']) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 0bb377dd3..9e761f264 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -53,7 +53,6 @@ class TimeoutError(StandardError): pass -@jsonify def key_init(self, arg_dict): """Handles the Diffie-Hellman key exchange with the agent to establish the shared secret key used to encrypt/decrypt sensitive @@ -72,7 +71,6 @@ def key_init(self, arg_dict): return resp -@jsonify def password(self, arg_dict): """Writes a request to xenstore that tells the agent to set the root password for the given VM. The password should be -- cgit From 8c336fa339d9038f5430f7ffd82df3a54e67196f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 6 May 2011 15:06:21 -0500 Subject: Review feedback --- nova/compute/api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index a7276ad2d..63884be97 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -484,14 +484,11 @@ class API(base.Base): def _find_host(self, context, instance_id): """Find the host associated with an instance.""" - host = None - attempts = 10 - while attempts: + for attempts in xrange(10): instance = self.get(context, instance_id) host = instance["host"] if host: return host - attempts -= 1 time.sleep(1) raise exception.Error(_("Unable to find host for Instance %s") % instance_id) -- cgit From 7860f72c911dd91b69082cb5cdb2e625710526c1 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 6 May 2011 15:19:55 -0500 Subject: Review feedback --- plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py index 5b45a9845..55f3911e7 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py @@ -171,7 +171,7 @@ def _paths_from_ls(recs): def _run_command(cmd): """Abstracts out the basics of issuing system commands. If the command returns anything in stderr, a PluginError is raised with that information. - Otherwise, the output from stdout is returned. + Otherwise, a tuple of (return code, stdout) is returned. """ pipe = subprocess.PIPE proc = subprocess.Popen(cmd, stdin=pipe, stdout=pipe, stderr=pipe, -- cgit From 6425095be3bd89c48b73b5305afeb2d5d45e434b Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 6 May 2011 15:41:36 -0500 Subject: Review feedback --- plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py index 55f3911e7..6c589ed29 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/xenstore.py @@ -171,7 +171,7 @@ def _paths_from_ls(recs): def _run_command(cmd): """Abstracts out the basics of issuing system commands. If the command returns anything in stderr, a PluginError is raised with that information. - Otherwise, a tuple of (return code, stdout) is returned. + Otherwise, a tuple of (return code, stdout data) is returned. """ pipe = subprocess.PIPE proc = subprocess.Popen(cmd, stdin=pipe, stdout=pipe, stderr=pipe, -- cgit From 3c0d31a1ae91e30e06f1b33d35915037472b3691 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 08:23:25 -0700 Subject: basic test working --- nova/tests/test_xenapi.py | 40 ++++++++++++++++++++++++++++++++++++++++ nova/virt/xenapi_conn.py | 4 ++++ 2 files changed, 44 insertions(+) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 375480a2e..756a289bd 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -17,6 +17,7 @@ """Test suite for XenAPI.""" import functools +import json import os import re import stubout @@ -665,3 +666,42 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_VHD self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_VHD) + + +class FakeXenApi(object): + """Fake XenApi for testing HostState.""" + + class FakeSR(object): + def get_record(self, ref): + return {'virtual_allocation':10000, + 'physical_utilisation':20000} + + SR = FakeSR() + + +class FakeSession(object): + """Fake Session class for HostState testing.""" + + def async_call_plugin(self, *args): + return None + + def wait_for_task(self, *args): + return json.dumps({}) + + def get_xenapi(self): + return FakeXenApi() + + +class HostStateTestCase(test.TestCase): + """Tests HostState, which holds metrics from XenServer that get + reported back to the Schedulers.""" + + def _fake_safe_find_sr(self, session): + """None SR ref since we're ignoring it in FakeSR.""" + return None + + def test_host_state(self): + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(vm_utils, 'safe_find_sr', self._fake_safe_find_sr) + host_state = xenapi_conn.HostState(FakeSession()) + diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 63a53af2e..0e545150f 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -57,6 +57,8 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. - suffix "_rec" for record objects """ +import json +import random import sys import urlparse import xmlrpclib @@ -67,10 +69,12 @@ from eventlet import timeout from nova import context from nova import db +from nova import exception from nova import utils from nova import flags from nova import log as logging from nova.virt import driver +from nova.virt.xenapi import vm_utils from nova.virt.xenapi.vmops import VMOps from nova.virt.xenapi.volumeops import VolumeOps -- cgit From d087e1d0f0e235de01a8f140815fbe905008cb36 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 09:08:56 -0700 Subject: capabilities flattened and tests fixed --- nova/compute/manager.py | 1 + nova/scheduler/host_filter.py | 16 ++++++++-------- nova/tests/test_host_filter.py | 26 +++++++++++++------------- nova/tests/test_xenapi.py | 12 ++++++------ nova/virt/xenapi_conn.py | 4 ++-- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 97c1bbded..abf1a478b 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -130,6 +130,7 @@ class ComputeManager(manager.SchedulerDependentManager): self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) self.network_api = network.API() + self._last_host_check = 0 super(ComputeManager, self).__init__(service_name="compute", *args, **kwargs) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 3e831b76f..885878e14 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -96,8 +96,8 @@ class FlavorFilter(HostFilter): selected_hosts = [] for host, services in zone_manager.service_states.iteritems(): capabilities = services.get('compute', {}) - host_ram_mb = capabilities['host_memory']['free'] - disk_bytes = capabilities['disk']['available'] + host_ram_mb = capabilities['host_memory_free'] + disk_bytes = capabilities['disk_available'] if host_ram_mb >= instance_type['memory_mb'] and \ disk_bytes >= instance_type['local_gb']: selected_hosts.append((host, capabilities)) @@ -106,10 +106,10 @@ class FlavorFilter(HostFilter): #host entries (currently) are like: # {'host_name-description': 'Default install of XenServer', # 'host_hostname': 'xs-mini', -# 'host_memory': {'total': 8244539392, -# 'overhead': 184225792, -# 'free': 3868327936, -# 'free-computed': 3840843776}, +# 'host_memory_total': 8244539392, +# 'host_memory_overhead': 184225792, +# 'host_memory_free': 3868327936, +# 'host_memory_free-computed': 3840843776}, # 'host_other-config': {}, # 'host_ip_address': '192.168.1.109', # 'host_cpu_info': {}, @@ -221,8 +221,8 @@ class JsonFilter(HostFilter): required_ram = instance_type['memory_mb'] required_disk = instance_type['local_gb'] query = ['and', - ['>=', '$compute.host_memory.free', required_ram], - ['>=', '$compute.disk.available', required_disk] + ['>=', '$compute.host_memory_free', required_ram], + ['>=', '$compute.disk_available', required_disk] ] return (self._full_name(), json.dumps(query)) diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py index 31e40ae1d..c029d41e6 100644 --- a/nova/tests/test_host_filter.py +++ b/nova/tests/test_host_filter.py @@ -43,16 +43,16 @@ class HostFilterTestCase(test.TestCase): # which means ... don't go above 10 hosts. return {'host_name-description': 'XenServer %s' % multiplier, 'host_hostname': 'xs-%s' % multiplier, - 'host_memory': {'total': 100, - 'overhead': 10, - 'free': 10 + multiplier * 10, - 'free-computed': 10 + multiplier * 10}, + 'host_memory_total': 100, + 'host_memory_overhead': 10, + 'host_memory_free': 10 + multiplier * 10, + 'host_memory_free-computed': 10 + multiplier * 10, 'host_other-config': {}, 'host_ip_address': '192.168.1.%d' % (100 + multiplier), 'host_cpu_info': {}, - 'disk': {'available': 100 + multiplier * 100, - 'total': 1000, - 'used': 0}, + 'disk_available': 100 + multiplier * 100, + 'disk_total': 1000, + 'disk_used': 0, 'host_uuid': 'xxx-%d' % multiplier, 'host_name-label': 'xs-%s' % multiplier} @@ -131,12 +131,12 @@ class HostFilterTestCase(test.TestCase): raw = ['or', ['and', - ['<', '$compute.host_memory.free', 30], - ['<', '$compute.disk.available', 300] + ['<', '$compute.host_memory_free', 30], + ['<', '$compute.disk_available', 300] ], ['and', - ['>', '$compute.host_memory.free', 70], - ['>', '$compute.disk.available', 700] + ['>', '$compute.host_memory_free', 70], + ['>', '$compute.disk_available', 700] ] ] cooked = json.dumps(raw) @@ -149,7 +149,7 @@ class HostFilterTestCase(test.TestCase): self.assertEquals('host%02d' % index, host) raw = ['not', - ['=', '$compute.host_memory.free', 30], + ['=', '$compute.host_memory_free', 30], ] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) @@ -160,7 +160,7 @@ class HostFilterTestCase(test.TestCase): for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): self.assertEquals('host%02d' % index, host) - raw = ['in', '$compute.host_memory.free', 20, 40, 60, 80, 100] + raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) hosts = driver.filter_hosts(self.zone_manager, cooked) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 0f1b2aa48..678291579 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -709,9 +709,9 @@ class HostStateTestCase(test.TestCase): self.stubs.Set(vm_utils, 'safe_find_sr', self._fake_safe_find_sr) host_state = xenapi_conn.HostState(FakeSession()) stats = host_state._stats - self.assertEquals('disk_total', 10000) - self.assertEquals('disk_used', 20000) - self.assertEquals('host_memory_total', 10) - self.assertEquals('host_memory_overhead', 20) - self.assertEquals('host_memory_free', 30) - self.assertEquals('host_memory_free-computed', 40) + self.assertEquals(stats['disk_total'], 10000) + self.assertEquals(stats['disk_used'], 20000) + self.assertEquals(stats['host_memory_total'], 10) + self.assertEquals(stats['host_memory_overhead'], 20) + self.assertEquals(stats['host_memory_free'], 30) + self.assertEquals(stats['host_memory_free-computed'], 40) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 563a1da77..671a340c7 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -505,8 +505,8 @@ class HostState(object): data["disk_total"] = total data["disk_used"] = used data["disk_available"] = total - used - memory = data.get('host_memory', None) - if memory: + host_memory = data.get('host_memory', None) + if host_memory: data["host_memory_total"] = host_memory.get('total', 0) data["host_memory_overhead"] = host_memory.get('overhead', 0) data["host_memory_free"] = host_memory.get('free', 0) -- cgit From a3f8d3c8ee77cd7cf764aec19033ab0c71703515 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 09:10:22 -0700 Subject: pep8 --- nova/tests/test_xenapi.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 678291579..6dbd1aee5 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -673,24 +673,24 @@ class FakeXenApi(object): class FakeSR(object): def get_record(self, ref): - return {'virtual_allocation':10000, - 'physical_utilisation':20000} + return {'virtual_allocation': 10000, + 'physical_utilisation': 20000} SR = FakeSR() class FakeSession(object): """Fake Session class for HostState testing.""" - + def async_call_plugin(self, *args): return None def wait_for_task(self, *args): - vm = {'total':10, - 'overhead':20, - 'free':30, - 'free-computed':40} - return json.dumps({'host_memory':vm}) + vm = {'total': 10, + 'overhead': 20, + 'free': 30, + 'free-computed': 40} + return json.dumps({'host_memory': vm}) def get_xenapi(self): return FakeXenApi() -- cgit From 559bba1270378a430cc85abec144c0c574e65294 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 9 May 2011 12:57:56 -0700 Subject: unified underscore/dash issue --- nova/scheduler/host_filter.py | 2 +- nova/tests/test_xenapi.py | 2 +- nova/virt/xenapi_conn.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 885878e14..483f3225c 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -109,7 +109,7 @@ class FlavorFilter(HostFilter): # 'host_memory_total': 8244539392, # 'host_memory_overhead': 184225792, # 'host_memory_free': 3868327936, -# 'host_memory_free-computed': 3840843776}, +# 'host_memory_free_computed': 3840843776}, # 'host_other-config': {}, # 'host_ip_address': '192.168.1.109', # 'host_cpu_info': {}, diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 6dbd1aee5..6072f5455 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -714,4 +714,4 @@ class HostStateTestCase(test.TestCase): self.assertEquals(stats['host_memory_total'], 10) self.assertEquals(stats['host_memory_overhead'], 20) self.assertEquals(stats['host_memory_free'], 30) - self.assertEquals(stats['host_memory_free-computed'], 40) + self.assertEquals(stats['host_memory_free_computed'], 40) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 671a340c7..8e9085277 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -510,7 +510,7 @@ class HostState(object): data["host_memory_total"] = host_memory.get('total', 0) data["host_memory_overhead"] = host_memory.get('overhead', 0) data["host_memory_free"] = host_memory.get('free', 0) - data["host_memory_free-computed"] = \ + data["host_memory_free_computed"] = \ host_memory.get('free-computed', 0) del data['host_memory'] self._stats = data -- cgit From 6991faaac1eda14bf6162d1a2383e7f9ad6bdeae Mon Sep 17 00:00:00 2001 From: Justin Shepherd Date: Mon, 9 May 2011 22:36:01 -0500 Subject: Added GitPython to [install_dir]/tools/pip-requires. --- Authors | 1 + tools/pip-requires | 1 + 2 files changed, 2 insertions(+) diff --git a/Authors b/Authors index 60e1d2dad..d7f70f417 100644 --- a/Authors +++ b/Authors @@ -44,6 +44,7 @@ Josh Kearney Josh Kleinpeter Joshua McKenty Justin Santa Barbara +Justin Shepherd Kei Masumoto Ken Pepple Kevin Bringard diff --git a/tools/pip-requires b/tools/pip-requires index e438c2a41..f7eb1703e 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -33,3 +33,4 @@ nova-adminclient suds==0.4 coverage nosexcover +GitPython -- cgit