From 2a6ce075e19af5700960e3fb22c213e43a2e24b4 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 3 Mar 2011 16:28:04 -0400 Subject: start of fanout --- nova/rpc.py | 20 ++++++++++++++++++++ nova/scheduler/manager.py | 4 ++++ nova/scheduler/zone_manager.py | 4 ++++ nova/service.py | 6 ++++++ 4 files changed, 34 insertions(+) diff --git a/nova/rpc.py b/nova/rpc.py index 8fe4565dd..a02cdc90c 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -218,6 +218,16 @@ class TopicPublisher(Publisher): super(TopicPublisher, self).__init__(connection=connection) +class FanoutPublisher(Publisher): + """Publishes messages to a fanout exchange.""" + exchange_type = "fanout" + + def __init__(self, topic, connection=None): + self.exchange = "%s_fanout" % topic + self.durable = False + super(FanoutPublisher, self).__init__(connection=connection) + + class DirectConsumer(Consumer): """Consumes messages directly on a channel specified by msg_id""" exchange_type = "direct" @@ -360,6 +370,16 @@ def cast(context, topic, msg): publisher.close() +def fanout_cast(context, topic, msg): + """Sends a message on a fanout exchange without waiting for a response""" + LOG.debug(_("Making asynchronous fanout cast...")) + _pack_context(msg, context) + conn = Connection.instance() + publisher = FanoutPublisher(topic, connection=conn) + publisher.send(msg) + publisher.close() + + def generic_response(message_data, message): """Logs a result and exits""" LOG.debug(_('response %s'), message_data) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index c94397210..7541523b0 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,6 +59,10 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() + def update_compute_capabilities(self, context=None): + """Process a compute node update.""" + return self.zone_manager.update_compute_capabilities() + def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index edf9000cc..eedc5c235 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -105,6 +105,7 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} + self.compute_states = {} self.green_pool = greenpool.GreenPool() def get_zone_list(self): @@ -141,3 +142,6 @@ class ZoneManager(object): self.last_zone_db_check = datetime.now() self._refresh_from_db(context) self._poll_zones(context) + + def update_compute_capabilities(self): + logging.debug(_("****** UPDATE COMPUTE CAPABILITIES *******")) diff --git a/nova/service.py b/nova/service.py index f47358089..3ecf46525 100644 --- a/nova/service.py +++ b/nova/service.py @@ -84,6 +84,7 @@ class Service(object): conn1 = rpc.Connection.instance(new=True) conn2 = rpc.Connection.instance(new=True) + conn3 = rpc.Connection.instance(new=True) if self.report_interval: consumer_all = rpc.AdapterConsumer( connection=conn1, @@ -93,9 +94,14 @@ class Service(object): connection=conn2, topic='%s.%s' % (self.topic, self.host), proxy=self) + fanout = rpc.AdapterConsumer( + connection=conn2, + topic='%s_fanout' % self.topic, + proxy=self) self.timers.append(consumer_all.attach_to_eventlet()) self.timers.append(consumer_node.attach_to_eventlet()) + self.timers.append(fanout.attach_to_eventlet()) pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) -- cgit From 3d9d99a53d372abf9f69f1d6e66fa6c6491ec752 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 3 Mar 2011 19:27:15 -0400 Subject: tests passing --- nova/tests/test_service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 45d9afa6c..cb31a3c43 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -120,6 +120,12 @@ class ServiceTestCase(test.TestCase): proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) + rpc.AdapterConsumer(connection=mox.IgnoreArg(), + topic='%s_fanout' % topic, + proxy=mox.IsA(service.Service)).AndReturn( + rpc.AdapterConsumer) + + rpc.AdapterConsumer.attach_to_eventlet() rpc.AdapterConsumer.attach_to_eventlet() rpc.AdapterConsumer.attach_to_eventlet() -- cgit From a10d863e5e6127b8e987719ddfb60142b9f08db9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 13:36:45 -0800 Subject: scheduler manager --- nova/scheduler_manager.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 nova/scheduler_manager.py diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py new file mode 100644 index 000000000..c78b6fea5 --- /dev/null +++ b/nova/scheduler_manager.py @@ -0,0 +1,39 @@ +# Copyright 2011 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This module provides SchedulerDependentManager, a base class for +any Manager that has Capabilities that should be related to the +Scheduler. + +These Capabilities are hints that can help the scheduler route +requests to the appropriate service instance. +""" + +from nova import manager +from nova.scheduler import api + + +FLAGS = flags.FLAGS + + +def SchedulerDependentManage(manager.Manager): + def __init__(self, host=None, db_driver=None): + self.last_capabilities = {} + super(SchedulerDependentManager, self).__init__(host, db_driver) + + def periodic_tasks(self, context=None): + """Pass data back to the scheduler at a periodic interval""" + logging.debug(_("Notifying Schedulers of capabilities ...")) -- cgit From 5d821114fc20c88e36f079089cfe655d8188914a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 16:40:08 -0800 Subject: service ping working --- nova/compute/manager.py | 4 ++-- nova/scheduler_manager.py | 9 +++++++-- nova/service.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3af97683f..fd88158f7 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -44,7 +44,7 @@ import functools from nova import exception from nova import flags from nova import log as logging -from nova import manager +from nova import scheduler_manager from nova import rpc from nova import utils from nova.compute import power_state @@ -99,7 +99,7 @@ def checks_instance_lock(function): return decorated_function -class ComputeManager(manager.Manager): +class ComputeManager(scheduler_manager.SchedulerDependentManager): """Manages the running instances from creation to destruction.""" diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index c78b6fea5..987ca8e90 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -22,18 +22,23 @@ These Capabilities are hints that can help the scheduler route requests to the appropriate service instance. """ +import sys + +from nova import flags from nova import manager from nova.scheduler import api +from nova import log as logging FLAGS = flags.FLAGS -def SchedulerDependentManage(manager.Manager): +class SchedulerDependentManager(manager.Manager): def __init__(self, host=None, db_driver=None): self.last_capabilities = {} super(SchedulerDependentManager, self).__init__(host, db_driver) def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" - logging.debug(_("Notifying Schedulers of capabilities ...")) + logging.debug(_("*** Notifying Schedulers of capabilities ...")) + super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/service.py b/nova/service.py index 3ecf46525..66287b505 100644 --- a/nova/service.py +++ b/nova/service.py @@ -95,7 +95,7 @@ class Service(object): topic='%s.%s' % (self.topic, self.host), proxy=self) fanout = rpc.AdapterConsumer( - connection=conn2, + connection=conn3, topic='%s_fanout' % self.topic, proxy=self) -- cgit From 7297e6bf1536f20540200f28154c15d63372d943 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 20:58:59 -0800 Subject: fanout kinda working --- nova/rpc.py | 51 ++++++++++++++++++++++++++++++----------------- nova/scheduler/api.py | 7 +++++++ nova/scheduler/manager.py | 2 +- nova/scheduler_manager.py | 2 ++ nova/service.py | 8 ++++---- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index a02cdc90c..e0cf6d301 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -137,24 +137,7 @@ class Consumer(messaging.Consumer): return timer -class Publisher(messaging.Publisher): - """Publisher base class""" - pass - - -class TopicConsumer(Consumer): - """Consumes messages on a specific topic""" - exchange_type = "topic" - - def __init__(self, connection=None, topic="broadcast"): - self.queue = topic - self.routing_key = topic - self.exchange = FLAGS.control_exchange - self.durable = False - super(TopicConsumer, self).__init__(connection=connection) - - -class AdapterConsumer(TopicConsumer): +class AdapterConsumer(Consumer): """Calls methods on a proxy object based on method and args""" def __init__(self, connection=None, topic="broadcast", proxy=None): LOG.debug(_('Initing the Adapter Consumer for %s') % topic) @@ -207,6 +190,37 @@ class AdapterConsumer(TopicConsumer): return +class Publisher(messaging.Publisher): + """Publisher base class""" + pass + + +class TopicAdapterConsumer(AdapterConsumer): + """Consumes messages on a specific topic""" + exchange_type = "topic" + + def __init__(self, connection=None, topic="broadcast", proxy=None): + self.queue = topic + self.routing_key = topic + self.exchange = FLAGS.control_exchange + self.durable = False + super(TopicAdapterConsumer, self).__init__(connection=connection, + topic=topic, proxy=proxy) + + +class FanoutAdapterConsumer(AdapterConsumer): + """Consumes messages from a fanout exchange""" + exchange_type = "fanout" + + def __init__(self, connection=None, topic="broadcast", proxy=None): + self.exchange = "%s_fanout" % topic + self.routing_key = topic + self.queue = "ignored" + self.durable = False + super(FanoutAdapterConsumer, self).__init__(connection=connection, + topic=topic, proxy=proxy) + + class TopicPublisher(Publisher): """Publishes messages on a specific topic""" exchange_type = "topic" @@ -214,6 +228,7 @@ class TopicPublisher(Publisher): def __init__(self, connection=None, topic="broadcast"): self.routing_key = topic self.exchange = FLAGS.control_exchange + self.queue = "ignored" self.durable = False super(TopicPublisher, self).__init__(connection=connection) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 8491bf3a9..53d72507f 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -47,3 +47,10 @@ class API: for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') return items + + @classmethod + def update_service_capabilities(cls, context, service_name, capabilities): + kwargs = dict(method='update_service_capabilities', + service_name=service_name, capabilities=capabilities) + return rpc.fanout_cast(context, 'scheduler', kwargs) + diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 7541523b0..3ff43a9d9 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,7 +59,7 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() - def update_compute_capabilities(self, context=None): + def update_service_capabilities(self, context=None, service_name=None, capabilities={}): """Process a compute node update.""" return self.zone_manager.update_compute_capabilities() diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index 987ca8e90..a45301617 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -41,4 +41,6 @@ class SchedulerDependentManager(manager.Manager): def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" logging.debug(_("*** Notifying Schedulers of capabilities ...")) + api.API.update_service_capabilities(context, 'compute', self.last_capabilities) + super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/service.py b/nova/service.py index 66287b505..a98d6aac1 100644 --- a/nova/service.py +++ b/nova/service.py @@ -86,17 +86,17 @@ class Service(object): conn2 = rpc.Connection.instance(new=True) conn3 = rpc.Connection.instance(new=True) if self.report_interval: - consumer_all = rpc.AdapterConsumer( + consumer_all = rpc.TopicAdapterConsumer( connection=conn1, topic=self.topic, proxy=self) - consumer_node = rpc.AdapterConsumer( + consumer_node = rpc.TopicAdapterConsumer( connection=conn2, topic='%s.%s' % (self.topic, self.host), proxy=self) - fanout = rpc.AdapterConsumer( + fanout = rpc.FanoutAdapterConsumer( connection=conn3, - topic='%s_fanout' % self.topic, + topic=self.topic, proxy=self) self.timers.append(consumer_all.attach_to_eventlet()) -- cgit From 32f062c389a530b6af3f864eb4030a68d0a26eb1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 22:33:39 -0800 Subject: fanout works --- nova/rpc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index e0cf6d301..601e45b47 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -215,8 +215,12 @@ class FanoutAdapterConsumer(AdapterConsumer): def __init__(self, connection=None, topic="broadcast", proxy=None): self.exchange = "%s_fanout" % topic self.routing_key = topic - self.queue = "ignored" + unique = uuid.uuid4().hex + self.queue = "%s_fanout_%s" % (topic, unique) self.durable = False + LOG.info(_("Created '%(exchange)s' fanout exchange " + "with '%(key)s' routing key"), + dict(exchange=self.exchange, key=self.routing_key)) super(FanoutAdapterConsumer, self).__init__(connection=connection, topic=topic, proxy=proxy) @@ -228,7 +232,6 @@ class TopicPublisher(Publisher): def __init__(self, connection=None, topic="broadcast"): self.routing_key = topic self.exchange = FLAGS.control_exchange - self.queue = "ignored" self.durable = False super(TopicPublisher, self).__init__(connection=connection) @@ -239,7 +242,10 @@ class FanoutPublisher(Publisher): def __init__(self, topic, connection=None): self.exchange = "%s_fanout" % topic + self.queue = "%s_fanout" % topic self.durable = False + LOG.info(_("Writing to '%(exchange)s' fanout exchange"), + dict(exchange=self.exchange)) super(FanoutPublisher, self).__init__(connection=connection) -- cgit From 05065a72ab06879d8ddd48ab45bc870386a0562d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 23 Feb 2011 14:41:11 -0800 Subject: tests working again --- nova/tests/test_rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_rpc.py b/nova/tests/test_rpc.py index 4820e04fb..44d7c91eb 100644 --- a/nova/tests/test_rpc.py +++ b/nova/tests/test_rpc.py @@ -36,7 +36,7 @@ class RpcTestCase(test.TestCase): super(RpcTestCase, self).setUp() self.conn = rpc.Connection.instance(True) self.receiver = TestReceiver() - self.consumer = rpc.AdapterConsumer(connection=self.conn, + self.consumer = rpc.TopicAdapterConsumer(connection=self.conn, topic='test', proxy=self.receiver) self.consumer.attach_to_eventlet() @@ -97,7 +97,7 @@ class RpcTestCase(test.TestCase): nested = Nested() conn = rpc.Connection.instance(True) - consumer = rpc.AdapterConsumer(connection=conn, + consumer = rpc.TopicAdapterConsumer(connection=conn, topic='nested', proxy=nested) consumer.attach_to_eventlet() -- cgit From 05fc3ea219f36bc1c246179b25b1feb017888b01 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 23 Feb 2011 17:58:32 -0800 Subject: Tests all working again --- nova/compute/manager.py | 3 ++- nova/flags.py | 2 +- nova/network/manager.py | 7 ++++--- nova/scheduler/api.py | 1 - nova/scheduler/manager.py | 3 ++- nova/scheduler_manager.py | 22 ++++++++++++++++++---- nova/tests/test_service.py | 25 ++++++++++++++----------- nova/tests/test_test.py | 2 +- nova/volume/manager.py | 7 ++++--- 9 files changed, 46 insertions(+), 26 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index fd88158f7..b307ffa59 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -112,7 +112,8 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): self.driver = utils.import_object(compute_driver) self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) - super(ComputeManager, self).__init__(*args, **kwargs) + super(ComputeManager, self).__init__(service_name="compute", + *args, **kwargs) def init_host(self): """Do any initialization that needs to be run if this is a diff --git a/nova/flags.py b/nova/flags.py index 6f37c82f0..7036180fc 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -356,5 +356,5 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'kypervisor:xenserver;os:linux', +DEFINE_string('zone_capabilities', 'hypervisor:xenserver;os:linux', 'Key/Value tags which represent capabilities of this zone') diff --git a/nova/network/manager.py b/nova/network/manager.py index b36dd59cf..f5f5b17aa 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -55,7 +55,7 @@ from nova import db from nova import exception from nova import flags from nova import log as logging -from nova import manager +from nova import scheduler_manager from nova import utils from nova import rpc @@ -105,7 +105,7 @@ class AddressAlreadyAllocated(exception.Error): pass -class NetworkManager(manager.Manager): +class NetworkManager(scheduler_manager.SchedulerDependentManager): """Implements common network manager functionality. This class must be subclassed to support specific topologies. @@ -116,7 +116,8 @@ class NetworkManager(manager.Manager): if not network_driver: network_driver = FLAGS.network_driver self.driver = utils.import_object(network_driver) - super(NetworkManager, self).__init__(*args, **kwargs) + super(NetworkManager, self).__init__(service_name='network', + *args, **kwargs) def init_host(self): """Do any initialization that needs to be run if this is a diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 53d72507f..6a6bfc032 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -53,4 +53,3 @@ class API: kwargs = dict(method='update_service_capabilities', service_name=service_name, capabilities=capabilities) return rpc.fanout_cast(context, 'scheduler', kwargs) - diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 3ff43a9d9..6b5c6e246 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,7 +59,8 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() - def update_service_capabilities(self, context=None, service_name=None, capabilities={}): + def update_service_capabilities(self, context=None, service_name=None, + capabilities={}): """Process a compute node update.""" return self.zone_manager.update_compute_capabilities() diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index a45301617..65bd71c05 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -34,13 +34,27 @@ FLAGS = flags.FLAGS class SchedulerDependentManager(manager.Manager): - def __init__(self, host=None, db_driver=None): - self.last_capabilities = {} + + """Periodically send capability updates to the Scheduler services. + Services that need to update the Scheduler of their capabilities + should derive from this class. Otherwise they can derive from + manager.Manager directly. Updates are only sent after + update_service_capabilities is called with non-None values.""" + + def __init__(self, host=None, db_driver=None, service_name="undefined"): + self.last_capabilities = None + self.service_name = service_name super(SchedulerDependentManager, self).__init__(host, db_driver) + def update_service_capabilities(self, capabilities): + """Remember these capabilities to send on next periodic update.""" + self.last_capabilities = capabilities + def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" - logging.debug(_("*** Notifying Schedulers of capabilities ...")) - api.API.update_service_capabilities(context, 'compute', self.last_capabilities) + if self.last_capabilities: + logging.debug(_("*** Notifying Schedulers of capabilities ...")) + api.API.update_service_capabilities(context, self.service_name, + self.last_capabilities) super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index cb31a3c43..b006caadd 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -108,26 +108,29 @@ class ServiceTestCase(test.TestCase): app = service.Service.create(host=host, binary=binary) self.mox.StubOutWithMock(rpc, - 'AdapterConsumer', + 'TopicAdapterConsumer', use_mock_anything=True) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), + self.mox.StubOutWithMock(rpc, + 'FanoutAdapterConsumer', + use_mock_anything=True) + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), topic=topic, proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) + rpc.TopicAdapterConsumer) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), topic='%s.%s' % (topic, host), proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) + rpc.TopicAdapterConsumer) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='%s_fanout' % topic, + rpc.FanoutAdapterConsumer(connection=mox.IgnoreArg(), + topic=topic, proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) + rpc.FanoutAdapterConsumer) - rpc.AdapterConsumer.attach_to_eventlet() - rpc.AdapterConsumer.attach_to_eventlet() - rpc.AdapterConsumer.attach_to_eventlet() + rpc.TopicAdapterConsumer.attach_to_eventlet() + rpc.TopicAdapterConsumer.attach_to_eventlet() + rpc.FanoutAdapterConsumer.attach_to_eventlet() service_create = {'host': host, 'binary': binary, diff --git a/nova/tests/test_test.py b/nova/tests/test_test.py index e237674e6..35c838065 100644 --- a/nova/tests/test_test.py +++ b/nova/tests/test_test.py @@ -34,7 +34,7 @@ class IsolationTestCase(test.TestCase): def test_rpc_consumer_isolation(self): connection = rpc.Connection.instance(new=True) - consumer = rpc.TopicConsumer(connection, topic='compute') + consumer = rpc.TopicAdapterConsumer(connection, topic='compute') consumer.register_callback( lambda x, y: self.fail('I should never be called')) consumer.attach_to_eventlet() diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 3e8bc16b3..c53acf1e3 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -49,7 +49,7 @@ from nova import context from nova import exception from nova import flags from nova import log as logging -from nova import manager +from nova import scheduler_manager from nova import utils @@ -64,14 +64,15 @@ flags.DEFINE_boolean('use_local_volumes', True, 'if True, will not discover local volumes') -class VolumeManager(manager.Manager): +class VolumeManager(scheduler_manager.SchedulerDependentManager): """Manages attachable block storage devices.""" def __init__(self, volume_driver=None, *args, **kwargs): """Load the driver from the one specified in args, or from flags.""" if not volume_driver: volume_driver = FLAGS.volume_driver self.driver = utils.import_object(volume_driver) - super(VolumeManager, self).__init__(*args, **kwargs) + super(VolumeManager, self).__init__(service_name='volume', + *args, **kwargs) # NOTE(vish): Implementation specific db handling is done # by the driver. self.driver.db = self.db -- cgit From 65b9dfbea28f1607ef471e78b73ba77183d943f6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 01:53:01 -0800 Subject: capability aggregation working --- nova/api/openstack/zones.py | 15 ++++++++++++--- nova/scheduler/api.py | 15 +++++++++++++-- nova/scheduler/manager.py | 12 +++++++++--- nova/scheduler/zone_manager.py | 35 ++++++++++++++++++++++++++++++++--- nova/scheduler_manager.py | 6 ++---- 5 files changed, 68 insertions(+), 15 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 989b3a235..c6c27dd4b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -15,9 +15,10 @@ import common +from nova import db from nova import flags +from nova import log as logging from nova import wsgi -from nova import db from nova.scheduler import api @@ -67,8 +68,16 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - return dict(zone=dict(name=FLAGS.zone_name, - capabilities=FLAGS.zone_capabilities)) + items = api.API().get_zone_capabilities(req.environ['nova.context']) + + zone = dict(name=FLAGS.zone_name) + caps = FLAGS.zone_capabilities.split(';') + for cap in caps: + key_values = cap.split(':') + zone[key_values[0]] = key_values[1] + for item, (min_value, max_value) in items.iteritems(): + zone[item] = "%s,%s" % (min_value, max_value) + return dict(zone=zone) def show(self, req, id): """Return data about the given zone id""" diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6a6bfc032..ac38350ed 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -43,13 +43,24 @@ class API: return rpc.call(context, queue, kwargs) def get_zone_list(self, context): + """Return a list of zones assoicated with this zone.""" items = self._call_scheduler('get_zone_list', context) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') return items + def get_zone_capabilities(self, context, service=None): + """Returns a dict of key, value capabilities for this zone, + or for a particular class of services running in this zone.""" + return self._call_scheduler('get_zone_capabilities', context=context, + params=dict(service=service)) + @classmethod - def update_service_capabilities(cls, context, service_name, capabilities): + def update_service_capabilities(cls, context, service_name, host, + capabilities): + """Send an update to all the scheduler services informing them + of the capabilities of this service.""" kwargs = dict(method='update_service_capabilities', - service_name=service_name, capabilities=capabilities) + args=dict(service_name=service_name, host=host, + capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 6b5c6e246..1bda77d89 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,10 +59,16 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() + def get_zone_capabilities(self, context=None, service=None): + """Get the normalized set of capabilites for this zone, + or for a particular service.""" + return self.zone_manager.get_zone_capabilities(context, service) + def update_service_capabilities(self, context=None, service_name=None, - capabilities={}): - """Process a compute node update.""" - return self.zone_manager.update_compute_capabilities() + host=None, capabilities={}): + """Process a capability update from a service node.""" + self.zone_manager.update_service_capabilities(service_name, + host, capabilities) def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index eedc5c235..09c9811f3 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -105,13 +105,37 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} - self.compute_states = {} + self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" return [zone.to_dict() for zone in self.zone_states.values()] + def get_zone_capabilities(self, context, service=None): + """Roll up all the individual host info to generic 'service' + capabilities. Each capability is aggregated into + _min and _max values.""" + service_dict = self.service_states + if service: + service_dict = dict(service_name=service, + hosts=self.service_states.get(service, {})) + + # TODO(sandy) - be smarter about fabricating this structure. + # But it's likely to change once we understand what the Best-Match + # code will need better. + combined = {} # { _ : (min, max), ... } + for service_name, host_dict in service_dict.iteritems(): + for host, caps_dict in host_dict.iteritems(): + for cap, value in caps_dict.iteritems(): + key = "%s_%s" % (service_name, cap) + min_value, max_value = combined.get(key, (value, value)) + min_value = min(min_value, value) + max_value = max(max_value, value) + combined[key] = (min_value, max_value) + + return combined + def _refresh_from_db(self, context): """Make our zone state map match the db.""" # Add/update existing zones ... @@ -143,5 +167,10 @@ class ZoneManager(object): self._refresh_from_db(context) self._poll_zones(context) - def update_compute_capabilities(self): - logging.debug(_("****** UPDATE COMPUTE CAPABILITIES *******")) + def update_service_capabilities(self, service_name, host, capabilities): + """Update the per-service capabilities based on this notification.""" + logging.debug(_("Received %(service_name)s service update from " + "%(host)s: %(capabilities)s") % locals()) + service_caps = self.service_states.get(service_name, {}) + service_caps[host] = capabilities + self.service_states[service_name] = service_caps diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index 65bd71c05..ca39b85dd 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -34,13 +34,11 @@ FLAGS = flags.FLAGS class SchedulerDependentManager(manager.Manager): - """Periodically send capability updates to the Scheduler services. Services that need to update the Scheduler of their capabilities should derive from this class. Otherwise they can derive from manager.Manager directly. Updates are only sent after update_service_capabilities is called with non-None values.""" - def __init__(self, host=None, db_driver=None, service_name="undefined"): self.last_capabilities = None self.service_name = service_name @@ -53,8 +51,8 @@ class SchedulerDependentManager(manager.Manager): def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" if self.last_capabilities: - logging.debug(_("*** Notifying Schedulers of capabilities ...")) + logging.debug(_("Notifying Schedulers of capabilities ...")) api.API.update_service_capabilities(context, self.service_name, - self.last_capabilities) + self.host, self.last_capabilities) super(SchedulerDependentManager, self).periodic_tasks(context) -- cgit From c8df2602fd8f4f2cb7716e6283f3779c6895a479 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 14:32:25 -0800 Subject: service capabilities test --- nova/scheduler/driver.py | 7 +++++++ nova/scheduler/manager.py | 3 ++- nova/tests/test_zones.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 66e46c1b9..317a039cc 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -45,6 +45,13 @@ class WillNotSchedule(exception.Error): class Scheduler(object): """The base class that all Scheduler clases should inherit from.""" + def __init__(self): + self.zone_manager = None + + def set_zone_manager(self, zone_manager): + """Called by the Scheduler Service to supply a ZoneManager.""" + self.zone_manager = zone_manager + @staticmethod def service_is_up(service): """Check whether a service is up based on last heartbeat.""" diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 1bda77d89..d3d338943 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -41,10 +41,11 @@ flags.DEFINE_string('scheduler_driver', class SchedulerManager(manager.Manager): """Chooses a host to run instances on.""" def __init__(self, scheduler_driver=None, *args, **kwargs): + self.zone_manager = zone_manager.ZoneManager() if not scheduler_driver: scheduler_driver = FLAGS.scheduler_driver self.driver = utils.import_object(scheduler_driver) - self.zone_manager = zone_manager.ZoneManager() + self.driver.set_zone_manager(self.zone_manager) super(SchedulerManager, self).__init__(*args, **kwargs) def __getattr__(self, key): diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 5a52a0506..3ca71d5f1 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -76,6 +76,34 @@ class ZoneManagerTestCase(test.TestCase): self.assertEquals(len(zm.zone_states), 1) self.assertEquals(zm.zone_states[1].username, 'user1') + def test_service_capabilities(self): + zm = zone_manager.ZoneManager() + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, {}) + + zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2))) + + zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) + + zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) + + zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), + svc10_a=(99, 99), svc10_b=(99, 99))) + + zm.update_service_capabilities("svc1", "host3", dict(c=5)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), + svc1_c=(5, 5), svc10_a=(99, 99), + svc10_b=(99, 99))) + def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 47bbfaab52642f3ff79bcdefb8d705fb02b549f9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 15:23:15 -0800 Subject: new tests --- nova/api/openstack/zones.py | 4 ++-- nova/scheduler/api.py | 37 ++++++++++++++++++---------------- nova/scheduler/zone_manager.py | 3 +-- nova/tests/api/openstack/test_zones.py | 30 ++++++++++++++++++++++++--- nova/tests/test_zones.py | 6 ++++++ 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index c6c27dd4b..fecbd6fa3 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -53,7 +53,7 @@ class Controller(wsgi.Controller): """Return all zones in brief""" # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... - items = api.API().get_zone_list(req.environ['nova.context']) + items = api.API.get_zone_list(req.environ['nova.context']) if not items: items = db.zone_get_all(req.environ['nova.context']) @@ -68,7 +68,7 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - items = api.API().get_zone_capabilities(req.environ['nova.context']) + items = api.API.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) caps = FLAGS.zone_capabilities.split(';') diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index ac38350ed..fcff2f146 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -25,34 +25,37 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.api') -class API: - """API for interacting with the scheduler.""" +def _call_scheduler(method, context, params=None): + """Generic handler for RPC calls to the scheduler. - def _call_scheduler(self, method, context, params=None): - """Generic handler for RPC calls to the scheduler. + :param params: Optional dictionary of arguments to be passed to the + scheduler worker - :param params: Optional dictionary of arguments to be passed to the - scheduler worker + :retval: Result returned by scheduler worker + """ + if not params: + params = {} + queue = FLAGS.scheduler_topic + kwargs = {'method': method, 'args': params} + return rpc.call(context, queue, kwargs) - :retval: Result returned by scheduler worker - """ - if not params: - params = {} - queue = FLAGS.scheduler_topic - kwargs = {'method': method, 'args': params} - return rpc.call(context, queue, kwargs) - def get_zone_list(self, context): +class API: + """API for interacting with the scheduler.""" + + @classmethod + def get_zone_list(cls, context): """Return a list of zones assoicated with this zone.""" - items = self._call_scheduler('get_zone_list', context) + items = _call_scheduler('get_zone_list', context) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') return items - def get_zone_capabilities(self, context, service=None): + @classmethod + def get_zone_capabilities(cls, context, service=None): """Returns a dict of key, value capabilities for this zone, or for a particular class of services running in this zone.""" - return self._call_scheduler('get_zone_capabilities', context=context, + return _call_scheduler('get_zone_capabilities', context=context, params=dict(service=service)) @classmethod diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 09c9811f3..c1a50dbc3 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -118,8 +118,7 @@ class ZoneManager(object): _min and _max values.""" service_dict = self.service_states if service: - service_dict = dict(service_name=service, - hosts=self.service_states.get(service, {})) + service_dict = {service: self.service_states.get(service, {})} # TODO(sandy) - be smarter about fabricating this structure. # But it's likely to change once we understand what the Best-Match diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 82b892b9e..33a66df0b 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -75,6 +75,10 @@ def zone_get_all_db(context): ] +def zone_caps(method, context, params): + return dict() + + class ZonesTest(test.TestCase): def setUp(self): super(ZonesTest, self).setUp() @@ -93,13 +97,18 @@ class ZonesTest(test.TestCase): self.stubs.Set(nova.db, 'zone_create', zone_create) self.stubs.Set(nova.db, 'zone_delete', zone_delete) + self.old_zone_name = FLAGS.zone_name + self.old_zone_caps = FLAGS.zone_capabilities + def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + FLAGS.zone_name = self.old_zone_name + FLAGS.zone_capabilities = self.old_zone_caps super(ZonesTest, self).tearDown() def test_get_zone_list_scheduler(self): - self.stubs.Set(api.API, '_call_scheduler', zone_get_all_scheduler) + self.stubs.Set(api, '_call_scheduler', zone_get_all_scheduler) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -108,8 +117,7 @@ class ZonesTest(test.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_list_db(self): - self.stubs.Set(api.API, '_call_scheduler', - zone_get_all_scheduler_empty) + self.stubs.Set(api, '_call_scheduler', zone_get_all_scheduler_empty) self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) @@ -162,3 +170,19 @@ class ZonesTest(test.TestCase): self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('username' in res_dict['zone']) + + def test_zone_info(self): + FLAGS.zone_name = 'darksecret' + FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' + self.stubs.Set(api, '_call_scheduler', zone_caps) + + body = dict(zone=dict(username='zeb', password='sneaky')) + req = webob.Request.blank('/v1.0/zones/info') + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['zone']['name'], 'darksecret') + self.assertEqual(res_dict['zone']['cap1'], 'a,b') + self.assertEqual(res_dict['zone']['cap2'], 'c,d') + diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 3ca71d5f1..79d766f28 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -104,6 +104,12 @@ class ZoneManagerTestCase(test.TestCase): svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) + caps = zm.get_zone_capabilities(self, 'svc1') + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), + svc1_c=(5, 5))) + caps = zm.get_zone_capabilities(self, 'svc10') + self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) + def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 307dcb7906ff066e2883cdee8998dfa78ebc8221 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 15:44:27 -0800 Subject: sorry, pep8 --- nova/tests/api/openstack/test_zones.py | 5 ++--- nova/tests/test_zones.py | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 33a66df0b..a40d46749 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -76,7 +76,7 @@ def zone_get_all_db(context): def zone_caps(method, context, params): - return dict() + return dict() class ZonesTest(test.TestCase): @@ -175,7 +175,7 @@ class ZonesTest(test.TestCase): FLAGS.zone_name = 'darksecret' FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' self.stubs.Set(api, '_call_scheduler', zone_caps) - + body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/info') @@ -185,4 +185,3 @@ class ZonesTest(test.TestCase): self.assertEqual(res_dict['zone']['name'], 'darksecret') self.assertEqual(res_dict['zone']['cap1'], 'a,b') self.assertEqual(res_dict['zone']['cap2'], 'c,d') - diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 79d766f28..48e1442cf 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -88,28 +88,28 @@ class ZoneManagerTestCase(test.TestCase): zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) - + zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) - + zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc10_a=(99, 99), svc10_b=(99, 99))) - + zm.update_service_capabilities("svc1", "host3", dict(c=5)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) - + caps = zm.get_zone_capabilities(self, 'svc1') self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5))) caps = zm.get_zone_capabilities(self, 'svc10') self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) - + def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 7afebad78de462918b89d61f5d8e0cee8bc11068 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 4 Mar 2011 13:45:43 -0500 Subject: Fix api logging to show proper path and controller:action. --- nova/api/ec2/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5adc2c075..2493adc95 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -61,10 +61,13 @@ class RequestLogging(wsgi.Middleware): return rv def log_request_completion(self, response, request, start): - controller = request.environ.get('ec2.controller', None) - if controller: - controller = controller.__class__.__name__ - action = request.environ.get('ec2.action', None) + apireq = request.environ.get('ec2.request', None) + if apirequest: + controller = apireq.controller + action = apireq.action + else: + controller = None + action = None ctxt = request.environ.get('ec2.context', None) delta = utils.utcnow() - start seconds = delta.seconds @@ -75,7 +78,7 @@ class RequestLogging(wsgi.Middleware): microseconds, request.remote_addr, request.method, - request.path_info, + "%s%s" % (request.script_name, request.path_info), controller, action, response.status_int, -- cgit From ceccffaab6fb5fce3b0951b5a8eea65f523e8563 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 4 Mar 2011 19:13:27 -0500 Subject: apirequest -> apireq. --- nova/api/ec2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 2493adc95..b06289e67 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -62,7 +62,7 @@ class RequestLogging(wsgi.Middleware): def log_request_completion(self, response, request, start): apireq = request.environ.get('ec2.request', None) - if apirequest: + if apireq: controller = apireq.controller action = apireq.action else: -- cgit From f91d7925761f8204fdd46435ff57d74ae17483cf Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 15 Mar 2011 18:29:26 -0700 Subject: first pass openstack redirect working --- etc/api-paste.ini | 5 ++++- nova/api/openstack/servers.py | 1 + nova/compute/api.py | 3 ++- nova/compute/manager.py | 6 +++++- nova/exception.py | 6 +++--- nova/scheduler/api.py | 13 +++++++++++-- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/etc/api-paste.ini b/etc/api-paste.ini index 9f7e93d4c..c0077939b 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -70,7 +70,10 @@ use = egg:Paste#urlmap /v1.0: openstackapi [pipeline:openstackapi] -pipeline = faultwrap auth ratelimit osapiapp +pipeline = faultwrap zonerouter auth ratelimit osapiapp + +[filter:zonerouter] +paste.filter_factory = nova.api.zone_redirect:ZoneRedirectMiddleware.factory [filter:faultwrap] paste.filter_factory = nova.api.openstack:FaultWrapper.factory diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 85999764f..ffcbe628c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -343,6 +343,7 @@ class Controller(wsgi.Controller): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: + LOG.debug(_("*** Compute.api::pause %s"), id) self.compute_api.pause(ctxt, id) except: readable = traceback.format_exc() diff --git a/nova/compute/api.py b/nova/compute/api.py index 8865a1654..1185b9964 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -347,7 +347,8 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - rv = self.db.instance_get(context, instance_id) + rv = self.scheduler_api.get_instance_or_reroute(context, instance_id) + #rv = self.db.instance_get(context, instance_id) return dict(rv.iteritems()) def get_all(self, context, project_id=None, reservation_id=None, diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ebe1ce6f0..499b212e2 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -48,6 +48,7 @@ from nova import scheduler_manager from nova import rpc from nova import utils from nova.compute import power_state +from nova.scheduler import api as scheduler_api FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', @@ -521,7 +522,10 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): def pause_instance(self, context, instance_id): """Pause an instance on this server.""" context = context.elevated() - instance_ref = self.db.instance_get(context, instance_id) + LOG.debug(_('*** instance %s: starting pause'), instance_id) + instance_ref = scheduler_api.get_instance_or_reroute(context, + instance_id) + #instance_ref = self.db.instance_get(context, instance_id) LOG.audit(_('instance %s: pausing'), instance_id, context=context) self.db.instance_set_state(context, instance_id, diff --git a/nova/exception.py b/nova/exception.py index ce8daf048..d0baa2e29 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -93,9 +93,9 @@ class TimeoutException(Error): class ZoneRouteException(Error): - def __init__(self, zone, *args, **kwargs): - self.zone = zone - super(ZoneRouteException, self).__init__(args, kwargs) + def __init__(self, zones, *args, **kwargs): + self.zones = zones + super(ZoneRouteException, self).__init__(*args, **kwargs) class DBError(Error): diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 8f9806f77..c0e28a0a9 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -73,13 +73,22 @@ class API(object): args=dict(service_name=service_name, host=host, capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) + + @classmethod + def get_instance_or_reroute(cls, context, instance_id): + instance = db.instance_get(context, instance_id) + zones = db.zone_get_all(context) + + LOG.debug("*** Firing ZoneRouteException") + # Throw a reroute Exception for the middleware to pick up. + raise exception.ZoneRouteException(zones) @classmethod - def get_queue_for_instance(cls, context, service, instance_id) + def get_queue_for_instance(cls, context, service, instance_id): instance = db.instance_get(context, instance_id) zone = db.get_zone(instance.zone.id) if cls._is_current_zone(zone): - return db.queue_get_for(context, service, instance['host']): + return db.queue_get_for(context, service, instance['host']) # Throw a reroute Exception for the middleware to pick up. raise exception.ZoneRouteException(zone) -- cgit From 7a61965908ebfc076ad3b1d9cdc5773ade50bf75 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 15 Mar 2011 20:30:27 -0700 Subject: response working --- nova/api/zone_redirect.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 nova/api/zone_redirect.py diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py new file mode 100644 index 000000000..5e40a82dd --- /dev/null +++ b/nova/api/zone_redirect.py @@ -0,0 +1,100 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.import datetime + +"""Reroutes calls to child zones on ZoneRouteException's.""" + +import httplib +import re +import webob +import webob.dec +import webob.exc +import urlparse +import urllib + +from nova import exception +from nova import log as logging +from nova import wsgi + +import novaclient.client as client + +try: + import json +except ImportError: + import simplejson as json + + +LOG = logging.getLogger('server') + + +class ZoneRedirectMiddleware(wsgi.Middleware): + """Catches Zone Routing exceptions and delegates the call + to child zones.""" + + @webob.dec.wsgify + def __call__(self, req): + try: + return req.get_response(self.application) + except exception.ZoneRouteException as e: + if len(e.zones) == 0: + exc = webob.exc.HTTPInternalServerError(explanation= + _("No zones to reroute to.")) + return faults.Fault(exc) + + zone = e.zones[0] + # Todo(sandy): This only works for OpenStack API currently. + # Needs to be broken out into a driver. + url = zone.api_url + LOG.info(_("Zone redirect to:[url:%(api_url)s, username:%(username)s]" + % dict(api_url=zone.api_url, username=zone.username))) + + LOG.info(_("Zone Initial Req: %s"), req) + nova = client.OpenStackClient(zone.username, zone.password, + zone.api_url) + nova.authenticate() + new_req = req.copy() + #m = re.search('(https?://.+)/(v\d+\.\d+)/', url) + + scheme, netloc, path, query, frag = urlparse.urlsplit(new_req.path_qs) + query = urlparse.parse_qsl(query) + LOG.debug("**** QUERY=%s^%s^%s", path, query, frag) + query = [(key, value) for key, value in query if key != 'fresh'] + query = urllib.urlencode(query) + url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + + m = re.search('/(v\d+\.\d+)/(.+)', url) + version = m.group(1) + resource = m.group(2) + + LOG.info(_("New Request Data: %s"), new_req.body) + #LOG.info(_("New Request Headers: %s"), new_req.headers) + LOG.info(_("New Request Path: %s"), resource) + if req.method == 'GET': + response, body = nova.get(resource, body=new_req.body) + elif req.method == 'POST': + response, body = nova.post(resource, body=new_req.body) + elif req.method == 'PUT': + response, body = nova.put(resource, body=new_req.body) + elif req.method == 'DELETE': + response, body = nova.delete(resource, body=new_req.body) + #response, body = nova.request(req.path_qs, headers=new_req.headers, body=new_req.body) + LOG.info(_("Zone Response: %s / %s"), response, body) + res = webob.Response() + res.status = response['status'] + res.content_type = response['content-type'] + res.body = json.dumps(body) + LOG.info(_("Zone WebOb Response: %s"), res) + return res -- cgit From ddeede35a5036aa3c80742fde69468aedbc74892 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 06:09:00 -0700 Subject: Error codes handled properly now --- nova/api/zone_redirect.py | 94 ++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 5e40a82dd..ad47a6216 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -30,6 +30,7 @@ from nova import log as logging from nova import wsgi import novaclient.client as client +import novaclient.exceptions as osexceptions try: import json @@ -49,52 +50,63 @@ class ZoneRedirectMiddleware(wsgi.Middleware): try: return req.get_response(self.application) except exception.ZoneRouteException as e: - if len(e.zones) == 0: + if not e.zones: exc = webob.exc.HTTPInternalServerError(explanation= _("No zones to reroute to.")) return faults.Fault(exc) - zone = e.zones[0] # Todo(sandy): This only works for OpenStack API currently. # Needs to be broken out into a driver. - url = zone.api_url - LOG.info(_("Zone redirect to:[url:%(api_url)s, username:%(username)s]" - % dict(api_url=zone.api_url, username=zone.username))) - - LOG.info(_("Zone Initial Req: %s"), req) - nova = client.OpenStackClient(zone.username, zone.password, - zone.api_url) - nova.authenticate() - new_req = req.copy() - #m = re.search('(https?://.+)/(v\d+\.\d+)/', url) - - scheme, netloc, path, query, frag = urlparse.urlsplit(new_req.path_qs) - query = urlparse.parse_qsl(query) - LOG.debug("**** QUERY=%s^%s^%s", path, query, frag) - query = [(key, value) for key, value in query if key != 'fresh'] - query = urllib.urlencode(query) - url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - - m = re.search('/(v\d+\.\d+)/(.+)', url) - version = m.group(1) - resource = m.group(2) - - LOG.info(_("New Request Data: %s"), new_req.body) - #LOG.info(_("New Request Headers: %s"), new_req.headers) - LOG.info(_("New Request Path: %s"), resource) - if req.method == 'GET': - response, body = nova.get(resource, body=new_req.body) - elif req.method == 'POST': - response, body = nova.post(resource, body=new_req.body) - elif req.method == 'PUT': - response, body = nova.put(resource, body=new_req.body) - elif req.method == 'DELETE': - response, body = nova.delete(resource, body=new_req.body) - #response, body = nova.request(req.path_qs, headers=new_req.headers, body=new_req.body) - LOG.info(_("Zone Response: %s / %s"), response, body) + for zone in e.zones: + url = zone.api_url + LOG.info(_("Zone redirect to:[url:%(api_url)s, " + "username:%(username)s]" + % dict(api_url=zone.api_url, + username=zone.username))) + + nova = client.OpenStackClient(zone.username, zone.password, + zone.api_url) + nova.authenticate() + new_req = req.copy() + + scheme, netloc, path, query, frag = \ + urlparse.urlsplit(new_req.path_qs) + query = urlparse.parse_qsl(query) + query = [(key, value) for key, value in query if key != 'fresh'] + query = urllib.urlencode(query) + url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + + m = re.search('/(v\d+\.\d+)/(.+)', url) + version = m.group(1) + resource = m.group(2) + + #LOG.info(_("New Request Data: %s"), new_req.body) + #LOG.info(_("New Request Path: %s"), resource) + try: + if req.method == 'GET': + response, body = nova.get(resource, body=new_req.body) + elif req.method == 'POST': + response, body = nova.post(resource, body=new_req.body) + elif req.method == 'PUT': + response, body = nova.put(resource, body=new_req.body) + elif req.method == 'DELETE': + response, body = nova.delete(resource, + body=new_req.body) + except osexceptions.OpenStackException, e: + LOG.info(_("Zone returned error: %s ('%s', '%s')"), + e.code, e.message, e.details) + continue + + LOG.info(_("Zone Response: %s [%s]/ %s"), response, + response.status, body) + if response.status == 200: + res = webob.Response() + res.status = response['status'] + res.content_type = response['content-type'] + res.body = json.dumps(body) + return res + + LOG.info(_("Returning 404 ...")) res = webob.Response() - res.status = response['status'] - res.content_type = response['content-type'] - res.body = json.dumps(body) - LOG.info(_("Zone WebOb Response: %s"), res) + res.status = "404" return res -- cgit From 8dffae687e78a1fa2a8cf0d321d64ee95a35cc1f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 06:47:27 -0700 Subject: Checks locally before routing --- nova/api/zone_redirect.py | 34 +++++++++++++++------------------- nova/scheduler/api.py | 11 ++++++++--- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index ad47a6216..fec1b1af3 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -57,9 +57,20 @@ class ZoneRedirectMiddleware(wsgi.Middleware): # Todo(sandy): This only works for OpenStack API currently. # Needs to be broken out into a driver. + new_req = req.copy() + + scheme, netloc, path, query, frag = \ + urlparse.urlsplit(new_req.path_qs) + query = urlparse.parse_qsl(query) + query = [(key, value) for key, value in query if key != 'fresh'] + query = urllib.urlencode(query) + url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + + m = re.search('/v\d+\.\d+/(.+)', url) + resource = m.group(1) + for zone in e.zones: - url = zone.api_url - LOG.info(_("Zone redirect to:[url:%(api_url)s, " + LOG.debug(_("Zone redirect to:[url:%(api_url)s, " "username:%(username)s]" % dict(api_url=zone.api_url, username=zone.username))) @@ -67,21 +78,6 @@ class ZoneRedirectMiddleware(wsgi.Middleware): nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() - new_req = req.copy() - - scheme, netloc, path, query, frag = \ - urlparse.urlsplit(new_req.path_qs) - query = urlparse.parse_qsl(query) - query = [(key, value) for key, value in query if key != 'fresh'] - query = urllib.urlencode(query) - url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - - m = re.search('/(v\d+\.\d+)/(.+)', url) - version = m.group(1) - resource = m.group(2) - - #LOG.info(_("New Request Data: %s"), new_req.body) - #LOG.info(_("New Request Path: %s"), resource) try: if req.method == 'GET': response, body = nova.get(resource, body=new_req.body) @@ -97,7 +93,7 @@ class ZoneRedirectMiddleware(wsgi.Middleware): e.code, e.message, e.details) continue - LOG.info(_("Zone Response: %s [%s]/ %s"), response, + LOG.debug(_("Zone Response: %s [%s]/ %s"), response, response.status, body) if response.status == 200: res = webob.Response() @@ -106,7 +102,7 @@ class ZoneRedirectMiddleware(wsgi.Middleware): res.body = json.dumps(body) return res - LOG.info(_("Returning 404 ...")) + LOG.debug(_("Zone Redirect Middleware returning 404 ...")) res = webob.Response() res.status = "404" return res diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index c0e28a0a9..48da5bcfc 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -76,11 +76,16 @@ class API(object): @classmethod def get_instance_or_reroute(cls, context, instance_id): - instance = db.instance_get(context, instance_id) - zones = db.zone_get_all(context) + try: + instance = db.instance_get(context, instance_id) + return instance + except exception.InstanceNotFound, e: + LOG.debug(_("Instance %(instance_id)s not found locally: '%(e)s'" % + locals())) - LOG.debug("*** Firing ZoneRouteException") # Throw a reroute Exception for the middleware to pick up. + LOG.debug("Firing ZoneRouteException") + zones = db.zone_get_all(context) raise exception.ZoneRouteException(zones) @classmethod -- cgit From 4d057c9c2df77816ead6f30fa2795148aa8148d3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 11:44:40 -0700 Subject: Refactored ZoneRedirect into ZoneChildHelper so ZoneManager can use this too. --- nova/api/zone_redirect.py | 79 +++++++++++++++++++++++------------------- nova/compute/api.py | 8 ++--- nova/scheduler/api.py | 40 ++++++++++++++++----- nova/scheduler/zone_manager.py | 2 +- 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index fec1b1af3..0adf94046 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -28,6 +28,7 @@ import urllib from nova import exception from nova import log as logging from nova import wsgi +from nova.scheduler import api import novaclient.client as client import novaclient.exceptions as osexceptions @@ -41,6 +42,43 @@ except ImportError: LOG = logging.getLogger('server') +class RequestForwarder(api.ChildZoneHelper): + + def __init__(self, resource, method, body): + self.resource = resource + self.method = method + self.body = body + + def process(self, client, zone): + api_url = zone.api_url + LOG.debug(_("Zone redirect to: %(api_url)s, " % locals())) + try: + if self.method == 'GET': + response, body = client.get(self.resource, body=self.body) + elif self.method == 'POST': + response, body = client.post(self.resource, body=self.body) + elif self.method == 'PUT': + response, body = client.put(self.resource, body=self.body) + elif self.method == 'DELETE': + response, body = client.delete(self.resource, body=self.body) + except osexceptions.OpenStackException, e: + LOG.info(_("Zone returned error: %s ('%s', '%s')"), + e.code, e.message, e.details) + res = webob.Response() + res.status = "404" + return res + + status = response.status + LOG.debug(_("Zone %(api_url)s response: " + "%(response)s [%(status)s]/ %(body)s") % + locals()) + res = webob.Response() + res.status = response['status'] + res.content_type = response['content-type'] + res.body = json.dumps(body) + return res + + class ZoneRedirectMiddleware(wsgi.Middleware): """Catches Zone Routing exceptions and delegates the call to child zones.""" @@ -57,10 +95,8 @@ class ZoneRedirectMiddleware(wsgi.Middleware): # Todo(sandy): This only works for OpenStack API currently. # Needs to be broken out into a driver. - new_req = req.copy() - scheme, netloc, path, query, frag = \ - urlparse.urlsplit(new_req.path_qs) + urlparse.urlsplit(req.path_qs) query = urlparse.parse_qsl(query) query = [(key, value) for key, value in query if key != 'fresh'] query = urllib.urlencode(query) @@ -69,38 +105,11 @@ class ZoneRedirectMiddleware(wsgi.Middleware): m = re.search('/v\d+\.\d+/(.+)', url) resource = m.group(1) - for zone in e.zones: - LOG.debug(_("Zone redirect to:[url:%(api_url)s, " - "username:%(username)s]" - % dict(api_url=zone.api_url, - username=zone.username))) - - nova = client.OpenStackClient(zone.username, zone.password, - zone.api_url) - nova.authenticate() - try: - if req.method == 'GET': - response, body = nova.get(resource, body=new_req.body) - elif req.method == 'POST': - response, body = nova.post(resource, body=new_req.body) - elif req.method == 'PUT': - response, body = nova.put(resource, body=new_req.body) - elif req.method == 'DELETE': - response, body = nova.delete(resource, - body=new_req.body) - except osexceptions.OpenStackException, e: - LOG.info(_("Zone returned error: %s ('%s', '%s')"), - e.code, e.message, e.details) - continue - - LOG.debug(_("Zone Response: %s [%s]/ %s"), response, - response.status, body) - if response.status == 200: - res = webob.Response() - res.status = response['status'] - res.content_type = response['content-type'] - res.body = json.dumps(body) - return res + forwarder = RequestForwarder(resource, req.method, req.body) + for result in forwarder.start(e.zones): + # Todo(sandy): We need to aggregate multiple successes. + if result.status_int == 200: + return result LOG.debug(_("Zone Redirect Middleware returning 404 ...")) res = webob.Response() diff --git a/nova/compute/api.py b/nova/compute/api.py index 1185b9964..215257217 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -385,9 +385,7 @@ class API(base.Base): if not host: instance = self.get(context, instance_id) host = instance['host'] - #queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - queue = self.scheduler_api.get_queue_for_instance(context, - FLAGS.compute_topic, host) + queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) params['instance_id'] = instance_id kwargs = {'method': method, 'args': params} rpc.cast(context, queue, kwargs) @@ -406,9 +404,7 @@ class API(base.Base): if not host: instance = self.get(context, instance_id) host = instance["host"] - #queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - queue = self.scheduler_api.get_queue_for_instance(context, - FLAGS.compute_topic, host) + queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) params['instance_id'] = instance_id kwargs = {'method': method, 'args': params} return rpc.call(context, queue, kwargs) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 48da5bcfc..073784f31 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -23,6 +23,10 @@ from nova import flags from nova import log as logging from nova import rpc +import novaclient.client as client + +from eventlet import greenpool + FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.api') @@ -76,6 +80,8 @@ class API(object): @classmethod def get_instance_or_reroute(cls, context, instance_id): + """Return an instance from the db or throw a ZoneRouteException + if not found.""" try: instance = db.instance_get(context, instance_id) return instance @@ -88,12 +94,30 @@ class API(object): zones = db.zone_get_all(context) raise exception.ZoneRouteException(zones) - @classmethod - def get_queue_for_instance(cls, context, service, instance_id): - instance = db.instance_get(context, instance_id) - zone = db.get_zone(instance.zone.id) - if cls._is_current_zone(zone): - return db.queue_get_for(context, service, instance['host']) - # Throw a reroute Exception for the middleware to pick up. - raise exception.ZoneRouteException(zone) +def _wrap_method(function, self): + def _wrap(*args, **kwargs): + return function(self, *args, **kwargs) + return _wrap + + +def _process(self, zone): + nova = client.OpenStackClient(zone.username, zone.password, + zone.api_url) + nova.authenticate() + return self.process(nova, zone) + + +class ChildZoneHelper(object): + """Delegate a call to a set of Child Zones and wait for their + responses. Could be used for Zone Redirect or by the Scheduler + plug-ins to query the children.""" + + def start(self, zone_list): + self.green_pool = greenpool.GreenPool() + return [ result for result in self.green_pool.imap( + _wrap_method(_process, self), zone_list)] + + def process(self, client, zone): + """Derived class must override.""" + pass diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index c1a50dbc3..d32cc2e8f 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -104,7 +104,7 @@ class ZoneManager(object): """Keeps the zone states updated.""" def __init__(self): self.last_zone_db_check = datetime.min - self.zone_states = {} + self.zone_states = {} # { : ZoneState } self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() -- cgit From 703e680aa6d0da1953ec6f8ae3a6aa66dc9fad7e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 16 Mar 2011 16:13:24 -0700 Subject: Fix the errors that pylint was reporting on this file This was meant more as a test of whether pylint was now returning false-positives. It looks like the bugs it's reporting are at least partially real. --- nova/api/openstack/servers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 3ecd4fb01..dfaf35128 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -25,8 +25,9 @@ from nova import compute from nova import exception from nova import flags from nova import log as logging -from nova import wsgi +from nova import quota from nova import utils +from nova import wsgi from nova.api.openstack import common from nova.api.openstack import faults from nova.auth import manager as auth_manager @@ -188,7 +189,7 @@ class Controller(wsgi.Controller): key_data=key_data, metadata=metadata, injected_files=injected_files) - except QuotaError as error: + except quota.QuotaError as error: self._handle_quota_error(error) server = _translate_keys(instances[0]) @@ -238,7 +239,7 @@ class Controller(wsgi.Controller): injected_files.append((path, contents)) return injected_files - def _handle_quota_errors(self, error): + def _handle_quota_error(self, error): """ Reraise quota errors as api-specific http exceptions """ -- cgit From a1e2959312b51757653447de3e8c9e92029da6fd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 16 Mar 2011 16:23:31 -0700 Subject: Fix a few of the more obvious non-errors while we're in here --- nova/api/openstack/servers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index dfaf35128..42cf693de 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -15,11 +15,10 @@ import base64 import hashlib -import json import traceback -from xml.dom import minidom from webob import exc +from xml.dom import minidom from nova import compute from nova import exception @@ -33,7 +32,6 @@ from nova.api.openstack import faults from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state -import nova.api.openstack LOG = logging.getLogger('server') @@ -270,7 +268,7 @@ class Controller(wsgi.Controller): update_dict['admin_pass'] = inst_dict['server']['adminPass'] try: self.compute_api.set_admin_password(ctxt, id) - except exception.TimeoutException, e: + except exception.TimeoutException: return exc.HTTPRequestTimeout() if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] -- cgit From c9158dfcf4efd2cf22df9aed7b1bb01e037e8eb2 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 19:04:27 -0700 Subject: moved scheduler API check into db.api decorator --- nova/api/zone_redirect.py | 5 ++++- nova/compute/api.py | 10 ++-------- nova/compute/manager.py | 5 +---- nova/db/api.py | 35 ++++++++++++++++++++++++++++++++++- nova/exception.py | 4 +++- nova/scheduler/api.py | 23 ++++++----------------- 6 files changed, 50 insertions(+), 32 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 0adf94046..4fe255c99 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -43,7 +43,7 @@ LOG = logging.getLogger('server') class RequestForwarder(api.ChildZoneHelper): - + """Worker for sending an OpenStack Request to each child zone.""" def __init__(self, resource, method, body): self.resource = resource self.method = method @@ -98,10 +98,13 @@ class ZoneRedirectMiddleware(wsgi.Middleware): scheme, netloc, path, query, frag = \ urlparse.urlsplit(req.path_qs) query = urlparse.parse_qsl(query) + # Remove any cache busters from old novaclient calls ... query = [(key, value) for key, value in query if key != 'fresh'] query = urllib.urlencode(query) url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + # Strip off the API version, since this is given when the + # child zone was added. m = re.search('/v\d+\.\d+/(.+)', url) resource = m.group(1) diff --git a/nova/compute/api.py b/nova/compute/api.py index 215257217..f4bfe720c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -34,7 +34,6 @@ from nova import rpc from nova import utils from nova import volume from nova.compute import instance_types -from nova.scheduler import api as scheduler from nova.db import base FLAGS = flags.FLAGS @@ -51,7 +50,7 @@ class API(base.Base): def __init__(self, image_service=None, network_api=None, volume_api=None, hostname_factory=generate_default_hostname, - scheduler_api=None, **kwargs): + **kwargs): if not image_service: image_service = utils.import_object(FLAGS.image_service) self.image_service = image_service @@ -61,9 +60,6 @@ class API(base.Base): if not volume_api: volume_api = volume.API() self.volume_api = volume_api - if not scheduler_api: - scheduler_api = scheduler.API() - self.scheduler_api = scheduler_api self.hostname_factory = hostname_factory super(API, self).__init__(**kwargs) @@ -347,8 +343,7 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - rv = self.scheduler_api.get_instance_or_reroute(context, instance_id) - #rv = self.db.instance_get(context, instance_id) + rv = self.db.instance_get(context, instance_id) return dict(rv.iteritems()) def get_all(self, context, project_id=None, reservation_id=None, @@ -513,7 +508,6 @@ class API(base.Base): def get_ajax_console(self, context, instance_id): """Get a url to an AJAX Console""" - instance = self.get(context, instance_id) output = self._call_compute_message('get_ajax_console', context, instance_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 499b212e2..ce60c6b43 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -48,7 +48,6 @@ from nova import scheduler_manager from nova import rpc from nova import utils from nova.compute import power_state -from nova.scheduler import api as scheduler_api FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', @@ -523,9 +522,7 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): """Pause an instance on this server.""" context = context.elevated() LOG.debug(_('*** instance %s: starting pause'), instance_id) - instance_ref = scheduler_api.get_instance_or_reroute(context, - instance_id) - #instance_ref = self.db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) LOG.audit(_('instance %s: pausing'), instance_id, context=context) self.db.instance_set_state(context, instance_id, diff --git a/nova/db/api.py b/nova/db/api.py index 2ecfc0211..6298e16ad 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -34,6 +34,7 @@ The underlying driver is loaded as a :class:`LazyPluggable`. from nova import exception from nova import flags +from nova import log as logging from nova import utils @@ -52,6 +53,9 @@ IMPL = utils.LazyPluggable(FLAGS['db_backend'], sqlalchemy='nova.db.sqlalchemy.api') +LOG = logging.getLogger('server') + + class NoMoreAddresses(exception.Error): """No more available addresses.""" pass @@ -71,6 +75,34 @@ class NoMoreTargets(exception.Error): """No more available blades""" pass + +################### + + +def reroute_if_not_found(key_args_index=None): + """Decorator used to indicate that the method should throw + a RouteRedirectException if the query can't find anything. + """ + def wrap(f): + def wrapped_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except exception.InstanceNotFound, e: + context = args[0] + key = None + if key_args_index: + key = args[key_args_index] + LOG.debug(_("Instance %(key)s not found locally: '%(e)s'" % + locals())) + + # Throw a reroute Exception for the middleware to pick up. + LOG.debug("Firing ZoneRouteException") + zones = zone_get_all(context) + raise exception.ZoneRouteException(zones, e) + return wrapped_f + return wrap + + ################### @@ -367,7 +399,8 @@ def instance_destroy(context, instance_id): return IMPL.instance_destroy(context, instance_id) -def instance_get(context, instance_id): +@reroute_if_not_found(key_args_index=1) +def instance_get(context, instance_id, reroute=True): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) diff --git a/nova/exception.py b/nova/exception.py index d0baa2e29..cfed32a72 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -93,8 +93,10 @@ class TimeoutException(Error): class ZoneRouteException(Error): - def __init__(self, zones, *args, **kwargs): + """Thrown in API to reroute request to child zones.""" + def __init__(self, zones, original_exception, *args, **kwargs): self.zones = zones + self.original_exception = original_exception super(ZoneRouteException, self).__init__(*args, **kwargs) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 073784f31..2da2dabfe 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -78,30 +78,16 @@ class API(object): capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) - @classmethod - def get_instance_or_reroute(cls, context, instance_id): - """Return an instance from the db or throw a ZoneRouteException - if not found.""" - try: - instance = db.instance_get(context, instance_id) - return instance - except exception.InstanceNotFound, e: - LOG.debug(_("Instance %(instance_id)s not found locally: '%(e)s'" % - locals())) - - # Throw a reroute Exception for the middleware to pick up. - LOG.debug("Firing ZoneRouteException") - zones = db.zone_get_all(context) - raise exception.ZoneRouteException(zones) - def _wrap_method(function, self): + """Wrap method to supply 'self'.""" def _wrap(*args, **kwargs): return function(self, *args, **kwargs) return _wrap def _process(self, zone): + """Worker stub for green thread pool""" nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() @@ -114,10 +100,13 @@ class ChildZoneHelper(object): plug-ins to query the children.""" def start(self, zone_list): + """Spawn a green thread for each child zone, calling the + derived classes process() method as the worker. Returns + a list of HTTP Responses. 1 per child.""" self.green_pool = greenpool.GreenPool() return [ result for result in self.green_pool.imap( _wrap_method(_process, self), zone_list)] def process(self, client, zone): - """Derived class must override.""" + """Worker Method. Derived class must override.""" pass -- cgit From cfe77c1236b68aa96dd85503582e08a07a23f77f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 19:21:32 -0700 Subject: cleanup --- nova/api/openstack/servers.py | 1 - nova/compute/manager.py | 1 - nova/db/api.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ffcbe628c..85999764f 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -343,7 +343,6 @@ class Controller(wsgi.Controller): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: - LOG.debug(_("*** Compute.api::pause %s"), id) self.compute_api.pause(ctxt, id) except: readable = traceback.format_exc() diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ce60c6b43..ebe1ce6f0 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -521,7 +521,6 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): def pause_instance(self, context, instance_id): """Pause an instance on this server.""" context = context.elevated() - LOG.debug(_('*** instance %s: starting pause'), instance_id) instance_ref = self.db.instance_get(context, instance_id) LOG.audit(_('instance %s: pausing'), instance_id, context=context) self.db.instance_set_state(context, diff --git a/nova/db/api.py b/nova/db/api.py index 6298e16ad..d56d6f404 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -400,7 +400,7 @@ def instance_destroy(context, instance_id): @reroute_if_not_found(key_args_index=1) -def instance_get(context, instance_id, reroute=True): +def instance_get(context, instance_id): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) -- cgit From 609a912fa8a816c1f47140489dcc1131356cd67c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 19:26:54 -0700 Subject: pep8 --- nova/api/zone_redirect.py | 8 ++++---- nova/scheduler/api.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 4fe255c99..7ebae1401 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -48,7 +48,7 @@ class RequestForwarder(api.ChildZoneHelper): self.resource = resource self.method = method self.body = body - + def process(self, client, zone): api_url = zone.api_url LOG.debug(_("Zone redirect to: %(api_url)s, " % locals())) @@ -89,12 +89,12 @@ class ZoneRedirectMiddleware(wsgi.Middleware): return req.get_response(self.application) except exception.ZoneRouteException as e: if not e.zones: - exc = webob.exc.HTTPInternalServerError(explanation= - _("No zones to reroute to.")) + exc = webob.exc.HTTPInternalServerError(explanation=_( + "No zones to reroute to.")) return faults.Fault(exc) # Todo(sandy): This only works for OpenStack API currently. - # Needs to be broken out into a driver. + # Needs to be broken out into a driver. scheme, netloc, path, query, frag = \ urlparse.urlsplit(req.path_qs) query = urlparse.parse_qsl(query) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 2da2dabfe..f0b645c09 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -77,7 +77,7 @@ class API(object): args=dict(service_name=service_name, host=host, capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) - + def _wrap_method(function, self): """Wrap method to supply 'self'.""" @@ -92,7 +92,7 @@ def _process(self, zone): zone.api_url) nova.authenticate() return self.process(nova, zone) - + class ChildZoneHelper(object): """Delegate a call to a set of Child Zones and wait for their @@ -104,9 +104,9 @@ class ChildZoneHelper(object): derived classes process() method as the worker. Returns a list of HTTP Responses. 1 per child.""" self.green_pool = greenpool.GreenPool() - return [ result for result in self.green_pool.imap( + return [result for result in self.green_pool.imap( _wrap_method(_process, self), zone_list)] - + def process(self, client, zone): """Worker Method. Derived class must override.""" pass -- cgit From 2f72127478405f5d87a40d799cc04e77e744f35b Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 23:31:06 -0300 Subject: removed dead method --- nova/scheduler/api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 804dade6b..71d211fe9 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -49,10 +49,6 @@ def _call_scheduler(method, context, params=None): class API(object): """API for interacting with the scheduler.""" - @classmethod - def _is_current_zone(cls, zone): - return True - @classmethod def get_zone_list(cls, context): """Return a list of zones assoicated with this zone.""" -- cgit From 82a65107ba3e3df8ec52984bb835b71fe4283b4c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 23:33:32 -0300 Subject: fix up copyright --- nova/api/zone_redirect.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 7ebae1401..c600b9ab5 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -1,6 +1,4 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may -- cgit From abc6c82449dfc46a33dcd8190840e51f44b5b930 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Mar 2011 07:30:22 -0700 Subject: Replaced capability flags with List --- nova/api/openstack/zones.py | 4 ++-- nova/flags.py | 5 +++-- nova/tests/api/openstack/test_zones.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 547920901..ebfc7743c 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -71,9 +71,9 @@ class Controller(wsgi.Controller): items = api.API.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) - caps = FLAGS.zone_capabilities.split(';') + caps = FLAGS.zone_capabilities for cap in caps: - key_values = cap.split(':') + key_values = cap.split('=') zone[key_values[0]] = key_values[1] for item, (min_value, max_value) in items.iteritems(): zone[item] = "%s,%s" % (min_value, max_value) diff --git a/nova/flags.py b/nova/flags.py index c05cef373..3a8ec1a39 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -358,5 +358,6 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'hypervisor:xenserver;os:linux', - 'Key/Value tags which represent capabilities of this zone') +DEFINE_list('zone_capabilities', + ['hypervisor=xenserver;kvm', 'os=linux;windows'], + 'Key/Multi-value list representng capabilities of this zone') diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 5e3aee4a7..12d39fd29 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -178,7 +178,7 @@ class ZonesTest(test.TestCase): def test_zone_info(self): FLAGS.zone_name = 'darksecret' - FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' + FLAGS.zone_capabilities = ['cap1=a;b', 'cap2=c;d'] self.stubs.Set(api, '_call_scheduler', zone_caps) body = dict(zone=dict(username='zeb', password='sneaky')) @@ -188,5 +188,5 @@ class ZonesTest(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) self.assertEqual(res_dict['zone']['name'], 'darksecret') - self.assertEqual(res_dict['zone']['cap1'], 'a,b') - self.assertEqual(res_dict['zone']['cap2'], 'c,d') + self.assertEqual(res_dict['zone']['cap1'], 'a;b') + self.assertEqual(res_dict['zone']['cap2'], 'c;d') -- cgit From 25199b6b93d17ff7dc192306e44932969846d9b6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Mar 2011 19:55:55 -0700 Subject: decorator more generic now --- nova/scheduler/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 0e2c69f75..190eb363e 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -99,6 +99,7 @@ def child_zone_helper(zone_list, func): def _issue_novaclient_command(nova, zone, method_name, instance_id): server = None try: + manager = getattr(nova, "servers") if isinstance(instance_id, int) or instance_id.isdigit(): server = manager.get(int(instance_id)) else: -- cgit From 0bc393bd1a0b722b08a2834873a8a825b86035c2 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 06:38:02 -0700 Subject: enable_zone_routing flag --- nova/compute/api.py | 20 ++++++++--------- nova/scheduler/api.py | 60 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 9fb4c8ae2..96538dd00 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -343,7 +343,7 @@ class API(base.Base): rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) - #@scheduler_api.reroute_if_not_found("delete") + @scheduler_api.reroute("delete") def delete(self, context, instance_id): LOG.debug(_("Going to try to terminate %s"), instance_id) try: @@ -373,12 +373,10 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - LOG.debug("*** COMPUTE.API::GET") rv = self.db.instance_get(context, instance_id) - LOG.debug("*** COMPUTE.API::GET OUT CLEAN") return dict(rv.iteritems()) - @scheduler_api.reroute_if_not_found("get") + @scheduler_api.reroute_compute("get") def routing_get(self, context, instance_id): """Use this method instead of get() if this is the only operation you intend to to. It will route to novaclient.get @@ -502,17 +500,17 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, }},) - #@scheduler_api.reroute_if_not_found("pause") + @scheduler_api.reroute_compute("pause") def pause(self, context, instance_id): """Pause the given instance.""" self._cast_compute_message('pause_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("unpause") + @scheduler_api.reroute_compute("unpause") def unpause(self, context, instance_id): """Unpause the given instance.""" self._cast_compute_message('unpause_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("diagnostics") + @scheduler_api.reroute_compute("diagnostics") def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for the given instance.""" return self._call_compute_message( @@ -524,22 +522,22 @@ class API(base.Base): """Retrieve actions for the given instance.""" return self.db.instance_get_actions(context, instance_id) - #@scheduler_api.reroute_if_not_found("suspend") + @scheduler_api.reroute_compute("suspend") def suspend(self, context, instance_id): """suspend the instance with instance_id""" self._cast_compute_message('suspend_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("resume") + @scheduler_api.reroute_compute("resume") def resume(self, context, instance_id): """resume the instance with instance_id""" self._cast_compute_message('resume_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("rescue") + @scheduler_api.reroute_compute("rescue") def rescue(self, context, instance_id): """Rescue the given instance.""" self._cast_compute_message('rescue_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("unrescue") + @scheduler_api.reroute_compute("unrescue") def unrescue(self, context, instance_id): """Unrescue the given instance.""" self._cast_compute_message('unrescue_instance', context, instance_id) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 190eb363e..90b92d7ed 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -28,6 +28,10 @@ import novaclient.client as client from eventlet import greenpool FLAGS = flags.FLAGS +flags.DEFINE_bool('enable_zone_routing', + False, + 'When True, routing to child zones will occur.') + LOG = logging.getLogger('nova.scheduler.api') @@ -83,7 +87,8 @@ def _wrap_method(function, self): def _process(func, zone): - """Worker stub for green thread pool""" + """Worker stub for green thread pool. Give the worker + an authenticated nova client and zone info.""" nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() @@ -91,36 +96,42 @@ def _process(func, zone): def child_zone_helper(zone_list, func): + """Fire off a command to each zone in the list.""" green_pool = greenpool.GreenPool() return [result for result in green_pool.imap( _wrap_method(_process, func), zone_list)] -def _issue_novaclient_command(nova, zone, method_name, instance_id): - server = None +def _issue_novaclient_command(nova, zone, collection, method_name, \ + item_id): + """Use novaclient to issue command to a single child zone. + One of these will be run in parallel for each child zone.""" + item = None try: - manager = getattr(nova, "servers") - if isinstance(instance_id, int) or instance_id.isdigit(): - server = manager.get(int(instance_id)) + manager = getattr(nova, collection) + if isinstance(item_id, int) or item_id.isdigit(): + item = manager.get(int(item_id)) else: - server = manager.find(name=instance_id) + item = manager.find(name=item_id) except novaclient.NotFound: url = zone.api_url - LOG.debug(_("Instance %(instance_id)s not found on '%(url)s'" % + LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % locals())) return - return getattr(server, method_name)() + return getattr(item, method_name)() -def wrap_novaclient_function(f, method_name, instance_id): +def wrap_novaclient_function(f, collection, method_name, item_id): + """Appends collection, method_name and item_id to the incoming + (nova, zone) call from child_zone_helper.""" def inner(nova, zone): - return f(nova, zone, method_name, instance_id) + return f(nova, zone, collection, method_name, item_id) return inner -class reroute_if_not_found(object): +class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query can't find anything. @@ -130,19 +141,32 @@ class reroute_if_not_found(object): def __call__(self, f): def wrapped_f(*args, **kwargs): - LOG.debug("***REROUTE-3: %s / %s" % (args, kwargs)) - context = args[1] - instance_id = args[2] + collection, context, item_id = \ + self.get_collection_context_and_id() try: return f(*args, **kwargs) except exception.InstanceNotFound, e: - LOG.debug(_("Instance %(instance_id)s not found " + LOG.debug(_("Instance %(item_id)s not found " "locally: '%(e)s'" % locals())) + if not FLAGS.enable_zone_routing: + raise + zones = db.zone_get_all(context) + if not zones: + raise + result = child_zone_helper(zones, wrap_novaclient_function(_issue_novaclient_command, - self.method_name, instance_id)) + collection, self.method_name, item_id)) LOG.debug("***REROUTE: %s" % result) - return result + return self.unmarshall_result(result) return wrapped_f + + def get_collection_context_and_id(self, args): + """Returns a tuple of (novaclient collection name, security + context and resource id. Derived class should override this.""" + return ("servers", args[1], args[2]) + + def unmarshall_result(self, result): + return result -- cgit From 70e8b431334989ad067f0a5543aea408b7186c5c Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 18 Mar 2011 10:34:08 -0400 Subject: Fixed 'Undefined variable' errors generated by pylint (E0602). --- nova/api/openstack/accounts.py | 7 ++++--- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 5 +++-- nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py | 2 -- nova/virt/fake.py | 2 +- nova/virt/libvirt_conn.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 2510ffb61..86066fa20 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -14,6 +14,7 @@ # under the License. import common +import webob.exc from nova import exception from nova import flags @@ -51,10 +52,10 @@ class Controller(wsgi.Controller): raise exception.NotAuthorized(_("Not admin user.")) def index(self, req): - raise faults.Fault(exc.HTTPNotImplemented()) + raise faults.Fault(webob.exc.HTTPNotImplemented()) def detail(self, req): - raise faults.Fault(exc.HTTPNotImplemented()) + raise faults.Fault(webob.exc.HTTPNotImplemented()) def show(self, req, id): """Return data about the given account id""" @@ -69,7 +70,7 @@ class Controller(wsgi.Controller): def create(self, req): """We use update with create-or-update semantics because the id comes from an external source""" - raise faults.Fault(exc.HTTPNotImplemented()) + raise faults.Fault(webob.exc.HTTPNotImplemented()) def update(self, req, id): """This is really create or update.""" diff --git a/nova/compute/api.py b/nova/compute/api.py index 32577af82..058b600bf 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -479,7 +479,7 @@ class API(base.Base): self._cast_compute_message('confirm_resize', context, instance_id, migration_ref['source_compute'], params=params) - self.db.migration_update(context, migration_id, + self.db.migration_update(context, migration_ref['id'], {'status': 'confirmed'}) self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 9d9b86c1d..98e6f938a 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2204,7 +2204,7 @@ def migration_get(context, id, session=None): filter_by(id=id).first() if not result: raise exception.NotFound(_("No migration found with id %s") - % migration_id) + % id) return result @@ -2216,7 +2216,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") - % migration_id) + % id) return result @@ -2427,6 +2427,7 @@ def zone_create(context, values): @require_admin_context def zone_update(context, zone_id, values): + session = get_session() zone = session.query(models.Zone).filter_by(id=zone_id).first() if not zone: raise exception.NotFound(_("No zone with id %(zone_id)s") % locals()) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py b/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py index 66609054e..8b962bf7f 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py @@ -55,7 +55,6 @@ def upgrade(migrate_engine): try: instance_types.create() except Exception: - logging.info(repr(table)) logging.exception('Exception while creating instance_types table') raise @@ -76,7 +75,6 @@ def upgrade(migrate_engine): 'local_gb': values["local_gb"], 'flavorid': values["flavorid"]}) except Exception: - logging.info(repr(table)) logging.exception('Exception while seeding instance_types table') raise diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 3a06284a1..451760721 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -323,7 +323,7 @@ class FakeConnection(object): Note that this function takes an instance ID, not a compute.service.Instance, so that it can be called by compute.monitor. """ - return [0L, 0L, 0L, 0L, null] + return [0L, 0L, 0L, 0L, None] def interface_stats(self, instance_name, iface_id): """ diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e80b9fbdf..8a59c5bba 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -46,7 +46,7 @@ import time import uuid from xml.dom import minidom - +from eventlet import greenthread from eventlet import tpool from eventlet import semaphore -- cgit From 047bff904817838279199a7099023b505e35343f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 07:43:42 -0700 Subject: whoopsy --- nova/compute/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 96538dd00..270600664 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -343,7 +343,7 @@ class API(base.Base): rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) - @scheduler_api.reroute("delete") + @scheduler_api.reroute_compute("delete") def delete(self, context, instance_id): LOG.debug(_("Going to try to terminate %s"), instance_id) try: -- cgit From 930d7bf1987c1b270ec0e456f982efb70527ed15 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 07:47:23 -0700 Subject: whoopsy2 --- nova/scheduler/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 90b92d7ed..b639ae786 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -142,7 +142,7 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): collection, context, item_id = \ - self.get_collection_context_and_id() + self.get_collection_context_and_id(args) try: return f(*args, **kwargs) except exception.InstanceNotFound, e: -- cgit From 12ffa884c07b55c982a1ad60a94e72c955db81c3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 09:02:36 -0700 Subject: fixed up novaclient usage to include managers --- nova/scheduler/api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index b639ae786..7efc28072 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -17,14 +17,14 @@ Handles all requests relating to schedulers. """ +import novaclient + from nova import db from nova import exception from nova import flags from nova import log as logging from nova import rpc -import novaclient.client as client - from eventlet import greenpool FLAGS = flags.FLAGS @@ -80,7 +80,7 @@ class API(object): def _wrap_method(function, self): - """Wrap method to supply 'self'.""" + """Wrap method to supply self.""" def _wrap(*args, **kwargs): return function(self, *args, **kwargs) return _wrap @@ -89,8 +89,7 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" - nova = client.OpenStackClient(zone.username, zone.password, - zone.api_url) + nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) nova.authenticate() return func(nova, zone) @@ -134,8 +133,7 @@ def wrap_novaclient_function(f, collection, method_name, item_id): class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query - can't find anything. - """ + can't find anything.""" def __init__(self, method_name): self.method_name = method_name -- cgit From ef33d6bde27276fb4c93ed6bbcb972977f03a370 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 09:21:08 -0700 Subject: results --- nova/api/openstack/servers.py | 2 +- nova/scheduler/api.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 9f14a6b82..49f714d47 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -130,7 +130,7 @@ class Controller(wsgi.Controller): try: LOG.debug(_("***SHOW")) instance = self.compute_api.routing_get(req.environ['nova.context'], id) - LOG.debug(_("***SHOW")) + LOG.debug(_("***SHOW OUT %s" % instance)) return _translate_detail_keys(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 7efc28072..6b0f804f9 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -118,7 +118,10 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ locals())) return - return getattr(item, method_name)() + LOG.debug("***CALLING CHILD ZONE") + result = getattr(item, method_name)() + LOG.debug("***CHILD ZONE GAVE %s", result) + return result def wrap_novaclient_function(f, collection, method_name, item_id): -- cgit From 705020cc4acded862633aa5e02d5bb46c88dbc51 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 11:46:27 -0700 Subject: api decorator --- nova/api/openstack/servers.py | 2 ++ nova/scheduler/api.py | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 49f714d47..17d620562 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -33,6 +33,7 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack +from nova.scheduler import api as scheduler_api LOG = logging.getLogger('server') @@ -125,6 +126,7 @@ class Controller(wsgi.Controller): res = [entity_maker(inst)['server'] for inst in limited_list] return dict(servers=res) + @scheduler_api.redirect_handler def show(self, req, id): """ Returns server details by server id """ try: diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6b0f804f9..f5df446b3 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -105,22 +105,25 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ item_id): """Use novaclient to issue command to a single child zone. One of these will be run in parallel for each child zone.""" - item = None + result = None try: manager = getattr(nova, collection) if isinstance(item_id, int) or item_id.isdigit(): - item = manager.get(int(item_id)) + result = manager.get(int(item_id)) else: - item = manager.find(name=item_id) + result = manager.find(name=item_id) except novaclient.NotFound: url = zone.api_url LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % locals())) return - LOG.debug("***CALLING CHILD ZONE") - result = getattr(item, method_name)() - LOG.debug("***CHILD ZONE GAVE %s", result) + if method_name.lower() not in ['get', 'find']: + LOG.debug("***CALLING CHILD ZONE") + m = getattr(item, method_name) + LOG.debug("***METHOD ATTR %s" % m) + result = getattr(item, method_name)() + LOG.debug("***CHILD ZONE GAVE %s", result) return result @@ -133,6 +136,14 @@ def wrap_novaclient_function(f, collection, method_name, item_id): return inner +class RedirectResult(exception.Error): + """Used to the HTTP API know that these results are pre-cooked + and they can be returned to the caller directly.""" + def __init__(self, results): + self.results = results + super(RedirectResult, self).__init__( + message=_("Uncaught Zone redirection exception")) + class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query @@ -161,7 +172,7 @@ class reroute_compute(object): wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) LOG.debug("***REROUTE: %s" % result) - return self.unmarshall_result(result) + raise RedirectResult(self.unmarshall_result(result)) return wrapped_f def get_collection_context_and_id(self, args): @@ -170,4 +181,14 @@ class reroute_compute(object): return ("servers", args[1], args[2]) def unmarshall_result(self, result): - return result + return [server.__dict__ for server in result] + + +def redirect_handler(f): + def new_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except RedirectResult, e: + LOG.debug("***CAUGHT REROUTE: %s" % e.results) + return e.results + return new_f -- cgit From 37f2c3036890f9bbfd88a369dfd590744256aaf9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 12:00:35 -0700 Subject: works again. woo hoo --- nova/scheduler/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index f5df446b3..8b8457e8d 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -181,7 +181,13 @@ class reroute_compute(object): return ("servers", args[1], args[2]) def unmarshall_result(self, result): - return [server.__dict__ for server in result] + server = result[0].__dict__ + + for k in server.keys(): + if k[0] == '_' or k == 'manager': + del server[k] + + return dict(server=server) def redirect_handler(f): -- cgit From feb5c82e29303285d3f914c37116a59538fec28f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 12:23:57 -0700 Subject: fix ups --- nova/api/openstack/servers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 17d620562..86414fab2 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -137,6 +137,7 @@ class Controller(wsgi.Controller): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) + @scheduler_api.redirect_handler def delete(self, req, id): """ Destroys a server """ try: @@ -258,6 +259,7 @@ class Controller(wsgi.Controller): # if the original error is okay, just reraise it raise error + @scheduler_api.redirect_handler def update(self, req, id): """ Updates the server name or password """ if len(req.body) == 0: @@ -283,6 +285,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() + @scheduler_api.redirect_handler def action(self, req, id): """Multi-purpose method used to reboot, rebuild, or resize a server""" @@ -348,6 +351,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def lock(self, req, id): """ lock the instance with id @@ -363,6 +367,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def unlock(self, req, id): """ unlock the instance with id @@ -378,6 +383,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def get_lock(self, req, id): """ return the boolean state of (instance with id)'s lock @@ -392,6 +398,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def reset_network(self, req, id): """ Reset networking on an instance (admin only). @@ -406,6 +413,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def inject_network_info(self, req, id): """ Inject network info for an instance (admin only). @@ -420,6 +428,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def pause(self, req, id): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] @@ -431,6 +440,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def unpause(self, req, id): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] @@ -442,6 +452,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def suspend(self, req, id): """permit admins to suspend the server""" context = req.environ['nova.context'] @@ -453,6 +464,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def resume(self, req, id): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] @@ -464,6 +476,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def rescue(self, req, id): """Permit users to rescue the server.""" context = req.environ["nova.context"] @@ -475,6 +488,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def unrescue(self, req, id): """Permit users to unrescue the server.""" context = req.environ["nova.context"] @@ -486,6 +500,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def get_ajax_console(self, req, id): """ Returns a url to an instance's ajaxterm console. """ try: @@ -495,6 +510,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def diagnostics(self, req, id): """Permit Admins to retrieve server diagnostics.""" ctxt = req.environ["nova.context"] -- cgit -- cgit From 8f0b60f598c28b2f558f3ecdaa2f9604926393e6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 07:49:58 -0700 Subject: remove scheduler.api.API. naming changes. --- nova/api/openstack/zones.py | 4 ++-- nova/manager.py | 2 +- nova/rpc.py | 2 +- nova/scheduler/api.py | 43 +++++++++++++++------------------- nova/tests/api/openstack/test_zones.py | 8 +++---- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index ebfc7743c..d129cf34f 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -53,7 +53,7 @@ class Controller(wsgi.Controller): """Return all zones in brief""" # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... - items = api.API.get_zone_list(req.environ['nova.context']) + items = api.get_zone_list(req.environ['nova.context']) if not items: items = db.zone_get_all(req.environ['nova.context']) @@ -68,7 +68,7 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - items = api.API.get_zone_capabilities(req.environ['nova.context']) + items = api.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) caps = FLAGS.zone_capabilities diff --git a/nova/manager.py b/nova/manager.py index f384e3f0f..508f133ca 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -96,7 +96,7 @@ class SchedulerDependentManager(Manager): """Pass data back to the scheduler at a periodic interval""" if self.last_capabilities: logging.debug(_("Notifying Schedulers of capabilities ...")) - api.API.update_service_capabilities(context, self.service_name, + api.update_service_capabilities(context, self.service_name, self.host, self.last_capabilities) super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/rpc.py b/nova/rpc.py index 4918c0b95..2e3cd9057 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -244,7 +244,7 @@ class FanoutPublisher(Publisher): self.exchange = "%s_fanout" % topic self.queue = "%s_fanout" % topic self.durable = False - LOG.info(_("Writing to '%(exchange)s' fanout exchange"), + LOG.info(_("Creating '%(exchange)s' fanout exchange"), dict(exchange=self.exchange)) super(FanoutPublisher, self).__init__(connection=connection) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index b6d27dacc..e2cf3b6a3 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -40,30 +40,25 @@ def _call_scheduler(method, context, params=None): return rpc.call(context, queue, kwargs) -class API(object): - """API for interacting with the scheduler.""" +def get_zone_list(context): + """Return a list of zones assoicated with this zone.""" + items = _call_scheduler('get_zone_list', context) + for item in items: + item['api_url'] = item['api_url'].replace('\\/', '/') + return items - @classmethod - def get_zone_list(cls, context): - """Return a list of zones assoicated with this zone.""" - items = _call_scheduler('get_zone_list', context) - for item in items: - item['api_url'] = item['api_url'].replace('\\/', '/') - return items - @classmethod - def get_zone_capabilities(cls, context, service=None): - """Returns a dict of key, value capabilities for this zone, - or for a particular class of services running in this zone.""" - return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) +def get_zone_capabilities(context, service=None): + """Returns a dict of key, value capabilities for this zone, + or for a particular class of services running in this zone.""" + return _call_scheduler('get_zone_capabilities', context=context, + params=dict(service=service)) - @classmethod - def update_service_capabilities(cls, context, service_name, host, - capabilities): - """Send an update to all the scheduler services informing them - of the capabilities of this service.""" - kwargs = dict(method='update_service_capabilities', - args=dict(service_name=service_name, host=host, - capabilities=capabilities)) - return rpc.fanout_cast(context, 'scheduler', kwargs) + +def update_service_capabilities(context, service_name, host, capabilities): + """Send an update to all the scheduler services informing them + of the capabilities of this service.""" + kwargs = dict(method='update_service_capabilities', + args=dict(service_name=service_name, host=host, + capabilities=capabilities)) + return rpc.fanout_cast(context, 'scheduler', kwargs) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 12d39fd29..a3f191aaa 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -75,7 +75,7 @@ def zone_get_all_db(context): ] -def zone_caps(method, context, params): +def zone_capabilities(method, context, params): return dict() @@ -98,13 +98,13 @@ class ZonesTest(test.TestCase): self.stubs.Set(nova.db, 'zone_delete', zone_delete) self.old_zone_name = FLAGS.zone_name - self.old_zone_caps = FLAGS.zone_capabilities + self.old_zone_capabilities = FLAGS.zone_capabilities def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin FLAGS.zone_name = self.old_zone_name - FLAGS.zone_capabilities = self.old_zone_caps + FLAGS.zone_capabilities = self.old_zone_capabilities super(ZonesTest, self).tearDown() def test_get_zone_list_scheduler(self): @@ -179,7 +179,7 @@ class ZonesTest(test.TestCase): def test_zone_info(self): FLAGS.zone_name = 'darksecret' FLAGS.zone_capabilities = ['cap1=a;b', 'cap2=c;d'] - self.stubs.Set(api, '_call_scheduler', zone_caps) + self.stubs.Set(api, '_call_scheduler', zone_capabilities) body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/info') -- cgit From 7976fb08d89a8e8b6bf8c276a50e30ae11584ce3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 11:01:34 -0700 Subject: more robust extraction of arguments --- nova/scheduler/api.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index c7acd3548..935e7b366 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -149,7 +149,7 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): collection, context, item_id = \ - self.get_collection_context_and_id(args) + self.get_collection_context_and_id(args, kwargs) try: return f(*args, **kwargs) except exception.InstanceNotFound, e: @@ -170,10 +170,16 @@ class reroute_compute(object): raise RedirectResult(self.unmarshall_result(result)) return wrapped_f - def get_collection_context_and_id(self, args): + def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" - return ("servers", args[1], args[2]) + context = kwargs.get('context', None) + instance_id = kwargs.get('instance_id', None) + if len(args) > 0 and not context: + context = args[1] + if len(args) > 1 and not instance_id: + context = args[2] + return ("servers", context, instance_id) def unmarshall_result(self, result): server = result[0].__dict__ -- cgit From b1def6b2b104a143b7491cef9a01babe9ab3e75d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 11:07:19 -0700 Subject: pep8 --- nova/scheduler/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 935e7b366..aebfe1770 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -127,7 +127,7 @@ def wrap_novaclient_function(f, collection, method_name, item_id): (nova, zone) call from child_zone_helper.""" def inner(nova, zone): return f(nova, zone, collection, method_name, item_id) - + return inner @@ -139,6 +139,7 @@ class RedirectResult(exception.Error): super(RedirectResult, self).__init__( message=_("Uncaught Zone redirection exception")) + class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query @@ -158,7 +159,7 @@ class reroute_compute(object): if not FLAGS.enable_zone_routing: raise - + zones = db.zone_get_all(context) if not zones: raise -- cgit From 08d06d1219a00b90ae211fb44fc7e33ba71c7a76 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 18:16:35 -0700 Subject: better comments. First redirect test --- nova/scheduler/api.py | 49 +++++++++++++++++++++++-------- nova/tests/test_scheduler.py | 70 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 12 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index aebfe1770..ff7e21679 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -84,13 +84,18 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" + LOG.debug("*** PROCESS %s/%s" % (func, zone)) nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) nova.authenticate() return func(nova, zone) def child_zone_helper(zone_list, func): - """Fire off a command to each zone in the list.""" + """Fire off a command to each zone in the list. + The return is [novaclient return objects] from each child zone. + For example, if you are calling server.pause(), the list will + be whatever the response from server.pause() is. One entry + per child zone called.""" green_pool = greenpool.GreenPool() return [result for result in green_pool.imap( _wrap_method(_process, func), zone_list)] @@ -103,6 +108,7 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ result = None try: manager = getattr(nova, collection) + LOG.debug("***MANAGER %s" % manager) if isinstance(item_id, int) or item_id.isdigit(): result = manager.get(int(item_id)) else: @@ -115,9 +121,9 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ if method_name.lower() not in ['get', 'find']: LOG.debug("***CALLING CHILD ZONE") - m = getattr(item, method_name) + m = getattr(result, method_name) LOG.debug("***METHOD ATTR %s" % m) - result = getattr(item, method_name)() + result = getattr(result, method_name)() LOG.debug("***CHILD ZONE GAVE %s", result) return result @@ -152,6 +158,7 @@ class reroute_compute(object): collection, context, item_id = \ self.get_collection_context_and_id(args, kwargs) try: + # Call the original function ... return f(*args, **kwargs) except exception.InstanceNotFound, e: LOG.debug(_("Instance %(item_id)s not found " @@ -164,32 +171,50 @@ class reroute_compute(object): if not zones: raise + # Ask the children to provide an answer ... result = child_zone_helper(zones, wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) LOG.debug("***REROUTE: %s" % result) + # Scrub the results and raise another exception + # so the API layers can bail out gracefully ... raise RedirectResult(self.unmarshall_result(result)) return wrapped_f def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" + LOG.debug("***COLLECT: %s/%s" % (args, kwargs)) context = kwargs.get('context', None) instance_id = kwargs.get('instance_id', None) if len(args) > 0 and not context: context = args[1] if len(args) > 1 and not instance_id: - context = args[2] + instance_id = args[2] return ("servers", context, instance_id) - def unmarshall_result(self, result): - server = result[0].__dict__ - - for k in server.keys(): - if k[0] == '_' or k == 'manager': - del server[k] - - return dict(server=server) + def unmarshall_result(self, zone_responses): + """Result is a list of responses from each child zone. + Each decorator derivation is responsible to turning this + into a format expected by the calling method. For + example, this one is expected to return a single Server + dict {'server':{k:v}}. Others may return a list of them, like + {'servers':[{k,v}]}""" + reduced_response = [] + for zone_response in zone_responses: + if not zone_response: + continue + + server = zone_response.__dict__ + + for k in server.keys(): + if k[0] == '_' or k == 'manager': + del server[k] + + reduced_response.append(dict(server=server)) + if reduced_response: + return reduced_response[0] # first for now. + return {} def redirect_handler(f): diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 244e43bd9..50e2429ba 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -21,6 +21,8 @@ Tests For Scheduler import datetime import mox +import stubout +import webob from mox import IgnoreArg from nova import context @@ -32,6 +34,7 @@ from nova import test from nova import rpc from nova import utils from nova.auth import manager as auth_manager +from nova.scheduler import api from nova.scheduler import manager from nova.scheduler import driver from nova.compute import power_state @@ -937,3 +940,70 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) + + +class FakeZone(object): + def __init__(self, api_url, username, password): + self.api_url = api_url + self.username = username + self.password = password + +def zone_get_all(context): + return [ + FakeZone('http://example.com', 'bob', 'xxx'), + ] + + +def go_boom(self, context, instance): + raise exception.InstanceNotFound("boom message", instance) + + +def fake_openstack_init(self, username, password, api): + servers=[] + + +def fake_auth(self): + pass + +class FakeServer: + def foo(self): + pass + +class FakeManager: + def get(self, id): + return FakeServer() + +class FakeOpenStack: + + def __init__(self, username, api, auth): + self.servers = FakeManager() + + def authenticate(self): + pass + + +class ZoneRedirectTest(test.TestCase): + def setUp(self): + super(ZoneRedirectTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + + self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) + self.stubs.Set(db, 'zone_get_all', zone_get_all) + + self.enable_zone_routing = FLAGS.enable_zone_routing + FLAGS.enable_zone_routing = True + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.enable_zone_routing = self.enable_zone_routing + super(ZoneRedirectTest, self).tearDown() + + def test_trap_found_locally(self): + decorator = api.reroute_compute("foo") + try: + wrapper = decorator(go_boom) + result = wrapper(None, None, 1) # self, context, id + except api.RedirectResult, e: + pass + + -- cgit From 380731ce71e8909615da6138bb7d5e7226e375ac Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 18:56:59 -0700 Subject: better comments. First redirect test --- nova/tests/test_scheduler.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 50e2429ba..6d55cad04 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -958,23 +958,23 @@ def go_boom(self, context, instance): raise exception.InstanceNotFound("boom message", instance) -def fake_openstack_init(self, username, password, api): - servers=[] +class FakeServer(object): + def __init__(self): + self.name = 'myserver' + self.kvm = 'kvm' + self.manager = 100 + self._hidden = True - -def fake_auth(self): - pass - -class FakeServer: def foo(self): - pass + return None + -class FakeManager: +class FakeManager(object): def get(self, id): return FakeServer() -class FakeOpenStack: +class FakeOpenStack: def __init__(self, username, api, auth): self.servers = FakeManager() @@ -987,7 +987,6 @@ class ZoneRedirectTest(test.TestCase): super(ZoneRedirectTest, self).setUp() self.stubs = stubout.StubOutForTesting() - self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) self.stubs.Set(db, 'zone_get_all', zone_get_all) self.enable_zone_routing = FLAGS.enable_zone_routing @@ -999,11 +998,12 @@ class ZoneRedirectTest(test.TestCase): super(ZoneRedirectTest, self).tearDown() def test_trap_found_locally(self): + self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) decorator = api.reroute_compute("foo") try: wrapper = decorator(go_boom) result = wrapper(None, None, 1) # self, context, id except api.RedirectResult, e: - pass + self.assertTrue(e.results, {}) -- cgit From 8303d0f280a7bfbc5c5fb128465549b03badc1f1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 21:41:41 -0700 Subject: routing test coverage --- nova/scheduler/api.py | 18 +++---- nova/tests/test_scheduler.py | 121 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index ff7e21679..4f189fe37 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -84,7 +84,6 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" - LOG.debug("*** PROCESS %s/%s" % (func, zone)) nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) nova.authenticate() return func(nova, zone) @@ -108,7 +107,6 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ result = None try: manager = getattr(nova, collection) - LOG.debug("***MANAGER %s" % manager) if isinstance(item_id, int) or item_id.isdigit(): result = manager.get(int(item_id)) else: @@ -117,14 +115,10 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ url = zone.api_url LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % locals())) - return + return None if method_name.lower() not in ['get', 'find']: - LOG.debug("***CALLING CHILD ZONE") - m = getattr(result, method_name) - LOG.debug("***METHOD ATTR %s" % m) result = getattr(result, method_name)() - LOG.debug("***CHILD ZONE GAVE %s", result) return result @@ -172,19 +166,22 @@ class reroute_compute(object): raise # Ask the children to provide an answer ... - result = child_zone_helper(zones, + result = self._call_child_zones(zones, wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) - LOG.debug("***REROUTE: %s" % result) # Scrub the results and raise another exception # so the API layers can bail out gracefully ... raise RedirectResult(self.unmarshall_result(result)) return wrapped_f + def _call_child_zones(self, zones, function): + """Ask the child zones to perform this operation. + Broken out for testing.""" + return child_zone_helper(zones, function) + def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" - LOG.debug("***COLLECT: %s/%s" % (args, kwargs)) context = kwargs.get('context', None) instance_id = kwargs.get('instance_id', None) if len(args) > 0 and not context: @@ -222,6 +219,5 @@ def redirect_handler(f): try: return f(*args, **kwargs) except RedirectResult, e: - LOG.debug("***CAUGHT REROUTE: %s" % e.results) return e.results return new_f diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 6d55cad04..0aebd0380 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -21,6 +21,7 @@ Tests For Scheduler import datetime import mox +import novaclient.exceptions import stubout import webob @@ -954,34 +955,33 @@ def zone_get_all(context): ] -def go_boom(self, context, instance): - raise exception.InstanceNotFound("boom message", instance) +class FakeRerouteCompute(api.reroute_compute): + def _call_child_zones(self, zones, function): + return [ ] + + def get_collection_context_and_id(self, args, kwargs): + return ("servers", None, 1) + def unmarshall_result(self, zone_responses): + return dict(magic="found me") -class FakeServer(object): - def __init__(self): - self.name = 'myserver' - self.kvm = 'kvm' - self.manager = 100 - self._hidden = True - def foo(self): - return None +def go_boom(self, context, instance): + raise exception.InstanceNotFound("boom message", instance) -class FakeManager(object): - def get(self, id): - return FakeServer() +def found_instance(self, context, instance): + return dict(name='myserver') -class FakeOpenStack: - def __init__(self, username, api, auth): - self.servers = FakeManager() +class FakeResource(object): + def __init__(self, attribute_dict): + for k, v in attribute_dict.iteritems(): + setattr(self, k, v) - def authenticate(self): + def pause(self): pass - class ZoneRedirectTest(test.TestCase): def setUp(self): super(ZoneRedirectTest, self).setUp() @@ -998,12 +998,87 @@ class ZoneRedirectTest(test.TestCase): super(ZoneRedirectTest, self).tearDown() def test_trap_found_locally(self): - self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) - decorator = api.reroute_compute("foo") + decorator = FakeRerouteCompute("foo") try: - wrapper = decorator(go_boom) - result = wrapper(None, None, 1) # self, context, id + result = decorator(found_instance)(None, None, 1) except api.RedirectResult, e: - self.assertTrue(e.results, {}) + self.fail(_("Successful database hit should succeed")) + def test_trap_not_found_locally(self): + decorator = FakeRerouteCompute("foo") + try: + result = decorator(go_boom)(None, None, 1) + except api.RedirectResult, e: + self.assertEquals(e.results['magic'], 'found me') + def test_get_collection_context_and_id(self): + decorator = api.reroute_compute("foo") + self.assertEquals(decorator.get_collection_context_and_id( + (None, 10, 20), {}), ("servers", 10, 20)) + self.assertEquals(decorator.get_collection_context_and_id( + (None, 11,), dict(instance_id=21)), ("servers", 11, 21)) + self.assertEquals(decorator.get_collection_context_and_id( + (None,), dict(context=12, instance_id=22)), ("servers", 12, 22)) + + def test_unmarshal_single_server(self): + decorator = api.reroute_compute("foo") + self.assertEquals(decorator.unmarshall_result([]), {}) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(a=1, b=2)),]), + dict(server=dict(a=1, b=2))) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(a=1, _b=2)),]), + dict(server=dict(a=1,))) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(a=1, manager=2)),]), + dict(server=dict(a=1,))) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(_a=1, manager=2)),]), + dict(server={})) + +class FakeServerCollection(object): + def get(self, instance_id): + return FakeResource(dict(a=10, b=20)) + + def find(self, name): + return FakeResource(dict(a=11, b=22)) + +class FakeEmptyServerCollection(object): + def get(self, f): + raise novaclient.NotFound(1) + + def find(self, name): + raise novaclient.NotFound(2) + +class FakeNovaClient(object): + def __init__(self, collection): + self.servers = collection + +class DynamicNovaClientTest(test.TestCase): + def test_issue_novaclient_command_found(self): + zone = FakeZone('http://example.com', 'bob', 'xxx') + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeServerCollection()), + zone, "servers", "get", 100).a, 10) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeServerCollection()), + zone, "servers", "find", "name").b, 22) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeServerCollection()), + zone, "servers", "pause", 100), None) + + def test_issue_novaclient_command_not_found(self): + zone = FakeZone('http://example.com', 'bob', 'xxx') + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeEmptyServerCollection()), + zone, "servers", "get", 100), None) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeEmptyServerCollection()), + zone, "servers", "find", "name"), None) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeEmptyServerCollection()), + zone, "servers", "any", "name"), None) -- cgit From e74482f30c602530313faf15e0d429acefee7bde Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 21:47:58 -0700 Subject: routing test coverage --- nova/tests/test_scheduler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 0aebd0380..8434f5a43 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -1008,9 +1008,18 @@ class ZoneRedirectTest(test.TestCase): decorator = FakeRerouteCompute("foo") try: result = decorator(go_boom)(None, None, 1) + self.assertFail(_("Should have rerouted.")) except api.RedirectResult, e: self.assertEquals(e.results['magic'], 'found me') + def test_routing_flags(self): + FLAGS.enable_zone_routing = False + decorator = FakeRerouteCompute("foo") + try: + result = decorator(go_boom)(None, None, 1) + except exception.InstanceNotFound, e: + self.assertEquals(e.message, 'boom message') + def test_get_collection_context_and_id(self): decorator = api.reroute_compute("foo") self.assertEquals(decorator.get_collection_context_and_id( -- cgit From 65482f5d9513c3dda64171d0460001e299be9673 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 21:51:14 -0700 Subject: added zone routing flag test --- nova/tests/test_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 8434f5a43..277ffe367 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -1017,6 +1017,7 @@ class ZoneRedirectTest(test.TestCase): decorator = FakeRerouteCompute("foo") try: result = decorator(go_boom)(None, None, 1) + self.assertFail(_("Should have thrown exception.")) except exception.InstanceNotFound, e: self.assertEquals(e.message, 'boom message') -- cgit From 97e8f300af824145c8b92949ccbdfe81c0d7ca95 Mon Sep 17 00:00:00 2001 From: Josh Kleinpeter Date: Tue, 22 Mar 2011 12:33:34 -0500 Subject: Changed default for disabled on service_get_all to None. Changed calls to service_get_all so that the results should still be as they previously were. --- bin/nova-manage | 2 +- nova/api/ec2/admin.py | 2 +- nova/api/ec2/cloud.py | 4 ++-- nova/db/api.py | 2 +- nova/db/sqlalchemy/api.py | 13 ++++++++----- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 69cbf6f95..c5a4bea7e 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -610,7 +610,7 @@ class ServiceCommands(object): args: [host] [service]""" ctxt = context.get_admin_context() now = datetime.datetime.utcnow() - services = db.service_get_all(ctxt) + db.service_get_all(ctxt, True) + services = db.service_get_all(ctxt) if host: services = [s for s in services if s['host'] == host] if service: diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index d9a4ef999..3ae29d8ce 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -299,7 +299,7 @@ class AdminController(object): * Volume (up, down, None) * Volume Count """ - services = db.service_get_all(context) + services = db.service_get_all(context, False) now = datetime.datetime.utcnow() hosts = [] rv = [] diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index e257e44e7..2afcea77c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -196,7 +196,7 @@ class CloudController(object): def _describe_availability_zones(self, context, **kwargs): ctxt = context.elevated() - enabled_services = db.service_get_all(ctxt) + enabled_services = db.service_get_all(ctxt, False) disabled_services = db.service_get_all(ctxt, True) available_zones = [] for zone in [service.availability_zone for service @@ -221,7 +221,7 @@ class CloudController(object): rv = {'availabilityZoneInfo': [{'zoneName': 'nova', 'zoneState': 'available'}]} - services = db.service_get_all(context) + services = db.service_get_all(context, False) now = datetime.datetime.utcnow() hosts = [] for host in [service['host'] for service in services]: diff --git a/nova/db/api.py b/nova/db/api.py index add5bd83e..b3ca861e2 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -89,7 +89,7 @@ def service_get_by_host_and_topic(context, host, topic): return IMPL.service_get_by_host_and_topic(context, host, topic) -def service_get_all(context, disabled=False): +def service_get_all(context, disabled=None): """Get all services.""" return IMPL.service_get_all(context, disabled) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 3bf4f5eb8..321efe0e5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -143,12 +143,15 @@ def service_get(context, service_id, session=None): @require_admin_context -def service_get_all(context, disabled=False): +def service_get_all(context, disabled=None): session = get_session() - return session.query(models.Service).\ - filter_by(deleted=can_read_deleted(context)).\ - filter_by(disabled=disabled).\ - all() + query = session.query(models.Service).\ + filter_by(deleted=can_read_deleted(context)) + + if disabled is not None: + query = query.filter_by(disabled=disabled) + + return query.all() @require_admin_context -- cgit From 2a38aa7583be37ece6c42ba9307c2db0232dbed3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 22 Mar 2011 10:37:56 -0700 Subject: Whoops --- nova/api/openstack/servers.py | 5 +++-- nova/scheduler/api.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index be423c572..199d89c6d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -92,9 +92,10 @@ class Controller(wsgi.Controller): """ Returns server details by server id """ try: LOG.debug(_("***SHOW")) - instance = self.compute_api.get(req.environ['nova.context'], id) - builder = servers_views.get_view_builder(req) + instance = self.compute_api.routing_get( + req.environ['nova.context'], id) LOG.debug(_("***SHOW OUT %s" % instance)) + builder = servers_views.get_view_builder(req) return builder.build(instance, is_detail=True) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 4f189fe37..bd64f9b9b 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -149,8 +149,10 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): + LOG.debug(_("IN DECORATOR ...")) collection, context, item_id = \ self.get_collection_context_and_id(args, kwargs) + LOG.debug(_("IN DECORATOR 2...")) try: # Call the original function ... return f(*args, **kwargs) @@ -166,6 +168,7 @@ class reroute_compute(object): raise # Ask the children to provide an answer ... + LOG.debug(_("Asking child zones ...")) result = self._call_child_zones(zones, wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) -- cgit From 209da18033a49062bbcfaf7739db5959be87b142 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 22 Mar 2011 20:36:49 -0700 Subject: pep8 and fixed up zone-list --- nova/api/openstack/servers.py | 2 -- nova/api/openstack/zones.py | 18 +++++++----------- nova/db/api.py | 4 ---- nova/scheduler/api.py | 24 ++++++++++++++++++++---- nova/scheduler/zone_manager.py | 5 +++-- nova/tests/test_scheduler.py | 40 +++++++++++++++++++++++----------------- 6 files changed, 53 insertions(+), 40 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 199d89c6d..db6a1de97 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -91,10 +91,8 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - LOG.debug(_("***SHOW")) instance = self.compute_api.routing_get( req.environ['nova.context'], id) - LOG.debug(_("***SHOW OUT %s" % instance)) builder = servers_views.get_view_builder(req) return builder.build(instance, is_detail=True) except exception.NotFound: diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d129cf34f..6ce27e9a9 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -15,7 +15,6 @@ import common -from nova import db from nova import flags from nova import log as logging from nova import wsgi @@ -39,7 +38,8 @@ def _exclude_keys(item, keys): def _scrub_zone(zone): - return _filter_keys(zone, ('id', 'api_url')) + return _exclude_keys(zone, ('username', 'password', 'created_at', + 'deleted', 'deleted_at', 'updated_at')) class Controller(wsgi.Controller): @@ -54,12 +54,8 @@ class Controller(wsgi.Controller): # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... items = api.get_zone_list(req.environ['nova.context']) - if not items: - items = db.zone_get_all(req.environ['nova.context']) - items = common.limited(items, req) - items = [_exclude_keys(item, ['username', 'password']) - for item in items] + items = [_scrub_zone(item) for item in items] return dict(zones=items) def detail(self, req): @@ -82,23 +78,23 @@ class Controller(wsgi.Controller): def show(self, req, id): """Return data about the given zone id""" zone_id = int(id) - zone = db.zone_get(req.environ['nova.context'], zone_id) + zone = api.zone_get(req.environ['nova.context'], zone_id) return dict(zone=_scrub_zone(zone)) def delete(self, req, id): zone_id = int(id) - db.zone_delete(req.environ['nova.context'], zone_id) + api.zone_delete(req.environ['nova.context'], zone_id) return {} def create(self, req): context = req.environ['nova.context'] env = self._deserialize(req.body, req.get_content_type()) - zone = db.zone_create(context, env["zone"]) + zone = api.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id): context = req.environ['nova.context'] env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) - zone = db.zone_update(context, zone_id, env["zone"]) + zone = api.zone_update(context, zone_id, env["zone"]) return dict(zone=_scrub_zone(zone)) diff --git a/nova/db/api.py b/nova/db/api.py index a4cdb2ae2..7aedaa772 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -34,7 +34,6 @@ The underlying driver is loaded as a :class:`LazyPluggable`. from nova import exception from nova import flags -from nova import log as logging from nova import utils @@ -53,9 +52,6 @@ IMPL = utils.LazyPluggable(FLAGS['db_backend'], sqlalchemy='nova.db.sqlalchemy.api') -LOG = logging.getLogger('server') - - class NoMoreAddresses(exception.Error): """No more available addresses.""" pass diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index bd64f9b9b..c1417dfe4 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -55,9 +55,27 @@ def get_zone_list(context): items = _call_scheduler('get_zone_list', context) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') + if not items: + items = db.zone_get_all(context) return items +def zone_get(context, zone_id): + return db.zone_get(context, zone_id) + + +def zone_delete(context, zone_id): + return db.zone_delete(context, zone_id) + + +def zone_create(context, data): + return db.zone_create(context, data) + + +def zone_update(context, zone_id, data): + return db.zone_update(context, zone_id, data) + + def get_zone_capabilities(context, service=None): """Returns a dict of key, value capabilities for this zone, or for a particular class of services running in this zone.""" @@ -149,10 +167,8 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): - LOG.debug(_("IN DECORATOR ...")) collection, context, item_id = \ self.get_collection_context_and_id(args, kwargs) - LOG.debug(_("IN DECORATOR 2...")) try: # Call the original function ... return f(*args, **kwargs) @@ -181,7 +197,7 @@ class reroute_compute(object): """Ask the child zones to perform this operation. Broken out for testing.""" return child_zone_helper(zones, function) - + def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" @@ -212,7 +228,7 @@ class reroute_compute(object): del server[k] reduced_response.append(dict(server=server)) - if reduced_response: + if reduced_response: return reduced_response[0] # first for now. return {} diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index d32cc2e8f..198f9d4cc 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -58,8 +58,9 @@ class ZoneState(object): child zone.""" self.last_seen = datetime.now() self.attempt = 0 - self.name = zone_metadata["name"] - self.capabilities = zone_metadata["capabilities"] + self.name = zone_metadata.get("name", "n/a") + self.capabilities = ", ".join(["%s=%s" % (k, v) + for k, v in zone_metadata.iteritems() if k != 'name']) self.is_active = True def to_dict(self): diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 277ffe367..6df74dd61 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -949,6 +949,7 @@ class FakeZone(object): self.username = username self.password = password + def zone_get_all(context): return [ FakeZone('http://example.com', 'bob', 'xxx'), @@ -957,8 +958,8 @@ def zone_get_all(context): class FakeRerouteCompute(api.reroute_compute): def _call_child_zones(self, zones, function): - return [ ] - + return [] + def get_collection_context_and_id(self, args, kwargs): return ("servers", None, 1) @@ -982,6 +983,7 @@ class FakeResource(object): def pause(self): pass + class ZoneRedirectTest(test.TestCase): def setUp(self): super(ZoneRedirectTest, self).setUp() @@ -1024,27 +1026,28 @@ class ZoneRedirectTest(test.TestCase): def test_get_collection_context_and_id(self): decorator = api.reroute_compute("foo") self.assertEquals(decorator.get_collection_context_and_id( - (None, 10, 20), {}), ("servers", 10, 20)) + (None, 10, 20), {}), ("servers", 10, 20)) self.assertEquals(decorator.get_collection_context_and_id( - (None, 11,), dict(instance_id=21)), ("servers", 11, 21)) + (None, 11,), dict(instance_id=21)), ("servers", 11, 21)) self.assertEquals(decorator.get_collection_context_and_id( (None,), dict(context=12, instance_id=22)), ("servers", 12, 22)) def test_unmarshal_single_server(self): decorator = api.reroute_compute("foo") - self.assertEquals(decorator.unmarshall_result([]), {}) + self.assertEquals(decorator.unmarshall_result([]), {}) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(a=1, b=2)),]), - dict(server=dict(a=1, b=2))) + [FakeResource(dict(a=1, b=2)), ]), + dict(server=dict(a=1, b=2))) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(a=1, _b=2)),]), - dict(server=dict(a=1,))) + [FakeResource(dict(a=1, _b=2)), ]), + dict(server=dict(a=1,))) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(a=1, manager=2)),]), - dict(server=dict(a=1,))) + [FakeResource(dict(a=1, manager=2)), ]), + dict(server=dict(a=1,))) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(_a=1, manager=2)),]), - dict(server={})) + [FakeResource(dict(_a=1, manager=2)), ]), + dict(server={})) + class FakeServerCollection(object): def get(self, instance_id): @@ -1053,6 +1056,7 @@ class FakeServerCollection(object): def find(self, name): return FakeResource(dict(a=11, b=22)) + class FakeEmptyServerCollection(object): def get(self, f): raise novaclient.NotFound(1) @@ -1060,10 +1064,12 @@ class FakeEmptyServerCollection(object): def find(self, name): raise novaclient.NotFound(2) + class FakeNovaClient(object): def __init__(self, collection): self.servers = collection + class DynamicNovaClientTest(test.TestCase): def test_issue_novaclient_command_found(self): zone = FakeZone('http://example.com', 'bob', 'xxx') @@ -1078,17 +1084,17 @@ class DynamicNovaClientTest(test.TestCase): self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeServerCollection()), zone, "servers", "pause", 100), None) - + def test_issue_novaclient_command_not_found(self): zone = FakeZone('http://example.com', 'bob', 'xxx') self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "get", 100), None) + zone, "servers", "get", 100), None) self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "find", "name"), None) + zone, "servers", "find", "name"), None) self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "any", "name"), None) + zone, "servers", "any", "name"), None) -- cgit From 02db94dc33d72182201fd78651e5e5e82ab411c2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 01:22:11 -0700 Subject: Earlier versions of the python libvirt binding had getVersion in the libvirt namespace, not on the connection object. Check both. --- nova/virt/libvirt_conn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index f264cf619..0fabec4d0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -981,7 +981,11 @@ class LibvirtConnection(object): """ - return self._conn.getVersion() + # NOTE(justinsb): getVersion moved between libvirt versions + method = getattr(self._conn, 'getVersion', None) # Newer location + if method is None: + method = getattr(libvirt, 'getVersion') # Older location + return method() def get_cpu_info(self): """Get cpuinfo information. -- cgit From 327938fd67bb033597945bdabddaa155ae4bced6 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Wed, 23 Mar 2011 09:19:15 -0400 Subject: id -> instance_id --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 98e6f938a..6f08307be 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2216,7 +2216,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") - % id) + % instance_id) return result -- cgit From 05e6f82aa971606f7d33fb1de8f2c1c170d030de Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 23 Mar 2011 12:31:15 -0700 Subject: indenting cleanup --- nova/api/openstack/zones.py | 5 ++--- nova/compute/manager.py | 2 +- nova/flags.py | 2 +- nova/manager.py | 5 ++++- nova/rpc.py | 4 ++-- nova/scheduler/api.py | 2 +- nova/tests/test_zones.py | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d129cf34f..d4a59993b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -17,7 +17,6 @@ import common from nova import db from nova import flags -from nova import log as logging from nova import wsgi from nova.scheduler import api @@ -73,8 +72,8 @@ class Controller(wsgi.Controller): zone = dict(name=FLAGS.zone_name) caps = FLAGS.zone_capabilities for cap in caps: - key_values = cap.split('=') - zone[key_values[0]] = key_values[1] + key, value = cap.split('=') + zone[key] = value for item, (min_value, max_value) in items.iteritems(): zone[item] = "%s,%s" % (min_value, max_value) return dict(zone=zone) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index eae1fee68..289c91d8a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -118,7 +118,7 @@ class ComputeManager(manager.SchedulerDependentManager): self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(service_name="compute", - *args, **kwargs) + *args, **kwargs) def init_host(self): """Do any initialization that needs to be run if this is a diff --git a/nova/flags.py b/nova/flags.py index 3a8ec1a39..bf83b8e0f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -360,4 +360,4 @@ DEFINE_string('node_availability_zone', 'nova', DEFINE_string('zone_name', 'nova', 'name of this zone') DEFINE_list('zone_capabilities', ['hypervisor=xenserver;kvm', 'os=linux;windows'], - 'Key/Multi-value list representng capabilities of this zone') + 'Key/Multi-value list representng capabilities of this zone') diff --git a/nova/manager.py b/nova/manager.py index 508f133ca..804a50479 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -59,6 +59,8 @@ from nova.scheduler import api FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.manager') + class Manager(base.Base): def __init__(self, host=None, db_driver=None): @@ -83,6 +85,7 @@ class SchedulerDependentManager(Manager): should derive from this class. Otherwise they can derive from manager.Manager directly. Updates are only sent after update_service_capabilities is called with non-None values.""" + def __init__(self, host=None, db_driver=None, service_name="undefined"): self.last_capabilities = None self.service_name = service_name @@ -95,7 +98,7 @@ class SchedulerDependentManager(Manager): def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" if self.last_capabilities: - logging.debug(_("Notifying Schedulers of capabilities ...")) + LOG.debug(_("Notifying Schedulers of capabilities ...")) api.update_service_capabilities(context, self.service_name, self.host, self.last_capabilities) diff --git a/nova/rpc.py b/nova/rpc.py index 6ddaea092..388f78d69 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -219,8 +219,8 @@ class FanoutAdapterConsumer(AdapterConsumer): self.queue = "%s_fanout_%s" % (topic, unique) self.durable = False LOG.info(_("Created '%(exchange)s' fanout exchange " - "with '%(key)s' routing key"), - dict(exchange=self.exchange, key=self.routing_key)) + "with '%(key)s' routing key"), + dict(exchange=self.exchange, key=self.routing_key)) super(FanoutAdapterConsumer, self).__init__(connection=connection, topic=topic, proxy=proxy) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index e2cf3b6a3..19a05b716 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -52,7 +52,7 @@ def get_zone_capabilities(context, service=None): """Returns a dict of key, value capabilities for this zone, or for a particular class of services running in this zone.""" return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) + params=dict(service=service)) def update_service_capabilities(context, service_name, host, capabilities): diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 48e1442cf..688dc704d 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -96,7 +96,7 @@ class ZoneManagerTestCase(test.TestCase): zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), - svc10_a=(99, 99), svc10_b=(99, 99))) + svc10_a=(99, 99), svc10_b=(99, 99))) zm.update_service_capabilities("svc1", "host3", dict(c=5)) caps = zm.get_zone_capabilities(self, None) -- cgit From 05c4257545fb598222cb472d59d9b8be7ba9535a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 13:29:29 -0700 Subject: Give the user a nicer error message if they're using the Lucid libvirt --- nova/virt/libvirt_conn.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0fabec4d0..26d34b367 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -982,10 +982,14 @@ class LibvirtConnection(object): """ # NOTE(justinsb): getVersion moved between libvirt versions - method = getattr(self._conn, 'getVersion', None) # Newer location + # Trying to do be compatible with older versions is a lost cause + # But ... we can at least give the user a nice message + method = getattr(self._conn, 'getVersion', None) if method is None: - method = getattr(libvirt, 'getVersion') # Older location - return method() + raise exception.Error(_("libvirt version is too old" + " (does not support getVersion)")) + + return self._conn.getVersion() def get_cpu_info(self): """Get cpuinfo information. -- cgit From d966b1989224b8ba7bf580a3f3f8fc0f04b9a566 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 13:34:56 -0700 Subject: Keep the fallback code - we may want to do better version checking in future --- nova/virt/libvirt_conn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 26d34b367..ba794cfd8 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -988,8 +988,11 @@ class LibvirtConnection(object): if method is None: raise exception.Error(_("libvirt version is too old" " (does not support getVersion)")) + # NOTE(justinsb): If we wanted to get the version, we could: + # method = getattr(libvirt, 'getVersion', None) + # NOTE(justinsb): This would then rely on a proper version check - return self._conn.getVersion() + return method() def get_cpu_info(self): """Get cpuinfo information. -- cgit From 3c0fcc47be08ac4f3d508fd46f3b95036899aaad Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Mar 2011 13:39:01 -0700 Subject: fix utils.execute retries for osx also some minor misc cleanups --- nova/network/linux_net.py | 13 ++++++------- nova/tests/test_volume.py | 4 ++-- nova/utils.py | 9 +++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0a273588f..46158bbc0 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -212,10 +214,7 @@ class IptablesManager(object): """ def __init__(self, execute=None): if not execute: - if FLAGS.fake_network: - self.execute = lambda *args, **kwargs: ('', '') - else: - self.execute = utils.execute + self.execute = _execute else: self.execute = execute @@ -361,9 +360,6 @@ class IptablesManager(object): return new_filter -iptables_manager = IptablesManager() - - def metadata_forward(): """Create forwarding rule for metadata""" iptables_manager.ipv4['nat'].add_rule("PREROUTING", @@ -776,3 +772,6 @@ def _ip_bridge_cmd(action, params, device): cmd.extend(params) cmd.extend(['dev', device]) return cmd + + +iptables_manager = IptablesManager() diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index 5d68ca2ae..d71b75f3f 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -356,8 +356,8 @@ class ISCSITestCase(DriverTestCase): tid = db.volume_get_iscsi_target_num(self.context, volume_id_list[0]) self.mox.StubOutWithMock(self.volume.driver, '_execute') self.volume.driver._execute("sudo", "ietadm", "--op", "show", - "--tid=%(tid)d" % locals() - ).AndRaise(exception.ProcessExecutionError()) + "--tid=%(tid)d" % locals()).AndRaise( + exception.ProcessExecutionError()) self.mox.ReplayAll() self.assertRaises(exception.ProcessExecutionError, diff --git a/nova/utils.py b/nova/utils.py index 499af2039..249470636 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -170,10 +170,6 @@ def execute(*cmd, **kwargs): stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess - # call clean something up in between calls, without - # it two execute calls in a row hangs the second one - greenthread.sleep(0) return result except ProcessExecutionError: if not attempts: @@ -182,6 +178,11 @@ def execute(*cmd, **kwargs): LOG.debug(_("%r failed. Retrying."), cmd) if delay_on_retry: greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) def ssh_execute(ssh, cmd, process_input=None, -- cgit From 16372d3bc0181a57958ce185e89f1f21126b9e77 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 20:21:44 -0700 Subject: Don't try to parse a datetime if it is the empty string (or None) --- nova/image/glance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 171b28fde..9984a3ba1 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -73,7 +73,7 @@ class GlanceImageService(service.BaseImageService): Returns image with known timestamp fields converted to datetime objects """ for attr in ['created_at', 'updated_at', 'deleted_at']: - if image.get(attr) is not None: + if image.get(attr): image[attr] = self._parse_glance_iso8601_timestamp(image[attr]) return image -- cgit From 107c3f75d91dcb7aadf3136e964d1feb6c505dc7 Mon Sep 17 00:00:00 2001 From: Muneyuki Noguchi Date: Thu, 24 Mar 2011 16:21:50 +0900 Subject: Declare libvirt_type to avoid AttributeError in live_migration --- bin/nova-manage | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-manage b/bin/nova-manage index 69cbf6f95..6712fbadb 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -97,6 +97,7 @@ flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') flags.DECLARE('images_path', 'nova.image.local') +flags.DECLARE('libvirt_type', 'nova.virt.libvirt_conn') flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) -- cgit From f640d32bd8698fc2c30b2ca0454672d691f9b296 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Mar 2011 05:02:54 -0700 Subject: fix based on sirp's comments --- nova/scheduler/api.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index c1417dfe4..a4f304c62 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -118,16 +118,15 @@ def child_zone_helper(zone_list, func): _wrap_method(_process, func), zone_list)] -def _issue_novaclient_command(nova, zone, collection, method_name, \ - item_id): +def _issue_novaclient_command(nova, zone, collection, method_name, item_id): """Use novaclient to issue command to a single child zone. One of these will be run in parallel for each child zone.""" + manager = getattr(nova, collection) result = None try: - manager = getattr(nova, collection) - if isinstance(item_id, int) or item_id.isdigit(): + try: result = manager.get(int(item_id)) - else: + except ValueError, e: result = manager.find(name=item_id) except novaclient.NotFound: url = zone.api_url -- cgit