From 2a6ce075e19af5700960e3fb22c213e43a2e24b4 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 3 Mar 2011 16:28:04 -0400 Subject: start of fanout --- nova/rpc.py | 20 ++++++++++++++++++++ nova/scheduler/manager.py | 4 ++++ nova/scheduler/zone_manager.py | 4 ++++ nova/service.py | 6 ++++++ 4 files changed, 34 insertions(+) diff --git a/nova/rpc.py b/nova/rpc.py index 8fe4565dd..a02cdc90c 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -218,6 +218,16 @@ class TopicPublisher(Publisher): super(TopicPublisher, self).__init__(connection=connection) +class FanoutPublisher(Publisher): + """Publishes messages to a fanout exchange.""" + exchange_type = "fanout" + + def __init__(self, topic, connection=None): + self.exchange = "%s_fanout" % topic + self.durable = False + super(FanoutPublisher, self).__init__(connection=connection) + + class DirectConsumer(Consumer): """Consumes messages directly on a channel specified by msg_id""" exchange_type = "direct" @@ -360,6 +370,16 @@ def cast(context, topic, msg): publisher.close() +def fanout_cast(context, topic, msg): + """Sends a message on a fanout exchange without waiting for a response""" + LOG.debug(_("Making asynchronous fanout cast...")) + _pack_context(msg, context) + conn = Connection.instance() + publisher = FanoutPublisher(topic, connection=conn) + publisher.send(msg) + publisher.close() + + def generic_response(message_data, message): """Logs a result and exits""" LOG.debug(_('response %s'), message_data) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index c94397210..7541523b0 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,6 +59,10 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() + def update_compute_capabilities(self, context=None): + """Process a compute node update.""" + return self.zone_manager.update_compute_capabilities() + def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index edf9000cc..eedc5c235 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -105,6 +105,7 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} + self.compute_states = {} self.green_pool = greenpool.GreenPool() def get_zone_list(self): @@ -141,3 +142,6 @@ class ZoneManager(object): self.last_zone_db_check = datetime.now() self._refresh_from_db(context) self._poll_zones(context) + + def update_compute_capabilities(self): + logging.debug(_("****** UPDATE COMPUTE CAPABILITIES *******")) diff --git a/nova/service.py b/nova/service.py index f47358089..3ecf46525 100644 --- a/nova/service.py +++ b/nova/service.py @@ -84,6 +84,7 @@ class Service(object): conn1 = rpc.Connection.instance(new=True) conn2 = rpc.Connection.instance(new=True) + conn3 = rpc.Connection.instance(new=True) if self.report_interval: consumer_all = rpc.AdapterConsumer( connection=conn1, @@ -93,9 +94,14 @@ class Service(object): connection=conn2, topic='%s.%s' % (self.topic, self.host), proxy=self) + fanout = rpc.AdapterConsumer( + connection=conn2, + topic='%s_fanout' % self.topic, + proxy=self) self.timers.append(consumer_all.attach_to_eventlet()) self.timers.append(consumer_node.attach_to_eventlet()) + self.timers.append(fanout.attach_to_eventlet()) pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) -- cgit From 3d9d99a53d372abf9f69f1d6e66fa6c6491ec752 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 3 Mar 2011 19:27:15 -0400 Subject: tests passing --- nova/tests/test_service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 45d9afa6c..cb31a3c43 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -120,6 +120,12 @@ class ServiceTestCase(test.TestCase): proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) + rpc.AdapterConsumer(connection=mox.IgnoreArg(), + topic='%s_fanout' % topic, + proxy=mox.IsA(service.Service)).AndReturn( + rpc.AdapterConsumer) + + rpc.AdapterConsumer.attach_to_eventlet() rpc.AdapterConsumer.attach_to_eventlet() rpc.AdapterConsumer.attach_to_eventlet() -- cgit From a10d863e5e6127b8e987719ddfb60142b9f08db9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 13:36:45 -0800 Subject: scheduler manager --- nova/scheduler_manager.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 nova/scheduler_manager.py diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py new file mode 100644 index 000000000..c78b6fea5 --- /dev/null +++ b/nova/scheduler_manager.py @@ -0,0 +1,39 @@ +# Copyright 2011 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This module provides SchedulerDependentManager, a base class for +any Manager that has Capabilities that should be related to the +Scheduler. + +These Capabilities are hints that can help the scheduler route +requests to the appropriate service instance. +""" + +from nova import manager +from nova.scheduler import api + + +FLAGS = flags.FLAGS + + +def SchedulerDependentManage(manager.Manager): + def __init__(self, host=None, db_driver=None): + self.last_capabilities = {} + super(SchedulerDependentManager, self).__init__(host, db_driver) + + def periodic_tasks(self, context=None): + """Pass data back to the scheduler at a periodic interval""" + logging.debug(_("Notifying Schedulers of capabilities ...")) -- cgit From 5d821114fc20c88e36f079089cfe655d8188914a Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 16:40:08 -0800 Subject: service ping working --- nova/compute/manager.py | 4 ++-- nova/scheduler_manager.py | 9 +++++++-- nova/service.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3af97683f..fd88158f7 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -44,7 +44,7 @@ import functools from nova import exception from nova import flags from nova import log as logging -from nova import manager +from nova import scheduler_manager from nova import rpc from nova import utils from nova.compute import power_state @@ -99,7 +99,7 @@ def checks_instance_lock(function): return decorated_function -class ComputeManager(manager.Manager): +class ComputeManager(scheduler_manager.SchedulerDependentManager): """Manages the running instances from creation to destruction.""" diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index c78b6fea5..987ca8e90 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -22,18 +22,23 @@ These Capabilities are hints that can help the scheduler route requests to the appropriate service instance. """ +import sys + +from nova import flags from nova import manager from nova.scheduler import api +from nova import log as logging FLAGS = flags.FLAGS -def SchedulerDependentManage(manager.Manager): +class SchedulerDependentManager(manager.Manager): def __init__(self, host=None, db_driver=None): self.last_capabilities = {} super(SchedulerDependentManager, self).__init__(host, db_driver) def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" - logging.debug(_("Notifying Schedulers of capabilities ...")) + logging.debug(_("*** Notifying Schedulers of capabilities ...")) + super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/service.py b/nova/service.py index 3ecf46525..66287b505 100644 --- a/nova/service.py +++ b/nova/service.py @@ -95,7 +95,7 @@ class Service(object): topic='%s.%s' % (self.topic, self.host), proxy=self) fanout = rpc.AdapterConsumer( - connection=conn2, + connection=conn3, topic='%s_fanout' % self.topic, proxy=self) -- cgit From 7297e6bf1536f20540200f28154c15d63372d943 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 20:58:59 -0800 Subject: fanout kinda working --- nova/rpc.py | 51 ++++++++++++++++++++++++++++++----------------- nova/scheduler/api.py | 7 +++++++ nova/scheduler/manager.py | 2 +- nova/scheduler_manager.py | 2 ++ nova/service.py | 8 ++++---- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index a02cdc90c..e0cf6d301 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -137,24 +137,7 @@ class Consumer(messaging.Consumer): return timer -class Publisher(messaging.Publisher): - """Publisher base class""" - pass - - -class TopicConsumer(Consumer): - """Consumes messages on a specific topic""" - exchange_type = "topic" - - def __init__(self, connection=None, topic="broadcast"): - self.queue = topic - self.routing_key = topic - self.exchange = FLAGS.control_exchange - self.durable = False - super(TopicConsumer, self).__init__(connection=connection) - - -class AdapterConsumer(TopicConsumer): +class AdapterConsumer(Consumer): """Calls methods on a proxy object based on method and args""" def __init__(self, connection=None, topic="broadcast", proxy=None): LOG.debug(_('Initing the Adapter Consumer for %s') % topic) @@ -207,6 +190,37 @@ class AdapterConsumer(TopicConsumer): return +class Publisher(messaging.Publisher): + """Publisher base class""" + pass + + +class TopicAdapterConsumer(AdapterConsumer): + """Consumes messages on a specific topic""" + exchange_type = "topic" + + def __init__(self, connection=None, topic="broadcast", proxy=None): + self.queue = topic + self.routing_key = topic + self.exchange = FLAGS.control_exchange + self.durable = False + super(TopicAdapterConsumer, self).__init__(connection=connection, + topic=topic, proxy=proxy) + + +class FanoutAdapterConsumer(AdapterConsumer): + """Consumes messages from a fanout exchange""" + exchange_type = "fanout" + + def __init__(self, connection=None, topic="broadcast", proxy=None): + self.exchange = "%s_fanout" % topic + self.routing_key = topic + self.queue = "ignored" + self.durable = False + super(FanoutAdapterConsumer, self).__init__(connection=connection, + topic=topic, proxy=proxy) + + class TopicPublisher(Publisher): """Publishes messages on a specific topic""" exchange_type = "topic" @@ -214,6 +228,7 @@ class TopicPublisher(Publisher): def __init__(self, connection=None, topic="broadcast"): self.routing_key = topic self.exchange = FLAGS.control_exchange + self.queue = "ignored" self.durable = False super(TopicPublisher, self).__init__(connection=connection) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 8491bf3a9..53d72507f 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -47,3 +47,10 @@ class API: for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') return items + + @classmethod + def update_service_capabilities(cls, context, service_name, capabilities): + kwargs = dict(method='update_service_capabilities', + service_name=service_name, capabilities=capabilities) + return rpc.fanout_cast(context, 'scheduler', kwargs) + diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 7541523b0..3ff43a9d9 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,7 +59,7 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() - def update_compute_capabilities(self, context=None): + def update_service_capabilities(self, context=None, service_name=None, capabilities={}): """Process a compute node update.""" return self.zone_manager.update_compute_capabilities() diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index 987ca8e90..a45301617 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -41,4 +41,6 @@ class SchedulerDependentManager(manager.Manager): def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" logging.debug(_("*** Notifying Schedulers of capabilities ...")) + api.API.update_service_capabilities(context, 'compute', self.last_capabilities) + super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/service.py b/nova/service.py index 66287b505..a98d6aac1 100644 --- a/nova/service.py +++ b/nova/service.py @@ -86,17 +86,17 @@ class Service(object): conn2 = rpc.Connection.instance(new=True) conn3 = rpc.Connection.instance(new=True) if self.report_interval: - consumer_all = rpc.AdapterConsumer( + consumer_all = rpc.TopicAdapterConsumer( connection=conn1, topic=self.topic, proxy=self) - consumer_node = rpc.AdapterConsumer( + consumer_node = rpc.TopicAdapterConsumer( connection=conn2, topic='%s.%s' % (self.topic, self.host), proxy=self) - fanout = rpc.AdapterConsumer( + fanout = rpc.FanoutAdapterConsumer( connection=conn3, - topic='%s_fanout' % self.topic, + topic=self.topic, proxy=self) self.timers.append(consumer_all.attach_to_eventlet()) -- cgit From 32f062c389a530b6af3f864eb4030a68d0a26eb1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 20 Feb 2011 22:33:39 -0800 Subject: fanout works --- nova/rpc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index e0cf6d301..601e45b47 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -215,8 +215,12 @@ class FanoutAdapterConsumer(AdapterConsumer): def __init__(self, connection=None, topic="broadcast", proxy=None): self.exchange = "%s_fanout" % topic self.routing_key = topic - self.queue = "ignored" + unique = uuid.uuid4().hex + self.queue = "%s_fanout_%s" % (topic, unique) self.durable = False + LOG.info(_("Created '%(exchange)s' fanout exchange " + "with '%(key)s' routing key"), + dict(exchange=self.exchange, key=self.routing_key)) super(FanoutAdapterConsumer, self).__init__(connection=connection, topic=topic, proxy=proxy) @@ -228,7 +232,6 @@ class TopicPublisher(Publisher): def __init__(self, connection=None, topic="broadcast"): self.routing_key = topic self.exchange = FLAGS.control_exchange - self.queue = "ignored" self.durable = False super(TopicPublisher, self).__init__(connection=connection) @@ -239,7 +242,10 @@ class FanoutPublisher(Publisher): def __init__(self, topic, connection=None): self.exchange = "%s_fanout" % topic + self.queue = "%s_fanout" % topic self.durable = False + LOG.info(_("Writing to '%(exchange)s' fanout exchange"), + dict(exchange=self.exchange)) super(FanoutPublisher, self).__init__(connection=connection) -- cgit From 05065a72ab06879d8ddd48ab45bc870386a0562d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 23 Feb 2011 14:41:11 -0800 Subject: tests working again --- nova/tests/test_rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_rpc.py b/nova/tests/test_rpc.py index 4820e04fb..44d7c91eb 100644 --- a/nova/tests/test_rpc.py +++ b/nova/tests/test_rpc.py @@ -36,7 +36,7 @@ class RpcTestCase(test.TestCase): super(RpcTestCase, self).setUp() self.conn = rpc.Connection.instance(True) self.receiver = TestReceiver() - self.consumer = rpc.AdapterConsumer(connection=self.conn, + self.consumer = rpc.TopicAdapterConsumer(connection=self.conn, topic='test', proxy=self.receiver) self.consumer.attach_to_eventlet() @@ -97,7 +97,7 @@ class RpcTestCase(test.TestCase): nested = Nested() conn = rpc.Connection.instance(True) - consumer = rpc.AdapterConsumer(connection=conn, + consumer = rpc.TopicAdapterConsumer(connection=conn, topic='nested', proxy=nested) consumer.attach_to_eventlet() -- cgit From 05fc3ea219f36bc1c246179b25b1feb017888b01 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 23 Feb 2011 17:58:32 -0800 Subject: Tests all working again --- nova/compute/manager.py | 3 ++- nova/flags.py | 2 +- nova/network/manager.py | 7 ++++--- nova/scheduler/api.py | 1 - nova/scheduler/manager.py | 3 ++- nova/scheduler_manager.py | 22 ++++++++++++++++++---- nova/tests/test_service.py | 25 ++++++++++++++----------- nova/tests/test_test.py | 2 +- nova/volume/manager.py | 7 ++++--- 9 files changed, 46 insertions(+), 26 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index fd88158f7..b307ffa59 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -112,7 +112,8 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): self.driver = utils.import_object(compute_driver) self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) - super(ComputeManager, self).__init__(*args, **kwargs) + super(ComputeManager, self).__init__(service_name="compute", + *args, **kwargs) def init_host(self): """Do any initialization that needs to be run if this is a diff --git a/nova/flags.py b/nova/flags.py index 6f37c82f0..7036180fc 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -356,5 +356,5 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'kypervisor:xenserver;os:linux', +DEFINE_string('zone_capabilities', 'hypervisor:xenserver;os:linux', 'Key/Value tags which represent capabilities of this zone') diff --git a/nova/network/manager.py b/nova/network/manager.py index b36dd59cf..f5f5b17aa 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -55,7 +55,7 @@ from nova import db from nova import exception from nova import flags from nova import log as logging -from nova import manager +from nova import scheduler_manager from nova import utils from nova import rpc @@ -105,7 +105,7 @@ class AddressAlreadyAllocated(exception.Error): pass -class NetworkManager(manager.Manager): +class NetworkManager(scheduler_manager.SchedulerDependentManager): """Implements common network manager functionality. This class must be subclassed to support specific topologies. @@ -116,7 +116,8 @@ class NetworkManager(manager.Manager): if not network_driver: network_driver = FLAGS.network_driver self.driver = utils.import_object(network_driver) - super(NetworkManager, self).__init__(*args, **kwargs) + super(NetworkManager, self).__init__(service_name='network', + *args, **kwargs) def init_host(self): """Do any initialization that needs to be run if this is a diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 53d72507f..6a6bfc032 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -53,4 +53,3 @@ class API: kwargs = dict(method='update_service_capabilities', service_name=service_name, capabilities=capabilities) return rpc.fanout_cast(context, 'scheduler', kwargs) - diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 3ff43a9d9..6b5c6e246 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,7 +59,8 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() - def update_service_capabilities(self, context=None, service_name=None, capabilities={}): + def update_service_capabilities(self, context=None, service_name=None, + capabilities={}): """Process a compute node update.""" return self.zone_manager.update_compute_capabilities() diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index a45301617..65bd71c05 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -34,13 +34,27 @@ FLAGS = flags.FLAGS class SchedulerDependentManager(manager.Manager): - def __init__(self, host=None, db_driver=None): - self.last_capabilities = {} + + """Periodically send capability updates to the Scheduler services. + Services that need to update the Scheduler of their capabilities + should derive from this class. Otherwise they can derive from + manager.Manager directly. Updates are only sent after + update_service_capabilities is called with non-None values.""" + + def __init__(self, host=None, db_driver=None, service_name="undefined"): + self.last_capabilities = None + self.service_name = service_name super(SchedulerDependentManager, self).__init__(host, db_driver) + def update_service_capabilities(self, capabilities): + """Remember these capabilities to send on next periodic update.""" + self.last_capabilities = capabilities + def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" - logging.debug(_("*** Notifying Schedulers of capabilities ...")) - api.API.update_service_capabilities(context, 'compute', self.last_capabilities) + if self.last_capabilities: + logging.debug(_("*** Notifying Schedulers of capabilities ...")) + api.API.update_service_capabilities(context, self.service_name, + self.last_capabilities) super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index cb31a3c43..b006caadd 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -108,26 +108,29 @@ class ServiceTestCase(test.TestCase): app = service.Service.create(host=host, binary=binary) self.mox.StubOutWithMock(rpc, - 'AdapterConsumer', + 'TopicAdapterConsumer', use_mock_anything=True) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), + self.mox.StubOutWithMock(rpc, + 'FanoutAdapterConsumer', + use_mock_anything=True) + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), topic=topic, proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) + rpc.TopicAdapterConsumer) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), topic='%s.%s' % (topic, host), proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) + rpc.TopicAdapterConsumer) - rpc.AdapterConsumer(connection=mox.IgnoreArg(), - topic='%s_fanout' % topic, + rpc.FanoutAdapterConsumer(connection=mox.IgnoreArg(), + topic=topic, proxy=mox.IsA(service.Service)).AndReturn( - rpc.AdapterConsumer) + rpc.FanoutAdapterConsumer) - rpc.AdapterConsumer.attach_to_eventlet() - rpc.AdapterConsumer.attach_to_eventlet() - rpc.AdapterConsumer.attach_to_eventlet() + rpc.TopicAdapterConsumer.attach_to_eventlet() + rpc.TopicAdapterConsumer.attach_to_eventlet() + rpc.FanoutAdapterConsumer.attach_to_eventlet() service_create = {'host': host, 'binary': binary, diff --git a/nova/tests/test_test.py b/nova/tests/test_test.py index e237674e6..35c838065 100644 --- a/nova/tests/test_test.py +++ b/nova/tests/test_test.py @@ -34,7 +34,7 @@ class IsolationTestCase(test.TestCase): def test_rpc_consumer_isolation(self): connection = rpc.Connection.instance(new=True) - consumer = rpc.TopicConsumer(connection, topic='compute') + consumer = rpc.TopicAdapterConsumer(connection, topic='compute') consumer.register_callback( lambda x, y: self.fail('I should never be called')) consumer.attach_to_eventlet() diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 3e8bc16b3..c53acf1e3 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -49,7 +49,7 @@ from nova import context from nova import exception from nova import flags from nova import log as logging -from nova import manager +from nova import scheduler_manager from nova import utils @@ -64,14 +64,15 @@ flags.DEFINE_boolean('use_local_volumes', True, 'if True, will not discover local volumes') -class VolumeManager(manager.Manager): +class VolumeManager(scheduler_manager.SchedulerDependentManager): """Manages attachable block storage devices.""" def __init__(self, volume_driver=None, *args, **kwargs): """Load the driver from the one specified in args, or from flags.""" if not volume_driver: volume_driver = FLAGS.volume_driver self.driver = utils.import_object(volume_driver) - super(VolumeManager, self).__init__(*args, **kwargs) + super(VolumeManager, self).__init__(service_name='volume', + *args, **kwargs) # NOTE(vish): Implementation specific db handling is done # by the driver. self.driver.db = self.db -- cgit From 65b9dfbea28f1607ef471e78b73ba77183d943f6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 01:53:01 -0800 Subject: capability aggregation working --- nova/api/openstack/zones.py | 15 ++++++++++++--- nova/scheduler/api.py | 15 +++++++++++++-- nova/scheduler/manager.py | 12 +++++++++--- nova/scheduler/zone_manager.py | 35 ++++++++++++++++++++++++++++++++--- nova/scheduler_manager.py | 6 ++---- 5 files changed, 68 insertions(+), 15 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 989b3a235..c6c27dd4b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -15,9 +15,10 @@ import common +from nova import db from nova import flags +from nova import log as logging from nova import wsgi -from nova import db from nova.scheduler import api @@ -67,8 +68,16 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - return dict(zone=dict(name=FLAGS.zone_name, - capabilities=FLAGS.zone_capabilities)) + items = api.API().get_zone_capabilities(req.environ['nova.context']) + + zone = dict(name=FLAGS.zone_name) + caps = FLAGS.zone_capabilities.split(';') + for cap in caps: + key_values = cap.split(':') + zone[key_values[0]] = key_values[1] + for item, (min_value, max_value) in items.iteritems(): + zone[item] = "%s,%s" % (min_value, max_value) + return dict(zone=zone) def show(self, req, id): """Return data about the given zone id""" diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6a6bfc032..ac38350ed 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -43,13 +43,24 @@ class API: return rpc.call(context, queue, kwargs) def get_zone_list(self, context): + """Return a list of zones assoicated with this zone.""" items = self._call_scheduler('get_zone_list', context) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') return items + def get_zone_capabilities(self, context, service=None): + """Returns a dict of key, value capabilities for this zone, + or for a particular class of services running in this zone.""" + return self._call_scheduler('get_zone_capabilities', context=context, + params=dict(service=service)) + @classmethod - def update_service_capabilities(cls, context, service_name, capabilities): + def update_service_capabilities(cls, context, service_name, host, + capabilities): + """Send an update to all the scheduler services informing them + of the capabilities of this service.""" kwargs = dict(method='update_service_capabilities', - service_name=service_name, capabilities=capabilities) + args=dict(service_name=service_name, host=host, + capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 6b5c6e246..1bda77d89 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,10 +59,16 @@ class SchedulerManager(manager.Manager): """Get a list of zones from the ZoneManager.""" return self.zone_manager.get_zone_list() + def get_zone_capabilities(self, context=None, service=None): + """Get the normalized set of capabilites for this zone, + or for a particular service.""" + return self.zone_manager.get_zone_capabilities(context, service) + def update_service_capabilities(self, context=None, service_name=None, - capabilities={}): - """Process a compute node update.""" - return self.zone_manager.update_compute_capabilities() + host=None, capabilities={}): + """Process a capability update from a service node.""" + self.zone_manager.update_service_capabilities(service_name, + host, capabilities) def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index eedc5c235..09c9811f3 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -105,13 +105,37 @@ class ZoneManager(object): def __init__(self): self.last_zone_db_check = datetime.min self.zone_states = {} - self.compute_states = {} + self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() def get_zone_list(self): """Return the list of zones we know about.""" return [zone.to_dict() for zone in self.zone_states.values()] + def get_zone_capabilities(self, context, service=None): + """Roll up all the individual host info to generic 'service' + capabilities. Each capability is aggregated into + _min and _max values.""" + service_dict = self.service_states + if service: + service_dict = dict(service_name=service, + hosts=self.service_states.get(service, {})) + + # TODO(sandy) - be smarter about fabricating this structure. + # But it's likely to change once we understand what the Best-Match + # code will need better. + combined = {} # { _ : (min, max), ... } + for service_name, host_dict in service_dict.iteritems(): + for host, caps_dict in host_dict.iteritems(): + for cap, value in caps_dict.iteritems(): + key = "%s_%s" % (service_name, cap) + min_value, max_value = combined.get(key, (value, value)) + min_value = min(min_value, value) + max_value = max(max_value, value) + combined[key] = (min_value, max_value) + + return combined + def _refresh_from_db(self, context): """Make our zone state map match the db.""" # Add/update existing zones ... @@ -143,5 +167,10 @@ class ZoneManager(object): self._refresh_from_db(context) self._poll_zones(context) - def update_compute_capabilities(self): - logging.debug(_("****** UPDATE COMPUTE CAPABILITIES *******")) + def update_service_capabilities(self, service_name, host, capabilities): + """Update the per-service capabilities based on this notification.""" + logging.debug(_("Received %(service_name)s service update from " + "%(host)s: %(capabilities)s") % locals()) + service_caps = self.service_states.get(service_name, {}) + service_caps[host] = capabilities + self.service_states[service_name] = service_caps diff --git a/nova/scheduler_manager.py b/nova/scheduler_manager.py index 65bd71c05..ca39b85dd 100644 --- a/nova/scheduler_manager.py +++ b/nova/scheduler_manager.py @@ -34,13 +34,11 @@ FLAGS = flags.FLAGS class SchedulerDependentManager(manager.Manager): - """Periodically send capability updates to the Scheduler services. Services that need to update the Scheduler of their capabilities should derive from this class. Otherwise they can derive from manager.Manager directly. Updates are only sent after update_service_capabilities is called with non-None values.""" - def __init__(self, host=None, db_driver=None, service_name="undefined"): self.last_capabilities = None self.service_name = service_name @@ -53,8 +51,8 @@ class SchedulerDependentManager(manager.Manager): def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" if self.last_capabilities: - logging.debug(_("*** Notifying Schedulers of capabilities ...")) + logging.debug(_("Notifying Schedulers of capabilities ...")) api.API.update_service_capabilities(context, self.service_name, - self.last_capabilities) + self.host, self.last_capabilities) super(SchedulerDependentManager, self).periodic_tasks(context) -- cgit From c8df2602fd8f4f2cb7716e6283f3779c6895a479 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 14:32:25 -0800 Subject: service capabilities test --- nova/scheduler/driver.py | 7 +++++++ nova/scheduler/manager.py | 3 ++- nova/tests/test_zones.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 66e46c1b9..317a039cc 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -45,6 +45,13 @@ class WillNotSchedule(exception.Error): class Scheduler(object): """The base class that all Scheduler clases should inherit from.""" + def __init__(self): + self.zone_manager = None + + def set_zone_manager(self, zone_manager): + """Called by the Scheduler Service to supply a ZoneManager.""" + self.zone_manager = zone_manager + @staticmethod def service_is_up(service): """Check whether a service is up based on last heartbeat.""" diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 1bda77d89..d3d338943 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -41,10 +41,11 @@ flags.DEFINE_string('scheduler_driver', class SchedulerManager(manager.Manager): """Chooses a host to run instances on.""" def __init__(self, scheduler_driver=None, *args, **kwargs): + self.zone_manager = zone_manager.ZoneManager() if not scheduler_driver: scheduler_driver = FLAGS.scheduler_driver self.driver = utils.import_object(scheduler_driver) - self.zone_manager = zone_manager.ZoneManager() + self.driver.set_zone_manager(self.zone_manager) super(SchedulerManager, self).__init__(*args, **kwargs) def __getattr__(self, key): diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 5a52a0506..3ca71d5f1 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -76,6 +76,34 @@ class ZoneManagerTestCase(test.TestCase): self.assertEquals(len(zm.zone_states), 1) self.assertEquals(zm.zone_states[1].username, 'user1') + def test_service_capabilities(self): + zm = zone_manager.ZoneManager() + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, {}) + + zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2))) + + zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) + + zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) + + zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), + svc10_a=(99, 99), svc10_b=(99, 99))) + + zm.update_service_capabilities("svc1", "host3", dict(c=5)) + caps = zm.get_zone_capabilities(self, None) + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), + svc1_c=(5, 5), svc10_a=(99, 99), + svc10_b=(99, 99))) + def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 47bbfaab52642f3ff79bcdefb8d705fb02b549f9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 15:23:15 -0800 Subject: new tests --- nova/api/openstack/zones.py | 4 ++-- nova/scheduler/api.py | 37 ++++++++++++++++++---------------- nova/scheduler/zone_manager.py | 3 +-- nova/tests/api/openstack/test_zones.py | 30 ++++++++++++++++++++++++--- nova/tests/test_zones.py | 6 ++++++ 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index c6c27dd4b..fecbd6fa3 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -53,7 +53,7 @@ class Controller(wsgi.Controller): """Return all zones in brief""" # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... - items = api.API().get_zone_list(req.environ['nova.context']) + items = api.API.get_zone_list(req.environ['nova.context']) if not items: items = db.zone_get_all(req.environ['nova.context']) @@ -68,7 +68,7 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - items = api.API().get_zone_capabilities(req.environ['nova.context']) + items = api.API.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) caps = FLAGS.zone_capabilities.split(';') diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index ac38350ed..fcff2f146 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -25,34 +25,37 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.api') -class API: - """API for interacting with the scheduler.""" +def _call_scheduler(method, context, params=None): + """Generic handler for RPC calls to the scheduler. - def _call_scheduler(self, method, context, params=None): - """Generic handler for RPC calls to the scheduler. + :param params: Optional dictionary of arguments to be passed to the + scheduler worker - :param params: Optional dictionary of arguments to be passed to the - scheduler worker + :retval: Result returned by scheduler worker + """ + if not params: + params = {} + queue = FLAGS.scheduler_topic + kwargs = {'method': method, 'args': params} + return rpc.call(context, queue, kwargs) - :retval: Result returned by scheduler worker - """ - if not params: - params = {} - queue = FLAGS.scheduler_topic - kwargs = {'method': method, 'args': params} - return rpc.call(context, queue, kwargs) - def get_zone_list(self, context): +class API: + """API for interacting with the scheduler.""" + + @classmethod + def get_zone_list(cls, context): """Return a list of zones assoicated with this zone.""" - items = self._call_scheduler('get_zone_list', context) + items = _call_scheduler('get_zone_list', context) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') return items - def get_zone_capabilities(self, context, service=None): + @classmethod + def get_zone_capabilities(cls, context, service=None): """Returns a dict of key, value capabilities for this zone, or for a particular class of services running in this zone.""" - return self._call_scheduler('get_zone_capabilities', context=context, + return _call_scheduler('get_zone_capabilities', context=context, params=dict(service=service)) @classmethod diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 09c9811f3..c1a50dbc3 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -118,8 +118,7 @@ class ZoneManager(object): _min and _max values.""" service_dict = self.service_states if service: - service_dict = dict(service_name=service, - hosts=self.service_states.get(service, {})) + service_dict = {service: self.service_states.get(service, {})} # TODO(sandy) - be smarter about fabricating this structure. # But it's likely to change once we understand what the Best-Match diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 82b892b9e..33a66df0b 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -75,6 +75,10 @@ def zone_get_all_db(context): ] +def zone_caps(method, context, params): + return dict() + + class ZonesTest(test.TestCase): def setUp(self): super(ZonesTest, self).setUp() @@ -93,13 +97,18 @@ class ZonesTest(test.TestCase): self.stubs.Set(nova.db, 'zone_create', zone_create) self.stubs.Set(nova.db, 'zone_delete', zone_delete) + self.old_zone_name = FLAGS.zone_name + self.old_zone_caps = FLAGS.zone_capabilities + def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin + FLAGS.zone_name = self.old_zone_name + FLAGS.zone_capabilities = self.old_zone_caps super(ZonesTest, self).tearDown() def test_get_zone_list_scheduler(self): - self.stubs.Set(api.API, '_call_scheduler', zone_get_all_scheduler) + self.stubs.Set(api, '_call_scheduler', zone_get_all_scheduler) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -108,8 +117,7 @@ class ZonesTest(test.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_list_db(self): - self.stubs.Set(api.API, '_call_scheduler', - zone_get_all_scheduler_empty) + self.stubs.Set(api, '_call_scheduler', zone_get_all_scheduler_empty) self.stubs.Set(nova.db, 'zone_get_all', zone_get_all_db) req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) @@ -162,3 +170,19 @@ class ZonesTest(test.TestCase): self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://example.com') self.assertFalse('username' in res_dict['zone']) + + def test_zone_info(self): + FLAGS.zone_name = 'darksecret' + FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' + self.stubs.Set(api, '_call_scheduler', zone_caps) + + body = dict(zone=dict(username='zeb', password='sneaky')) + req = webob.Request.blank('/v1.0/zones/info') + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['zone']['name'], 'darksecret') + self.assertEqual(res_dict['zone']['cap1'], 'a,b') + self.assertEqual(res_dict['zone']['cap2'], 'c,d') + diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 3ca71d5f1..79d766f28 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -104,6 +104,12 @@ class ZoneManagerTestCase(test.TestCase): svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) + caps = zm.get_zone_capabilities(self, 'svc1') + self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), + svc1_c=(5, 5))) + caps = zm.get_zone_capabilities(self, 'svc10') + self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) + def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 307dcb7906ff066e2883cdee8998dfa78ebc8221 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Feb 2011 15:44:27 -0800 Subject: sorry, pep8 --- nova/tests/api/openstack/test_zones.py | 5 ++--- nova/tests/test_zones.py | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 33a66df0b..a40d46749 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -76,7 +76,7 @@ def zone_get_all_db(context): def zone_caps(method, context, params): - return dict() + return dict() class ZonesTest(test.TestCase): @@ -175,7 +175,7 @@ class ZonesTest(test.TestCase): FLAGS.zone_name = 'darksecret' FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' self.stubs.Set(api, '_call_scheduler', zone_caps) - + body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/info') @@ -185,4 +185,3 @@ class ZonesTest(test.TestCase): self.assertEqual(res_dict['zone']['name'], 'darksecret') self.assertEqual(res_dict['zone']['cap1'], 'a,b') self.assertEqual(res_dict['zone']['cap2'], 'c,d') - diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 79d766f28..48e1442cf 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -88,28 +88,28 @@ class ZoneManagerTestCase(test.TestCase): zm.update_service_capabilities("svc1", "host1", dict(a=2, b=3)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 2), svc1_b=(3, 3))) - + zm.update_service_capabilities("svc1", "host2", dict(a=20, b=30)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30))) - + zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc10_a=(99, 99), svc10_b=(99, 99))) - + zm.update_service_capabilities("svc1", "host3", dict(c=5)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5), svc10_a=(99, 99), svc10_b=(99, 99))) - + caps = zm.get_zone_capabilities(self, 'svc1') self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), svc1_c=(5, 5))) caps = zm.get_zone_capabilities(self, 'svc10') self.assertEquals(caps, dict(svc10_a=(99, 99), svc10_b=(99, 99))) - + def test_refresh_from_db_replace_existing(self): zm = zone_manager.ZoneManager() zone_state = zone_manager.ZoneState() -- cgit From 7afebad78de462918b89d61f5d8e0cee8bc11068 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 4 Mar 2011 13:45:43 -0500 Subject: Fix api logging to show proper path and controller:action. --- nova/api/ec2/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5adc2c075..2493adc95 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -61,10 +61,13 @@ class RequestLogging(wsgi.Middleware): return rv def log_request_completion(self, response, request, start): - controller = request.environ.get('ec2.controller', None) - if controller: - controller = controller.__class__.__name__ - action = request.environ.get('ec2.action', None) + apireq = request.environ.get('ec2.request', None) + if apirequest: + controller = apireq.controller + action = apireq.action + else: + controller = None + action = None ctxt = request.environ.get('ec2.context', None) delta = utils.utcnow() - start seconds = delta.seconds @@ -75,7 +78,7 @@ class RequestLogging(wsgi.Middleware): microseconds, request.remote_addr, request.method, - request.path_info, + "%s%s" % (request.script_name, request.path_info), controller, action, response.status_int, -- cgit From ceccffaab6fb5fce3b0951b5a8eea65f523e8563 Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Fri, 4 Mar 2011 19:13:27 -0500 Subject: apirequest -> apireq. --- nova/api/ec2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 2493adc95..b06289e67 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -62,7 +62,7 @@ class RequestLogging(wsgi.Middleware): def log_request_completion(self, response, request, start): apireq = request.environ.get('ec2.request', None) - if apirequest: + if apireq: controller = apireq.controller action = apireq.action else: -- cgit From 6b95c5133452ae26da2cb7f08267aa4cb056e7af Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 10 Mar 2011 15:05:04 -0500 Subject: Initial support fo extension resources. Tests. --- nova/api/openstack/__init__.py | 8 ++- nova/api/openstack/extensions.py | 29 ++++++++++ nova/tests/api/openstack/test_extensions.py | 83 +++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 nova/api/openstack/extensions.py create mode 100644 nova/tests/api/openstack/test_extensions.py diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index ab9dbb780..28e2a1691 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -30,6 +30,7 @@ from nova import wsgi from nova.api.openstack import faults from nova.api.openstack import backup_schedules from nova.api.openstack import consoles +from nova.api.openstack import extensions from nova.api.openstack import flavors from nova.api.openstack import images from nova.api.openstack import servers @@ -68,7 +69,7 @@ class APIRouter(wsgi.Router): """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" return cls() - def __init__(self): + def __init__(self, ext_manager=None): mapper = routes.Mapper() server_members = {'action': 'POST'} @@ -111,6 +112,11 @@ class APIRouter(wsgi.Router): collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) + if ext_manager is None: + ext_manager = extensions.ExtensionManager() + for resource in ext_manager.get_resources(): + resource.add_routes(mapper) + super(APIRouter, self).__init__(mapper) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py new file mode 100644 index 000000000..1c539c500 --- /dev/null +++ b/nova/api/openstack/extensions.py @@ -0,0 +1,29 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +class ExtensionManager(object): + + def get_resources(self): + """ + returns a list of ExtensionResource objects + """ + return [] + +class ExtensionResource(object): + + def add_routes(self, mapper): + pass diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py new file mode 100644 index 000000000..f5332c84a --- /dev/null +++ b/nova/tests/api/openstack/test_extensions.py @@ -0,0 +1,83 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +import webob + +from nova.api import openstack +import nova.wsgi + +class StubController(nova.wsgi.Controller): + + def __init__(self, body): + self.body = body + + def index(self, req): + return self.body + +class StubExtensionManager(object): + + def __init__(self, resources): + self.resources = resources + + def get_resources(self): + return self.resources + +class WidgetExtensionResource(object): + + def __init__(self, name, collection, wsgi_app): + self.name = name + self.collection = collection + self.wsgi_app = wsgi_app + + def add_routes(self, mapper): + mapper.resource(self.name, self.collection, controller=self.wsgi_app) + +class ExtensionTest(unittest.TestCase): + + def test_no_extension_present(self): + manager = StubExtensionManager([]) + router = openstack.APIRouter(manager) + request = webob.Request.blank("/widgets") + response = request.get_response(router) + self.assertEqual(404, response.status_int) + + def test_get_resources(self): + response_body = "Buy more widgets!" + response = webob.Response() + response.body = response_body + resource1 = WidgetExtensionResource("widget", "widgets", response) + manager = StubExtensionManager([resource1]) + router = openstack.APIRouter(manager) + request = webob.Request.blank("/widgets") + response = request.get_response(router) + self.assertEqual(200, response.status_int) + self.assertEqual(response_body, response.body) + + def test_get_resources_with_controller(self): + response_body = "Buy more widgets!" + controller = StubController(response_body) + resource1 = WidgetExtensionResource("widget", "widgets", controller) + manager = StubExtensionManager([resource1]) + router = openstack.APIRouter(manager) + request = webob.Request.blank("/widgets") + response = request.get_response(router) + self.assertEqual(200, response.status_int) + self.assertEqual(response_body, response.body) + + -- cgit From 7d4eae131f2f844f368aa5ff79c68191756775b6 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 11 Mar 2011 14:33:12 -0500 Subject: Add config for osapi_extensions_path. Update the ExtensionManager so that it loads extensions in the osapi_extensions_path. --- CA/openssl.cnf.tmpl | 2 +- nova/api/openstack/__init__.py | 9 ++++---- nova/api/openstack/extensions.py | 32 +++++++++++++++++++++++++- nova/flags.py | 2 ++ nova/tests/api/openstack/extensions/widgets.py | 27 ++++++++++++++++++++++ nova/tests/api/openstack/test_extensions.py | 19 +++++++++++++++ 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 nova/tests/api/openstack/extensions/widgets.py diff --git a/CA/openssl.cnf.tmpl b/CA/openssl.cnf.tmpl index dd81f1c2b..cf8bac828 100644 --- a/CA/openssl.cnf.tmpl +++ b/CA/openssl.cnf.tmpl @@ -43,7 +43,7 @@ policy = policy_match [ policy_match ] countryName = match -stateOrProvinceName = match +stateOrProvinceName = optional organizationName = optional organizationalUnitName = optional commonName = supplied diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 28e2a1691..8f6076511 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -69,7 +69,7 @@ class APIRouter(wsgi.Router): """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" return cls() - def __init__(self, ext_manager=None): + def __init__(self, ext_mgr=None): mapper = routes.Mapper() server_members = {'action': 'POST'} @@ -112,9 +112,10 @@ class APIRouter(wsgi.Router): collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) - if ext_manager is None: - ext_manager = extensions.ExtensionManager() - for resource in ext_manager.get_resources(): + if ext_mgr is None: + ext_mgr = extensions.ExtensionManager(FLAGS.osapi_extensions_path) + for resource in ext_mgr.get_resources(): + print resource resource.add_routes(mapper) super(APIRouter, self).__init__(mapper) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 1c539c500..24846d9cd 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -15,15 +15,45 @@ # License for the specific language governing permissions and limitations # under the License. +import imp +import os +import sys + + class ExtensionManager(object): + def __init__(self, path): + + self.path = path + self.extensions = [] + self._load_extensions() + def get_resources(self): """ returns a list of ExtensionResource objects """ - return [] + resources = [] + for ext in self.extensions: + resources.append(ext.get_resources()) + return resources + + def _load_extensions(self): + if not os.path.exists(self.path): + return + + for f in os.listdir(self.path): + mod_name, file_ext = os.path.splitext(os.path.split(f)[-1]) + ext_path = os.path.join(self.path, f) + if file_ext.lower() == '.py': + mod = imp.load_source(mod_name, ext_path) + self.extensions.append(getattr(mod, 'get_extension')()) + class ExtensionResource(object): + """ + Example ExtensionResource object. All ExtensionResource objects should + adhere to this interface. + """ def add_routes(self, mapper): pass diff --git a/nova/flags.py b/nova/flags.py index 9123e9ac7..63e4cb5b6 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -298,6 +298,8 @@ DEFINE_string('ec2_dmz_host', '$my_ip', 'internal ip of api server') DEFINE_integer('ec2_port', 8773, 'cloud controller port') DEFINE_string('ec2_scheme', 'http', 'prefix for ec2') DEFINE_string('ec2_path', '/services/Cloud', 'suffix for ec2') +DEFINE_string('osapi_extensions_path', '/var/lib/nova/extensions', + 'default directory for nova extensions') DEFINE_string('osapi_host', '$my_ip', 'ip of api server') DEFINE_string('osapi_scheme', 'http', 'prefix for openstack') DEFINE_integer('osapi_port', 8774, 'OpenStack API port') diff --git a/nova/tests/api/openstack/extensions/widgets.py b/nova/tests/api/openstack/extensions/widgets.py new file mode 100644 index 000000000..bc0947223 --- /dev/null +++ b/nova/tests/api/openstack/extensions/widgets.py @@ -0,0 +1,27 @@ +from nova import wsgi + +class WidgetController(wsgi.Controller): + + def index(self, req): + return "Buy more widgets!" + +class WidgetExtensionResource(object): + + def __init__(self): + pass + + def add_routes(self, mapper): + mapper.resource('widget', 'widgets', controller=WidgetController()) + + +class WidgetExtension(object): + + def __init__(self): + pass + + def get_resources(self): + return WidgetExtensionResource() + + +def get_extension(): + return WidgetExtension() diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index f5332c84a..ff41d6d99 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -16,12 +16,16 @@ # under the License. import unittest +import os.path import webob +from nova import flags from nova.api import openstack import nova.wsgi +FLAGS = flags.FLAGS + class StubController(nova.wsgi.Controller): def __init__(self, body): @@ -81,3 +85,18 @@ class ExtensionTest(unittest.TestCase): self.assertEqual(response_body, response.body) +class ExtensionManagerTest(unittest.TestCase): + + def setUp(self): + FLAGS.osapi_extensions_path = os.path.join(os.path.dirname(__file__), + "extensions") + + def test_get_resources(self): + router = openstack.APIRouter() + request = webob.Request.blank("/widgets") + response = request.get_response(router) + self.assertEqual(200, response.status_int) + self.assertEqual("Buy more widgets!", response.body) + + + -- cgit From 2bfa7b29c7882da559041cea771b9243555828fa Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Sun, 13 Mar 2011 13:51:42 -0400 Subject: The extension name is constructed from the camel cased module_name + 'Extension'. --- nova/api/openstack/extensions.py | 9 ++++++++- nova/tests/api/openstack/extensions/widgets.py | 14 +++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 24846d9cd..13789863b 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -38,6 +38,12 @@ class ExtensionManager(object): return resources def _load_extensions(self): + """ + Load extensions from the configured path. The extension name is + constructed from the camel cased module_name + 'Extension'. If your + extension module was named widgets.py the extension class within that + module should be 'WidgetsExtension'. + """ if not os.path.exists(self.path): return @@ -46,7 +52,8 @@ class ExtensionManager(object): ext_path = os.path.join(self.path, f) if file_ext.lower() == '.py': mod = imp.load_source(mod_name, ext_path) - self.extensions.append(getattr(mod, 'get_extension')()) + ext_name = mod_name[0].upper() + mod_name[1:] + 'Extension' + self.extensions.append(getattr(mod, ext_name)()) class ExtensionResource(object): diff --git a/nova/tests/api/openstack/extensions/widgets.py b/nova/tests/api/openstack/extensions/widgets.py index bc0947223..e03fc7776 100644 --- a/nova/tests/api/openstack/extensions/widgets.py +++ b/nova/tests/api/openstack/extensions/widgets.py @@ -1,27 +1,23 @@ from nova import wsgi -class WidgetController(wsgi.Controller): +class WidgetsController(wsgi.Controller): def index(self, req): return "Buy more widgets!" -class WidgetExtensionResource(object): +class WidgetsExtensionResource(object): def __init__(self): pass def add_routes(self, mapper): - mapper.resource('widget', 'widgets', controller=WidgetController()) + mapper.resource('widget', 'widgets', controller=WidgetsController()) -class WidgetExtension(object): +class WidgetsExtension(object): def __init__(self): pass def get_resources(self): - return WidgetExtensionResource() - - -def get_extension(): - return WidgetExtension() + return WidgetsExtensionResource() -- cgit From 229c5bc3324d5df39ca959d71a540a806bc5ad3e Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 14 Mar 2011 16:58:03 -0400 Subject: Implement action extensions. --- etc/api-paste.ini | 5 +- nova/api/openstack/__init__.py | 1 - nova/api/openstack/extensions.py | 106 +++++++++++++++++++++++++ nova/tests/api/openstack/extensions/widgets.py | 22 +++++ nova/tests/api/openstack/test_extensions.py | 44 +++++++++- 5 files changed, 174 insertions(+), 4 deletions(-) diff --git a/etc/api-paste.ini b/etc/api-paste.ini index 9f7e93d4c..c7fa62fca 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -70,7 +70,7 @@ use = egg:Paste#urlmap /v1.0: openstackapi [pipeline:openstackapi] -pipeline = faultwrap auth ratelimit osapiapp +pipeline = faultwrap auth ratelimit extensions osapiapp [filter:faultwrap] paste.filter_factory = nova.api.openstack:FaultWrapper.factory @@ -81,6 +81,9 @@ paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory [filter:ratelimit] paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory +[filter:extensions] +paste.filter_factory = nova.api.openstack.extensions:ExtensionsRouter.factory + [app:osapiapp] paste.app_factory = nova.api.openstack:APIRouter.factory diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 9b7b76a91..8a458eea1 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -124,7 +124,6 @@ class APIRouter(wsgi.Router): if ext_mgr is None: ext_mgr = extensions.ExtensionManager(FLAGS.osapi_extensions_path) for resource in ext_mgr.get_resources(): - print resource resource.add_routes(mapper) super(APIRouter, self).__init__(mapper) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 13789863b..e41de3120 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -18,11 +18,101 @@ import imp import os import sys +import routes +import webob.dec +import webob.exc + +from nova import flags +from nova import log as logging +from nova import wsgi + + +LOG = logging.getLogger('extensions') + + +FLAGS = flags.FLAGS + + +class ExtensionActionController(wsgi.Controller): + + def __init__(self, application, action_name, handler): + + self.application = application + self.action_name = action_name + self.handler = handler + + def action(self, req, id): + + input_dict = self._deserialize(req.body, req.get_content_type()) + if self.action_name in input_dict: + return self.handler(input_dict, req, id) + # no action handler found (bump to downstream application) + res = self.application + return res + + +class ExtensionMiddleware(wsgi.Middleware): + """ + Extensions middleware that intercepts configured routes for extensions. + """ + @classmethod + def factory(cls, global_config, **local_config): + """ paste factory """ + def _factory(app): + return cls(app, **local_config) + return _factory + + def __init__(self, application, ext_mgr=None): + mapper = routes.Mapper() + + if ext_mgr is None: + ext_mgr = ExtensionManager(FLAGS.osapi_extensions_path) + + # create custom mapper connections for extended actions + for action in ext_mgr.get_actions(): + controller = ExtensionActionController(application, action.name, + action.handler) + mapper.connect("/%s/{id}/action.:(format)" % action.collection, + action='action', + controller=controller, + conditions=dict(method=['POST'])) + mapper.connect("/%s/{id}/action" % action.collection, + action='action', + controller=controller, + conditions=dict(method=['POST'])) + + self._router = routes.middleware.RoutesMiddleware(self._dispatch, + mapper) + + super(ExtensionMiddleware, self).__init__(application) + + @webob.dec.wsgify(RequestClass=wsgi.Request) + def __call__(self, req): + """ + Route the incoming request with router. + """ + req.environ['extended.app'] = self.application + return self._router + + @staticmethod + @webob.dec.wsgify(RequestClass=wsgi.Request) + def _dispatch(req): + """ + Called by self._router after matching the incoming request to a route + and putting the information into req.environ. Either returns the + routed WSGI app's response or defers to the extended application. + """ + match = req.environ['wsgiorg.routing_args'][1] + if not match: + return req.environ['extended.app'] + app = match['controller'] + return app class ExtensionManager(object): def __init__(self, path): + LOG.audit(_('Initializing extension manager.')) self.path = path self.extensions = [] @@ -37,6 +127,12 @@ class ExtensionManager(object): resources.append(ext.get_resources()) return resources + def get_actions(self): + actions = [] + for ext in self.extensions: + actions.extend(ext.get_actions()) + return actions + def _load_extensions(self): """ Load extensions from the configured path. The extension name is @@ -48,6 +144,7 @@ class ExtensionManager(object): return for f in os.listdir(self.path): + LOG.audit(_('Loading extension file: %s'), f) mod_name, file_ext = os.path.splitext(os.path.split(f)[-1]) ext_path = os.path.join(self.path, f) if file_ext.lower() == '.py': @@ -56,6 +153,15 @@ class ExtensionManager(object): self.extensions.append(getattr(mod, ext_name)()) +class ExtensionAction(object): + + def __init__(self, member, collection, name, handler): + self.member = member + self.collection = collection + self.name = name + self.handler = handler + + class ExtensionResource(object): """ Example ExtensionResource object. All ExtensionResource objects should diff --git a/nova/tests/api/openstack/extensions/widgets.py b/nova/tests/api/openstack/extensions/widgets.py index e03fc7776..6839d2bb2 100644 --- a/nova/tests/api/openstack/extensions/widgets.py +++ b/nova/tests/api/openstack/extensions/widgets.py @@ -1,10 +1,14 @@ from nova import wsgi +from nova.api.openstack import extensions + + class WidgetsController(wsgi.Controller): def index(self, req): return "Buy more widgets!" + class WidgetsExtensionResource(object): def __init__(self): @@ -21,3 +25,21 @@ class WidgetsExtension(object): def get_resources(self): return WidgetsExtensionResource() + + def get_actions(self): + actions = [] + actions.append(extensions.ExtensionAction('server', 'servers', + 'add_widget', + self._add_widget)) + actions.append(extensions.ExtensionAction('server', 'servers', + 'delete_widget', + self._delete_widget)) + return actions + + def _add_widget(self, input_dict, req, id): + + return "Widget Added." + + def _delete_widget(self, input_dict, req, id): + + return "Widget Deleted." diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index ff41d6d99..f8d217e9c 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -15,10 +15,10 @@ # License for the specific language governing permissions and limitations # under the License. +import json import unittest -import os.path - import webob +import os.path from nova import flags from nova.api import openstack @@ -26,6 +26,7 @@ import nova.wsgi FLAGS = flags.FLAGS + class StubController(nova.wsgi.Controller): def __init__(self, body): @@ -34,6 +35,7 @@ class StubController(nova.wsgi.Controller): def index(self, req): return self.body + class StubExtensionManager(object): def __init__(self, resources): @@ -42,6 +44,7 @@ class StubExtensionManager(object): def get_resources(self): return self.resources + class WidgetExtensionResource(object): def __init__(self, name, collection, wsgi_app): @@ -52,6 +55,7 @@ class WidgetExtensionResource(object): def add_routes(self, mapper): mapper.resource(self.name, self.collection, controller=self.wsgi_app) + class ExtensionTest(unittest.TestCase): def test_no_extension_present(self): @@ -99,4 +103,40 @@ class ExtensionManagerTest(unittest.TestCase): self.assertEqual("Buy more widgets!", response.body) +class ExtendedActionTest(unittest.TestCase): + def setUp(self): + FLAGS.osapi_extensions_path = os.path.join(os.path.dirname(__file__), + "extensions") + + def test_extended_action(self): + app = openstack.APIRouter() + ext_midware = openstack.extensions.ExtensionMiddleware(app) + body = dict(add_widget=dict(name="test")) + request = webob.Request.blank("/servers/1/action") + request.method = 'POST' + request.content_type = 'application/json' + request.body = json.dumps(body) + response = request.get_response(ext_midware) + self.assertEqual(200, response.status_int) + self.assertEqual("Widget Added.", response.body) + + def test_invalid_action_body(self): + app = openstack.APIRouter() + ext_midware = openstack.extensions.ExtensionMiddleware(app) + body = dict(blah=dict(name="test")) # Doesn't exist + request = webob.Request.blank("/servers/1/action") + request.method = 'POST' + request.content_type = 'application/json' + request.body = json.dumps(body) + response = request.get_response(ext_midware) + self.assertEqual(501, response.status_int) + + def test_invalid_action(self): + app = openstack.APIRouter() + ext_midware = openstack.extensions.ExtensionMiddleware(app) + request = webob.Request.blank("/asdf/1/action") + request.method = 'POST' + request.content_type = 'application/json' + response = request.get_response(ext_midware) + self.assertEqual(404, response.status_int) -- cgit From f0141b1616e1b1fc9e52e33b37cc3a1091c57587 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 14 Mar 2011 22:24:34 -0400 Subject: Moved extended resource code into the extensions.py module. --- nova/api/openstack/__init__.py | 6 --- nova/api/openstack/extensions.py | 29 +++++++---- nova/tests/api/openstack/extensions/widgets.py | 15 ++---- nova/tests/api/openstack/test_extensions.py | 66 +++++++++++++------------- 4 files changed, 59 insertions(+), 57 deletions(-) diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 8a458eea1..ff91c77cf 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -31,7 +31,6 @@ from nova.api.openstack import accounts from nova.api.openstack import faults from nova.api.openstack import backup_schedules from nova.api.openstack import consoles -from nova.api.openstack import extensions from nova.api.openstack import flavors from nova.api.openstack import images from nova.api.openstack import servers @@ -121,11 +120,6 @@ class APIRouter(wsgi.Router): collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) - if ext_mgr is None: - ext_mgr = extensions.ExtensionManager(FLAGS.osapi_extensions_path) - for resource in ext_mgr.get_resources(): - resource.add_routes(mapper) - super(APIRouter, self).__init__(mapper) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index e41de3120..f32471051 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -68,7 +68,15 @@ class ExtensionMiddleware(wsgi.Middleware): if ext_mgr is None: ext_mgr = ExtensionManager(FLAGS.osapi_extensions_path) - # create custom mapper connections for extended actions + # extended resources + for resource in ext_mgr.get_resources(): + mapper.resource(resource.member, resource.collection, + controller=resource.controller, + collection=resource.collection_actions, + member=resource.member_actions, + parent_resource=resource.parent) + + # extended actions for action in ext_mgr.get_actions(): controller = ExtensionActionController(application, action.name, action.handler) @@ -124,10 +132,13 @@ class ExtensionManager(object): """ resources = [] for ext in self.extensions: - resources.append(ext.get_resources()) + resources.extend(ext.get_resources()) return resources def get_actions(self): + """ + returns a list of ExtensionAction objects + """ actions = [] for ext in self.extensions: actions.extend(ext.get_actions()) @@ -163,10 +174,12 @@ class ExtensionAction(object): class ExtensionResource(object): - """ - Example ExtensionResource object. All ExtensionResource objects should - adhere to this interface. - """ - def add_routes(self, mapper): - pass + def __init__(self, member, collection, controller, + parent=None, collection_actions={}, member_actions={}): + self.member = member + self.collection = collection + self.controller = controller + self.parent = parent + self.collection_actions = collection_actions + self.member_actions = member_actions diff --git a/nova/tests/api/openstack/extensions/widgets.py b/nova/tests/api/openstack/extensions/widgets.py index 6839d2bb2..d5a2d95d9 100644 --- a/nova/tests/api/openstack/extensions/widgets.py +++ b/nova/tests/api/openstack/extensions/widgets.py @@ -9,22 +9,17 @@ class WidgetsController(wsgi.Controller): return "Buy more widgets!" -class WidgetsExtensionResource(object): - - def __init__(self): - pass - - def add_routes(self, mapper): - mapper.resource('widget', 'widgets', controller=WidgetsController()) - - class WidgetsExtension(object): def __init__(self): pass def get_resources(self): - return WidgetsExtensionResource() + resources = [] + widgets = extensions.ExtensionResource('widget', 'widgets', + WidgetsController()) + resources.append(widgets) + return resources def get_actions(self): actions = [] diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index f8d217e9c..080760c14 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -22,6 +22,7 @@ import os.path from nova import flags from nova.api import openstack +from nova.api.openstack import extensions import nova.wsgi FLAGS = flags.FLAGS @@ -38,53 +39,51 @@ class StubController(nova.wsgi.Controller): class StubExtensionManager(object): - def __init__(self, resources): - self.resources = resources + def __init__(self, resource): + self.resource = resource def get_resources(self): - return self.resources + resources = [] + if self.resource: + resources.append(self.resource) + return resources + def get_actions(self): + actions = [] + return actions -class WidgetExtensionResource(object): - def __init__(self, name, collection, wsgi_app): - self.name = name - self.collection = collection - self.wsgi_app = wsgi_app - - def add_routes(self, mapper): - mapper.resource(self.name, self.collection, controller=self.wsgi_app) - - -class ExtensionTest(unittest.TestCase): +class ExtensionResourceTest(unittest.TestCase): def test_no_extension_present(self): - manager = StubExtensionManager([]) - router = openstack.APIRouter(manager) + manager = StubExtensionManager(None) + app = openstack.APIRouter() + ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/widgets") - response = request.get_response(router) + response = request.get_response(ext_midware) self.assertEqual(404, response.status_int) def test_get_resources(self): response_body = "Buy more widgets!" - response = webob.Response() - response.body = response_body - resource1 = WidgetExtensionResource("widget", "widgets", response) - manager = StubExtensionManager([resource1]) - router = openstack.APIRouter(manager) + widgets = extensions.ExtensionResource('widget', 'widgets', + StubController(response_body)) + manager = StubExtensionManager(widgets) + app = openstack.APIRouter() + ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/widgets") - response = request.get_response(router) + response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) self.assertEqual(response_body, response.body) def test_get_resources_with_controller(self): response_body = "Buy more widgets!" - controller = StubController(response_body) - resource1 = WidgetExtensionResource("widget", "widgets", controller) - manager = StubExtensionManager([resource1]) - router = openstack.APIRouter(manager) + widgets = extensions.ExtensionResource('widget', 'widgets', + StubController(response_body)) + manager = StubExtensionManager(widgets) + app = openstack.APIRouter() + ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/widgets") - response = request.get_response(router) + response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) self.assertEqual(response_body, response.body) @@ -96,9 +95,10 @@ class ExtensionManagerTest(unittest.TestCase): "extensions") def test_get_resources(self): - router = openstack.APIRouter() + app = openstack.APIRouter() + ext_midware = extensions.ExtensionMiddleware(app) request = webob.Request.blank("/widgets") - response = request.get_response(router) + response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) self.assertEqual("Buy more widgets!", response.body) @@ -111,7 +111,7 @@ class ExtendedActionTest(unittest.TestCase): def test_extended_action(self): app = openstack.APIRouter() - ext_midware = openstack.extensions.ExtensionMiddleware(app) + ext_midware = extensions.ExtensionMiddleware(app) body = dict(add_widget=dict(name="test")) request = webob.Request.blank("/servers/1/action") request.method = 'POST' @@ -123,7 +123,7 @@ class ExtendedActionTest(unittest.TestCase): def test_invalid_action_body(self): app = openstack.APIRouter() - ext_midware = openstack.extensions.ExtensionMiddleware(app) + ext_midware = extensions.ExtensionMiddleware(app) body = dict(blah=dict(name="test")) # Doesn't exist request = webob.Request.blank("/servers/1/action") request.method = 'POST' @@ -134,7 +134,7 @@ class ExtendedActionTest(unittest.TestCase): def test_invalid_action(self): app = openstack.APIRouter() - ext_midware = openstack.extensions.ExtensionMiddleware(app) + ext_midware = extensions.ExtensionMiddleware(app) request = webob.Request.blank("/asdf/1/action") request.method = 'POST' request.content_type = 'application/json' -- cgit From f91d7925761f8204fdd46435ff57d74ae17483cf Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 15 Mar 2011 18:29:26 -0700 Subject: first pass openstack redirect working --- etc/api-paste.ini | 5 ++++- nova/api/openstack/servers.py | 1 + nova/compute/api.py | 3 ++- nova/compute/manager.py | 6 +++++- nova/exception.py | 6 +++--- nova/scheduler/api.py | 13 +++++++++++-- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/etc/api-paste.ini b/etc/api-paste.ini index 9f7e93d4c..c0077939b 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -70,7 +70,10 @@ use = egg:Paste#urlmap /v1.0: openstackapi [pipeline:openstackapi] -pipeline = faultwrap auth ratelimit osapiapp +pipeline = faultwrap zonerouter auth ratelimit osapiapp + +[filter:zonerouter] +paste.filter_factory = nova.api.zone_redirect:ZoneRedirectMiddleware.factory [filter:faultwrap] paste.filter_factory = nova.api.openstack:FaultWrapper.factory diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 85999764f..ffcbe628c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -343,6 +343,7 @@ class Controller(wsgi.Controller): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: + LOG.debug(_("*** Compute.api::pause %s"), id) self.compute_api.pause(ctxt, id) except: readable = traceback.format_exc() diff --git a/nova/compute/api.py b/nova/compute/api.py index 8865a1654..1185b9964 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -347,7 +347,8 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - rv = self.db.instance_get(context, instance_id) + rv = self.scheduler_api.get_instance_or_reroute(context, instance_id) + #rv = self.db.instance_get(context, instance_id) return dict(rv.iteritems()) def get_all(self, context, project_id=None, reservation_id=None, diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ebe1ce6f0..499b212e2 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -48,6 +48,7 @@ from nova import scheduler_manager from nova import rpc from nova import utils from nova.compute import power_state +from nova.scheduler import api as scheduler_api FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', @@ -521,7 +522,10 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): def pause_instance(self, context, instance_id): """Pause an instance on this server.""" context = context.elevated() - instance_ref = self.db.instance_get(context, instance_id) + LOG.debug(_('*** instance %s: starting pause'), instance_id) + instance_ref = scheduler_api.get_instance_or_reroute(context, + instance_id) + #instance_ref = self.db.instance_get(context, instance_id) LOG.audit(_('instance %s: pausing'), instance_id, context=context) self.db.instance_set_state(context, instance_id, diff --git a/nova/exception.py b/nova/exception.py index ce8daf048..d0baa2e29 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -93,9 +93,9 @@ class TimeoutException(Error): class ZoneRouteException(Error): - def __init__(self, zone, *args, **kwargs): - self.zone = zone - super(ZoneRouteException, self).__init__(args, kwargs) + def __init__(self, zones, *args, **kwargs): + self.zones = zones + super(ZoneRouteException, self).__init__(*args, **kwargs) class DBError(Error): diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 8f9806f77..c0e28a0a9 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -73,13 +73,22 @@ class API(object): args=dict(service_name=service_name, host=host, capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) + + @classmethod + def get_instance_or_reroute(cls, context, instance_id): + instance = db.instance_get(context, instance_id) + zones = db.zone_get_all(context) + + LOG.debug("*** Firing ZoneRouteException") + # Throw a reroute Exception for the middleware to pick up. + raise exception.ZoneRouteException(zones) @classmethod - def get_queue_for_instance(cls, context, service, instance_id) + def get_queue_for_instance(cls, context, service, instance_id): instance = db.instance_get(context, instance_id) zone = db.get_zone(instance.zone.id) if cls._is_current_zone(zone): - return db.queue_get_for(context, service, instance['host']): + return db.queue_get_for(context, service, instance['host']) # Throw a reroute Exception for the middleware to pick up. raise exception.ZoneRouteException(zone) -- cgit From 0053b776276e9cac617c812931c248be7e49fea2 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Tue, 15 Mar 2011 23:00:09 -0400 Subject: Add ResponseExtensions. --- nova/api/openstack/extensions.py | 144 ++++++++++++++++++++----- nova/tests/api/openstack/extensions/widgets.py | 26 +++-- nova/tests/api/openstack/test_extensions.py | 103 +++++++++++++----- 3 files changed, 213 insertions(+), 60 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index f32471051..8b8806c8a 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -33,24 +33,43 @@ LOG = logging.getLogger('extensions') FLAGS = flags.FLAGS -class ExtensionActionController(wsgi.Controller): +class ActionExtensionController(wsgi.Controller): - def __init__(self, application, action_name, handler): + def __init__(self, application): self.application = application - self.action_name = action_name - self.handler = handler + self.action_handlers = {} + + def add_action(self, action_name, handler): + self.action_handlers[action_name] = handler def action(self, req, id): input_dict = self._deserialize(req.body, req.get_content_type()) - if self.action_name in input_dict: - return self.handler(input_dict, req, id) + for action_name, handler in self.action_handlers.iteritems(): + if action_name in input_dict: + return handler(input_dict, req, id) # no action handler found (bump to downstream application) res = self.application return res +class ResponseExtensionController(wsgi.Controller): + + def __init__(self, application): + self.application = application + self.handlers = [] + + def add_handler(self, handler): + self.handlers.append(handler) + + def process(self, req, *args, **kwargs): + res = req.get_response(self.application) + # currently response handlers are un-ordered + for handler in self.handlers: + return handler(res) + + class ExtensionMiddleware(wsgi.Middleware): """ Extensions middleware that intercepts configured routes for extensions. @@ -62,33 +81,80 @@ class ExtensionMiddleware(wsgi.Middleware): return cls(app, **local_config) return _factory + def _actions_by_collection(self, application, ext_mgr): + """ + Return a dict of ActionExtensionController objects by collection + """ + action_controllers = {} + for action in ext_mgr.get_actions(): + if not action.collection in action_controllers.keys(): + controller = ActionExtensionController(application) + action_controllers[action.collection] = controller + return action_controllers + + def _responses_by_collection(self, application, ext_mgr): + """ + Return a dict of ResponseExtensionController objects by collection + """ + response_ext_controllers = {} + for resp_ext in ext_mgr.get_response_extensions(): + if not resp_ext.url_route in response_ext_controllers.keys(): + controller = ResponseExtensionController(application) + response_ext_controllers[resp_ext.url_route] = controller + return response_ext_controllers + def __init__(self, application, ext_mgr=None): - mapper = routes.Mapper() if ext_mgr is None: ext_mgr = ExtensionManager(FLAGS.osapi_extensions_path) + self.ext_mgr = ext_mgr + + mapper = routes.Mapper() # extended resources for resource in ext_mgr.get_resources(): - mapper.resource(resource.member, resource.collection, + LOG.debug(_('Extended resource: %s'), + resource.collection) + mapper.resource(resource.collection, resource.collection, controller=resource.controller, collection=resource.collection_actions, member=resource.member_actions, parent_resource=resource.parent) # extended actions + action_controllers = self._actions_by_collection(application, ext_mgr) for action in ext_mgr.get_actions(): - controller = ExtensionActionController(application, action.name, - action.handler) - mapper.connect("/%s/{id}/action.:(format)" % action.collection, + LOG.debug(_('Extended collection/action: %s/%s'), + action.collection, + action.action_name) + controller = action_controllers[action.collection] + controller.add_action(action.action_name, action.handler) + + mapper.connect("/%s/:(id)/action.:(format)" % action.collection, action='action', controller=controller, conditions=dict(method=['POST'])) - mapper.connect("/%s/{id}/action" % action.collection, + mapper.connect("/%s/:(id)/action" % action.collection, action='action', controller=controller, conditions=dict(method=['POST'])) + # extended responses + resp_controllers = self._responses_by_collection(application, ext_mgr) + for response_ext in ext_mgr.get_response_extensions(): + LOG.debug(_('Extended response: %s'), response_ext.url_route) + controller = resp_controllers[response_ext.url_route] + controller.add_handler(response_ext.handler) + mapper.connect(response_ext.url_route + '.:(format)', + action='process', + controller=controller, + conditions=response_ext.conditions) + + mapper.connect(response_ext.url_route, + action='process', + controller=controller, + conditions=response_ext.conditions) + self._router = routes.middleware.RoutesMiddleware(self._dispatch, mapper) @@ -106,9 +172,8 @@ class ExtensionMiddleware(wsgi.Middleware): @webob.dec.wsgify(RequestClass=wsgi.Request) def _dispatch(req): """ - Called by self._router after matching the incoming request to a route - and putting the information into req.environ. Either returns the - routed WSGI app's response or defers to the extended application. + Returns the routed WSGI app's response or defers to the extended + application. """ match = req.environ['wsgiorg.routing_args'][1] if not match: @@ -128,7 +193,7 @@ class ExtensionManager(object): def get_resources(self): """ - returns a list of ExtensionResource objects + returns a list of ResourceExtension objects """ resources = [] for ext in self.extensions: @@ -137,13 +202,22 @@ class ExtensionManager(object): def get_actions(self): """ - returns a list of ExtensionAction objects + returns a list of ActionExtension objects """ actions = [] for ext in self.extensions: actions.extend(ext.get_actions()) return actions + def get_response_extensions(self): + """ + returns a list of ResponseExtension objects + """ + response_exts = [] + for ext in self.extensions: + response_exts.extend(ext.get_response_extensions()) + return response_exts + def _load_extensions(self): """ Load extensions from the configured path. The extension name is @@ -160,24 +234,42 @@ class ExtensionManager(object): ext_path = os.path.join(self.path, f) if file_ext.lower() == '.py': mod = imp.load_source(mod_name, ext_path) - ext_name = mod_name[0].upper() + mod_name[1:] + 'Extension' + ext_name = mod_name[0].upper() + mod_name[1:] self.extensions.append(getattr(mod, ext_name)()) -class ExtensionAction(object): +class ResponseExtension(object): + """ + ResponseExtension objects can be used to add data to responses from + core nova OpenStack API controllers. + """ - def __init__(self, member, collection, name, handler): - self.member = member + def __init__(self, url_route, method, handler): + self.url_route = url_route + self.conditions = dict(method=[method]) + self.handler = handler + + +class ActionExtension(object): + """ + ActionExtension objects can be used to add custom actions to core nova + nova OpenStack API controllers. + """ + + def __init__(self, collection, action_name, handler): self.collection = collection - self.name = name + self.action_name = action_name self.handler = handler -class ExtensionResource(object): +class ResourceExtension(object): + """ + ResourceExtension objects can be used to add add top level resources + to the OpenStack API in nova. + """ - def __init__(self, member, collection, controller, - parent=None, collection_actions={}, member_actions={}): - self.member = member + def __init__(self, collection, controller, parent=None, + collection_actions={}, member_actions={}): self.collection = collection self.controller = controller self.parent = parent diff --git a/nova/tests/api/openstack/extensions/widgets.py b/nova/tests/api/openstack/extensions/widgets.py index d5a2d95d9..f463721f1 100644 --- a/nova/tests/api/openstack/extensions/widgets.py +++ b/nova/tests/api/openstack/extensions/widgets.py @@ -1,3 +1,5 @@ +import json + from nova import wsgi from nova.api.openstack import extensions @@ -9,28 +11,40 @@ class WidgetsController(wsgi.Controller): return "Buy more widgets!" -class WidgetsExtension(object): +class Widgets(object): def __init__(self): pass def get_resources(self): resources = [] - widgets = extensions.ExtensionResource('widget', 'widgets', + widgets = extensions.ResourceExtension('widgets', WidgetsController()) resources.append(widgets) return resources def get_actions(self): actions = [] - actions.append(extensions.ExtensionAction('server', 'servers', - 'add_widget', + actions.append(extensions.ActionExtension('servers', 'add_widget', self._add_widget)) - actions.append(extensions.ExtensionAction('server', 'servers', - 'delete_widget', + actions.append(extensions.ActionExtension('servers', 'delete_widget', self._delete_widget)) return actions + def get_response_extensions(self): + response_exts = [] + + def _resp_handler(res): + # only handle JSON responses + data = json.loads(res.body) + data['flavor']['widgets'] = "Buy more widgets!" + return data + + widgets = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', + _resp_handler) + response_exts.append(widgets) + return response_exts + def _add_widget(self, input_dict, req, id): return "Widget Added." diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 080760c14..8725c8f0e 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -16,13 +16,17 @@ # under the License. import json +import stubout import unittest import webob import os.path +from nova import context from nova import flags from nova.api import openstack from nova.api.openstack import extensions +from nova.api.openstack import flavors +from nova.tests.api.openstack import fakes import nova.wsgi FLAGS = flags.FLAGS @@ -39,21 +43,31 @@ class StubController(nova.wsgi.Controller): class StubExtensionManager(object): - def __init__(self, resource): - self.resource = resource + def __init__(self, resource_ext=None, action_ext=None, response_ext=None): + self.resource_ext = resource_ext + self.action_ext = action_ext + self.response_ext = response_ext def get_resources(self): - resources = [] - if self.resource: - resources.append(self.resource) - return resources + resource_exts = [] + if self.resource_ext: + resource_exts.append(self.resource_ext) + return resource_exts def get_actions(self): - actions = [] - return actions + action_exts = [] + if self.action_ext: + action_exts.append(self.action_ext) + return action_exts + def get_response_extensions(self): + response_exts = [] + if self.response_ext: + response_exts.append(self.response_ext) + return response_exts -class ExtensionResourceTest(unittest.TestCase): + +class ResourceExtensionTest(unittest.TestCase): def test_no_extension_present(self): manager = StubExtensionManager(None) @@ -65,7 +79,7 @@ class ExtensionResourceTest(unittest.TestCase): def test_get_resources(self): response_body = "Buy more widgets!" - widgets = extensions.ExtensionResource('widget', 'widgets', + widgets = extensions.ResourceExtension('widgets', StubController(response_body)) manager = StubExtensionManager(widgets) app = openstack.APIRouter() @@ -77,7 +91,7 @@ class ExtensionResourceTest(unittest.TestCase): def test_get_resources_with_controller(self): response_body = "Buy more widgets!" - widgets = extensions.ExtensionResource('widget', 'widgets', + widgets = extensions.ResourceExtension('widgets', StubController(response_body)) manager = StubExtensionManager(widgets) app = openstack.APIRouter() @@ -103,40 +117,73 @@ class ExtensionManagerTest(unittest.TestCase): self.assertEqual("Buy more widgets!", response.body) -class ExtendedActionTest(unittest.TestCase): +class ActionExtensionTest(unittest.TestCase): def setUp(self): FLAGS.osapi_extensions_path = os.path.join(os.path.dirname(__file__), "extensions") - def test_extended_action(self): + def _send_server_action_request(self, url, body): app = openstack.APIRouter() ext_midware = extensions.ExtensionMiddleware(app) - body = dict(add_widget=dict(name="test")) - request = webob.Request.blank("/servers/1/action") + request = webob.Request.blank(url) request.method = 'POST' request.content_type = 'application/json' request.body = json.dumps(body) response = request.get_response(ext_midware) + return response + + def test_extended_action(self): + body = dict(add_widget=dict(name="test")) + response = self._send_server_action_request("/servers/1/action", body) self.assertEqual(200, response.status_int) self.assertEqual("Widget Added.", response.body) + body = dict(delete_widget=dict(name="test")) + response = self._send_server_action_request("/servers/1/action", body) + self.assertEqual(200, response.status_int) + self.assertEqual("Widget Deleted.", response.body) + def test_invalid_action_body(self): - app = openstack.APIRouter() - ext_midware = extensions.ExtensionMiddleware(app) body = dict(blah=dict(name="test")) # Doesn't exist - request = webob.Request.blank("/servers/1/action") - request.method = 'POST' - request.content_type = 'application/json' - request.body = json.dumps(body) - response = request.get_response(ext_midware) + response = self._send_server_action_request("/servers/1/action", body) self.assertEqual(501, response.status_int) def test_invalid_action(self): - app = openstack.APIRouter() - ext_midware = extensions.ExtensionMiddleware(app) - request = webob.Request.blank("/asdf/1/action") - request.method = 'POST' - request.content_type = 'application/json' - response = request.get_response(ext_midware) + body = dict(blah=dict(name="test")) + response = self._send_server_action_request("/asdf/1/action", body) self.assertEqual(404, response.status_int) + + +class ResponseExtensionTest(unittest.TestCase): + + def setUp(self): + super(ResponseExtensionTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + fakes.FakeAuthManager.reset_fake_data() + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + self.context = context.get_admin_context() + + def test_get_resources(self): + + test_resp = "Buy more widgets!" + + def _resp_handler(res): + # only handle JSON responses + data = json.loads(res.body) + data['flavor']['widgets'] = test_resp + return data + + widgets = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', + _resp_handler) + manager = StubExtensionManager(None, None, widgets) + app = fakes.wsgi_app() + ext_midware = extensions.ExtensionMiddleware(app, manager) + request = webob.Request.blank("/v1.0/flavors/1") + response = request.get_response(ext_midware) + self.assertEqual(200, response.status_int) + response_data = json.loads(response.body) + self.assertEqual(test_resp, response_data['flavor']['widgets']) -- cgit From 7a61965908ebfc076ad3b1d9cdc5773ade50bf75 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 15 Mar 2011 20:30:27 -0700 Subject: response working --- nova/api/zone_redirect.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 nova/api/zone_redirect.py diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py new file mode 100644 index 000000000..5e40a82dd --- /dev/null +++ b/nova/api/zone_redirect.py @@ -0,0 +1,100 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.import datetime + +"""Reroutes calls to child zones on ZoneRouteException's.""" + +import httplib +import re +import webob +import webob.dec +import webob.exc +import urlparse +import urllib + +from nova import exception +from nova import log as logging +from nova import wsgi + +import novaclient.client as client + +try: + import json +except ImportError: + import simplejson as json + + +LOG = logging.getLogger('server') + + +class ZoneRedirectMiddleware(wsgi.Middleware): + """Catches Zone Routing exceptions and delegates the call + to child zones.""" + + @webob.dec.wsgify + def __call__(self, req): + try: + return req.get_response(self.application) + except exception.ZoneRouteException as e: + if len(e.zones) == 0: + exc = webob.exc.HTTPInternalServerError(explanation= + _("No zones to reroute to.")) + return faults.Fault(exc) + + zone = e.zones[0] + # Todo(sandy): This only works for OpenStack API currently. + # Needs to be broken out into a driver. + url = zone.api_url + LOG.info(_("Zone redirect to:[url:%(api_url)s, username:%(username)s]" + % dict(api_url=zone.api_url, username=zone.username))) + + LOG.info(_("Zone Initial Req: %s"), req) + nova = client.OpenStackClient(zone.username, zone.password, + zone.api_url) + nova.authenticate() + new_req = req.copy() + #m = re.search('(https?://.+)/(v\d+\.\d+)/', url) + + scheme, netloc, path, query, frag = urlparse.urlsplit(new_req.path_qs) + query = urlparse.parse_qsl(query) + LOG.debug("**** QUERY=%s^%s^%s", path, query, frag) + query = [(key, value) for key, value in query if key != 'fresh'] + query = urllib.urlencode(query) + url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + + m = re.search('/(v\d+\.\d+)/(.+)', url) + version = m.group(1) + resource = m.group(2) + + LOG.info(_("New Request Data: %s"), new_req.body) + #LOG.info(_("New Request Headers: %s"), new_req.headers) + LOG.info(_("New Request Path: %s"), resource) + if req.method == 'GET': + response, body = nova.get(resource, body=new_req.body) + elif req.method == 'POST': + response, body = nova.post(resource, body=new_req.body) + elif req.method == 'PUT': + response, body = nova.put(resource, body=new_req.body) + elif req.method == 'DELETE': + response, body = nova.delete(resource, body=new_req.body) + #response, body = nova.request(req.path_qs, headers=new_req.headers, body=new_req.body) + LOG.info(_("Zone Response: %s / %s"), response, body) + res = webob.Response() + res.status = response['status'] + res.content_type = response['content-type'] + res.body = json.dumps(body) + LOG.info(_("Zone WebOb Response: %s"), res) + return res -- cgit From ddeede35a5036aa3c80742fde69468aedbc74892 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 06:09:00 -0700 Subject: Error codes handled properly now --- nova/api/zone_redirect.py | 94 ++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 5e40a82dd..ad47a6216 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -30,6 +30,7 @@ from nova import log as logging from nova import wsgi import novaclient.client as client +import novaclient.exceptions as osexceptions try: import json @@ -49,52 +50,63 @@ class ZoneRedirectMiddleware(wsgi.Middleware): try: return req.get_response(self.application) except exception.ZoneRouteException as e: - if len(e.zones) == 0: + if not e.zones: exc = webob.exc.HTTPInternalServerError(explanation= _("No zones to reroute to.")) return faults.Fault(exc) - zone = e.zones[0] # Todo(sandy): This only works for OpenStack API currently. # Needs to be broken out into a driver. - url = zone.api_url - LOG.info(_("Zone redirect to:[url:%(api_url)s, username:%(username)s]" - % dict(api_url=zone.api_url, username=zone.username))) - - LOG.info(_("Zone Initial Req: %s"), req) - nova = client.OpenStackClient(zone.username, zone.password, - zone.api_url) - nova.authenticate() - new_req = req.copy() - #m = re.search('(https?://.+)/(v\d+\.\d+)/', url) - - scheme, netloc, path, query, frag = urlparse.urlsplit(new_req.path_qs) - query = urlparse.parse_qsl(query) - LOG.debug("**** QUERY=%s^%s^%s", path, query, frag) - query = [(key, value) for key, value in query if key != 'fresh'] - query = urllib.urlencode(query) - url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - - m = re.search('/(v\d+\.\d+)/(.+)', url) - version = m.group(1) - resource = m.group(2) - - LOG.info(_("New Request Data: %s"), new_req.body) - #LOG.info(_("New Request Headers: %s"), new_req.headers) - LOG.info(_("New Request Path: %s"), resource) - if req.method == 'GET': - response, body = nova.get(resource, body=new_req.body) - elif req.method == 'POST': - response, body = nova.post(resource, body=new_req.body) - elif req.method == 'PUT': - response, body = nova.put(resource, body=new_req.body) - elif req.method == 'DELETE': - response, body = nova.delete(resource, body=new_req.body) - #response, body = nova.request(req.path_qs, headers=new_req.headers, body=new_req.body) - LOG.info(_("Zone Response: %s / %s"), response, body) + for zone in e.zones: + url = zone.api_url + LOG.info(_("Zone redirect to:[url:%(api_url)s, " + "username:%(username)s]" + % dict(api_url=zone.api_url, + username=zone.username))) + + nova = client.OpenStackClient(zone.username, zone.password, + zone.api_url) + nova.authenticate() + new_req = req.copy() + + scheme, netloc, path, query, frag = \ + urlparse.urlsplit(new_req.path_qs) + query = urlparse.parse_qsl(query) + query = [(key, value) for key, value in query if key != 'fresh'] + query = urllib.urlencode(query) + url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + + m = re.search('/(v\d+\.\d+)/(.+)', url) + version = m.group(1) + resource = m.group(2) + + #LOG.info(_("New Request Data: %s"), new_req.body) + #LOG.info(_("New Request Path: %s"), resource) + try: + if req.method == 'GET': + response, body = nova.get(resource, body=new_req.body) + elif req.method == 'POST': + response, body = nova.post(resource, body=new_req.body) + elif req.method == 'PUT': + response, body = nova.put(resource, body=new_req.body) + elif req.method == 'DELETE': + response, body = nova.delete(resource, + body=new_req.body) + except osexceptions.OpenStackException, e: + LOG.info(_("Zone returned error: %s ('%s', '%s')"), + e.code, e.message, e.details) + continue + + LOG.info(_("Zone Response: %s [%s]/ %s"), response, + response.status, body) + if response.status == 200: + res = webob.Response() + res.status = response['status'] + res.content_type = response['content-type'] + res.body = json.dumps(body) + return res + + LOG.info(_("Returning 404 ...")) res = webob.Response() - res.status = response['status'] - res.content_type = response['content-type'] - res.body = json.dumps(body) - LOG.info(_("Zone WebOb Response: %s"), res) + res.status = "404" return res -- cgit From 8dffae687e78a1fa2a8cf0d321d64ee95a35cc1f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 06:47:27 -0700 Subject: Checks locally before routing --- nova/api/zone_redirect.py | 34 +++++++++++++++------------------- nova/scheduler/api.py | 11 ++++++++--- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index ad47a6216..fec1b1af3 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -57,9 +57,20 @@ class ZoneRedirectMiddleware(wsgi.Middleware): # Todo(sandy): This only works for OpenStack API currently. # Needs to be broken out into a driver. + new_req = req.copy() + + scheme, netloc, path, query, frag = \ + urlparse.urlsplit(new_req.path_qs) + query = urlparse.parse_qsl(query) + query = [(key, value) for key, value in query if key != 'fresh'] + query = urllib.urlencode(query) + url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + + m = re.search('/v\d+\.\d+/(.+)', url) + resource = m.group(1) + for zone in e.zones: - url = zone.api_url - LOG.info(_("Zone redirect to:[url:%(api_url)s, " + LOG.debug(_("Zone redirect to:[url:%(api_url)s, " "username:%(username)s]" % dict(api_url=zone.api_url, username=zone.username))) @@ -67,21 +78,6 @@ class ZoneRedirectMiddleware(wsgi.Middleware): nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() - new_req = req.copy() - - scheme, netloc, path, query, frag = \ - urlparse.urlsplit(new_req.path_qs) - query = urlparse.parse_qsl(query) - query = [(key, value) for key, value in query if key != 'fresh'] - query = urllib.urlencode(query) - url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) - - m = re.search('/(v\d+\.\d+)/(.+)', url) - version = m.group(1) - resource = m.group(2) - - #LOG.info(_("New Request Data: %s"), new_req.body) - #LOG.info(_("New Request Path: %s"), resource) try: if req.method == 'GET': response, body = nova.get(resource, body=new_req.body) @@ -97,7 +93,7 @@ class ZoneRedirectMiddleware(wsgi.Middleware): e.code, e.message, e.details) continue - LOG.info(_("Zone Response: %s [%s]/ %s"), response, + LOG.debug(_("Zone Response: %s [%s]/ %s"), response, response.status, body) if response.status == 200: res = webob.Response() @@ -106,7 +102,7 @@ class ZoneRedirectMiddleware(wsgi.Middleware): res.body = json.dumps(body) return res - LOG.info(_("Returning 404 ...")) + LOG.debug(_("Zone Redirect Middleware returning 404 ...")) res = webob.Response() res.status = "404" return res diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index c0e28a0a9..48da5bcfc 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -76,11 +76,16 @@ class API(object): @classmethod def get_instance_or_reroute(cls, context, instance_id): - instance = db.instance_get(context, instance_id) - zones = db.zone_get_all(context) + try: + instance = db.instance_get(context, instance_id) + return instance + except exception.InstanceNotFound, e: + LOG.debug(_("Instance %(instance_id)s not found locally: '%(e)s'" % + locals())) - LOG.debug("*** Firing ZoneRouteException") # Throw a reroute Exception for the middleware to pick up. + LOG.debug("Firing ZoneRouteException") + zones = db.zone_get_all(context) raise exception.ZoneRouteException(zones) @classmethod -- cgit From d714df5ed4a6a1d4f1c0f7680c2fbb6a6abb81a5 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 16 Mar 2011 11:02:22 -0400 Subject: Implement top level extensions. --- nova/api/openstack/extensions.py | 94 +++++++++++++++++++++-- nova/tests/api/openstack/extensions/foxinsocks.py | 70 +++++++++++++++++ nova/tests/api/openstack/extensions/widgets.py | 54 ------------- nova/tests/api/openstack/test_extensions.py | 68 +++++++++++----- 4 files changed, 204 insertions(+), 82 deletions(-) create mode 100644 nova/tests/api/openstack/extensions/foxinsocks.py delete mode 100644 nova/tests/api/openstack/extensions/widgets.py diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 8b8806c8a..66ddd8078 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -25,6 +25,7 @@ import webob.exc from nova import flags from nova import log as logging from nova import wsgi +from nova.api.openstack import faults LOG = logging.getLogger('extensions') @@ -70,6 +71,42 @@ class ResponseExtensionController(wsgi.Controller): return handler(res) +class ExtensionController(wsgi.Controller): + + def __init__(self, extension_manager): + self.extension_manager = extension_manager + + def _translate(self, ext): + ext_data = {} + ext_data['name'] = ext.get_name() + ext_data['alias'] = ext.get_alias() + ext_data['description'] = ext.get_description() + ext_data['namespace'] = ext.get_namespace() + ext_data['updated'] = ext.get_updated() + ext_data['links'] = [] # TODO: implement extension links + return ext_data + + def index(self, req): + extensions = [] + for alias, ext in self.extension_manager.extensions.iteritems(): + extensions.append(self._translate(ext)) + return dict(extensions=extensions) + + def show(self, req, id): + # NOTE: the extensions alias is used as the 'id' for show + ext = self.extension_manager.extensions[id] + return self._translate(ext) + + def delete(self, req, id): + raise faults.Fault(exc.HTTPNotFound()) + + def create(self, req): + raise faults.Fault(exc.HTTPNotFound()) + + def delete(self, req, id): + raise faults.Fault(exc.HTTPNotFound()) + + class ExtensionMiddleware(wsgi.Middleware): """ Extensions middleware that intercepts configured routes for extensions. @@ -183,12 +220,17 @@ class ExtensionMiddleware(wsgi.Middleware): class ExtensionManager(object): + """ + Load extensions from the configured extension path. + See nova/tests/api/openstack/extensions/foxinsocks.py for an example + extension implementation. + """ def __init__(self, path): LOG.audit(_('Initializing extension manager.')) self.path = path - self.extensions = [] + self.extensions = {} self._load_extensions() def get_resources(self): @@ -196,8 +238,14 @@ class ExtensionManager(object): returns a list of ResourceExtension objects """ resources = [] - for ext in self.extensions: - resources.extend(ext.get_resources()) + resources.append(ResourceExtension('extensions', + ExtensionController(self))) + for alias, ext in self.extensions.iteritems(): + try: + resources.extend(ext.get_resources()) + except AttributeError: + # NOTE: Extension aren't required to have resource extensions + pass return resources def get_actions(self): @@ -205,8 +253,12 @@ class ExtensionManager(object): returns a list of ActionExtension objects """ actions = [] - for ext in self.extensions: - actions.extend(ext.get_actions()) + for alias, ext in self.extensions.iteritems(): + try: + actions.extend(ext.get_actions()) + except AttributeError: + # NOTE: Extension aren't required to have action extensions + pass return actions def get_response_extensions(self): @@ -214,16 +266,36 @@ class ExtensionManager(object): returns a list of ResponseExtension objects """ response_exts = [] - for ext in self.extensions: - response_exts.extend(ext.get_response_extensions()) + for alias, ext in self.extensions.iteritems(): + try: + response_exts.extend(ext.get_response_extensions()) + except AttributeError: + # NOTE: Extension aren't required to have response extensions + pass return response_exts + def _check_extension(self, extension): + """ + Checks for required methods in extension objects. + """ + try: + LOG.debug(_('Ext name: %s'), extension.get_name()) + LOG.debug(_('Ext alias: %s'), extension.get_alias()) + LOG.debug(_('Ext description: %s'), extension.get_description()) + LOG.debug(_('Ext namespace: %s'), extension.get_namespace()) + LOG.debug(_('Ext updated: %s'), extension.get_updated()) + except AttributeError as ex: + LOG.exception(_("Exception loading extension: %s"), unicode(ex)) + def _load_extensions(self): """ Load extensions from the configured path. The extension name is constructed from the camel cased module_name + 'Extension'. If your extension module was named widgets.py the extension class within that module should be 'WidgetsExtension'. + + See nova/tests/api/openstack/extensions/foxinsocks.py for an example + extension implementation. """ if not os.path.exists(self.path): return @@ -235,7 +307,13 @@ class ExtensionManager(object): if file_ext.lower() == '.py': mod = imp.load_source(mod_name, ext_path) ext_name = mod_name[0].upper() + mod_name[1:] - self.extensions.append(getattr(mod, ext_name)()) + try: + new_ext = getattr(mod, ext_name)() + self._check_extension(new_ext) + self.extensions[new_ext.get_alias()] = new_ext + except AttributeError as ex: + LOG.exception(_("Exception loading extension: %s"), + unicode(ex)) class ResponseExtension(object): diff --git a/nova/tests/api/openstack/extensions/foxinsocks.py b/nova/tests/api/openstack/extensions/foxinsocks.py new file mode 100644 index 000000000..09a328273 --- /dev/null +++ b/nova/tests/api/openstack/extensions/foxinsocks.py @@ -0,0 +1,70 @@ +import json + +from nova import wsgi + +from nova.api.openstack import extensions + + +class FoxInSocksController(wsgi.Controller): + + def index(self, req): + return "Try to say this Mr. Knox, sir..." + + +class Foxinsocks(object): + + def __init__(self): + pass + + def get_name(self): + return "Fox In Socks" + + def get_alias(self): + return "FOXNSOX" + + def get_description(self): + return "The Fox In Socks Extension" + + def get_namespace(self): + return "http://www.fox.in.socks/api/ext/pie/v1.0" + + def get_updated(self): + return "2011-01-22T13:25:27-06:00" + + def get_resources(self): + resources = [] + resource = extensions.ResourceExtension('foxnsocks', + FoxInSocksController()) + resources.append(resource) + return resources + + def get_actions(self): + actions = [] + actions.append(extensions.ActionExtension('servers', 'add_tweedle', + self._add_tweedle)) + actions.append(extensions.ActionExtension('servers', 'delete_tweedle', + self._delete_tweedle)) + return actions + + def get_response_extensions(self): + response_exts = [] + + def _resp_handler(res): + #NOTE: This only handles JSON responses. + # You can use content type header to test for XML. + data = json.loads(res.body) + data['flavor']['googoose'] = "Gooey goo for chewy chewing!" + return data + + resp_ext = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', + _resp_handler) + response_exts.append(resp_ext) + return response_exts + + def _add_tweedle(self, input_dict, req, id): + + return "Tweedle Beetle Added." + + def _delete_tweedle(self, input_dict, req, id): + + return "Tweedle Beetle Deleted." diff --git a/nova/tests/api/openstack/extensions/widgets.py b/nova/tests/api/openstack/extensions/widgets.py deleted file mode 100644 index f463721f1..000000000 --- a/nova/tests/api/openstack/extensions/widgets.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -from nova import wsgi - -from nova.api.openstack import extensions - - -class WidgetsController(wsgi.Controller): - - def index(self, req): - return "Buy more widgets!" - - -class Widgets(object): - - def __init__(self): - pass - - def get_resources(self): - resources = [] - widgets = extensions.ResourceExtension('widgets', - WidgetsController()) - resources.append(widgets) - return resources - - def get_actions(self): - actions = [] - actions.append(extensions.ActionExtension('servers', 'add_widget', - self._add_widget)) - actions.append(extensions.ActionExtension('servers', 'delete_widget', - self._delete_widget)) - return actions - - def get_response_extensions(self): - response_exts = [] - - def _resp_handler(res): - # only handle JSON responses - data = json.loads(res.body) - data['flavor']['widgets'] = "Buy more widgets!" - return data - - widgets = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', - _resp_handler) - response_exts.append(widgets) - return response_exts - - def _add_widget(self, input_dict, req, id): - - return "Widget Added." - - def _delete_widget(self, input_dict, req, id): - - return "Widget Deleted." diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 8725c8f0e..0f99dec55 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -48,6 +48,15 @@ class StubExtensionManager(object): self.action_ext = action_ext self.response_ext = response_ext + def get_name(self): + return "Tweedle Beetle Extension" + + def get_alias(self): + return "TWDLBETL" + + def get_description(self): + return "Provides access to Tweedle Beetles" + def get_resources(self): resource_exts = [] if self.resource_ext: @@ -67,36 +76,53 @@ class StubExtensionManager(object): return response_exts +class ExtensionControllerTest(unittest.TestCase): + + def test_index(self): + app = openstack.APIRouter() + ext_midware = extensions.ExtensionMiddleware(app) + request = webob.Request.blank("/extensions") + response = request.get_response(ext_midware) + self.assertEqual(200, response.status_int) + + def test_get_by_alias(self): + app = openstack.APIRouter() + ext_midware = extensions.ExtensionMiddleware(app) + request = webob.Request.blank("/extensions/FOXNSOX") + response = request.get_response(ext_midware) + self.assertEqual(200, response.status_int) + +response_body = "Try to say this Mr. Knox, sir..." + class ResourceExtensionTest(unittest.TestCase): + def test_no_extension_present(self): manager = StubExtensionManager(None) app = openstack.APIRouter() ext_midware = extensions.ExtensionMiddleware(app, manager) - request = webob.Request.blank("/widgets") + request = webob.Request.blank("/blah") response = request.get_response(ext_midware) self.assertEqual(404, response.status_int) def test_get_resources(self): - response_body = "Buy more widgets!" - widgets = extensions.ResourceExtension('widgets', + res_ext = extensions.ResourceExtension('tweedles', StubController(response_body)) - manager = StubExtensionManager(widgets) + manager = StubExtensionManager(res_ext) app = openstack.APIRouter() ext_midware = extensions.ExtensionMiddleware(app, manager) - request = webob.Request.blank("/widgets") + request = webob.Request.blank("/tweedles") response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) self.assertEqual(response_body, response.body) def test_get_resources_with_controller(self): - response_body = "Buy more widgets!" - widgets = extensions.ResourceExtension('widgets', + res_ext = extensions.ResourceExtension('tweedles', StubController(response_body)) - manager = StubExtensionManager(widgets) + manager = StubExtensionManager(res_ext) app = openstack.APIRouter() ext_midware = extensions.ExtensionMiddleware(app, manager) - request = webob.Request.blank("/widgets") + request = webob.Request.blank("/tweedles") response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) self.assertEqual(response_body, response.body) @@ -104,6 +130,8 @@ class ResourceExtensionTest(unittest.TestCase): class ExtensionManagerTest(unittest.TestCase): + response_body = "Try to say this Mr. Knox, sir..." + def setUp(self): FLAGS.osapi_extensions_path = os.path.join(os.path.dirname(__file__), "extensions") @@ -111,10 +139,10 @@ class ExtensionManagerTest(unittest.TestCase): def test_get_resources(self): app = openstack.APIRouter() ext_midware = extensions.ExtensionMiddleware(app) - request = webob.Request.blank("/widgets") + request = webob.Request.blank("/foxnsocks") response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) - self.assertEqual("Buy more widgets!", response.body) + self.assertEqual(response_body, response.body) class ActionExtensionTest(unittest.TestCase): @@ -134,15 +162,15 @@ class ActionExtensionTest(unittest.TestCase): return response def test_extended_action(self): - body = dict(add_widget=dict(name="test")) + body = dict(add_tweedle=dict(name="test")) response = self._send_server_action_request("/servers/1/action", body) self.assertEqual(200, response.status_int) - self.assertEqual("Widget Added.", response.body) + self.assertEqual("Tweedle Beetle Added.", response.body) - body = dict(delete_widget=dict(name="test")) + body = dict(delete_tweedle=dict(name="test")) response = self._send_server_action_request("/servers/1/action", body) self.assertEqual(200, response.status_int) - self.assertEqual("Widget Deleted.", response.body) + self.assertEqual("Tweedle Beetle Deleted.", response.body) def test_invalid_action_body(self): body = dict(blah=dict(name="test")) # Doesn't exist @@ -169,21 +197,21 @@ class ResponseExtensionTest(unittest.TestCase): def test_get_resources(self): - test_resp = "Buy more widgets!" + test_resp = "Gooey goo for chewy chewing!" def _resp_handler(res): # only handle JSON responses data = json.loads(res.body) - data['flavor']['widgets'] = test_resp + data['flavor']['googoose'] = test_resp return data - widgets = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', + resp_ext = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', _resp_handler) - manager = StubExtensionManager(None, None, widgets) + manager = StubExtensionManager(None, None, resp_ext) app = fakes.wsgi_app() ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/v1.0/flavors/1") response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) response_data = json.loads(response.body) - self.assertEqual(test_resp, response_data['flavor']['widgets']) + self.assertEqual(test_resp, response_data['flavor']['googoose']) -- cgit From b2456e983178b97ad94f48c77ef210000d6d6ca4 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 16 Mar 2011 14:03:38 -0400 Subject: Move mapper code into the _action_ext_controllers and _response_ext_controllers methods. --- nova/api/openstack/extensions.py | 65 ++++++++++++----------- nova/tests/api/openstack/extensions/foxinsocks.py | 2 +- nova/tests/api/openstack/test_extensions.py | 7 +-- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 66ddd8078..4adfcfc5b 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -118,7 +118,7 @@ class ExtensionMiddleware(wsgi.Middleware): return cls(app, **local_config) return _factory - def _actions_by_collection(self, application, ext_mgr): + def _action_ext_controllers(self, application, ext_mgr, mapper): """ Return a dict of ActionExtensionController objects by collection """ @@ -126,18 +126,38 @@ class ExtensionMiddleware(wsgi.Middleware): for action in ext_mgr.get_actions(): if not action.collection in action_controllers.keys(): controller = ActionExtensionController(application) + mapper.connect("/%s/:(id)/action.:(format)" % + action.collection, + action='action', + controller=controller, + conditions=dict(method=['POST'])) + mapper.connect("/%s/:(id)/action" % action.collection, + action='action', + controller=controller, + conditions=dict(method=['POST'])) action_controllers[action.collection] = controller + return action_controllers - def _responses_by_collection(self, application, ext_mgr): + def _response_ext_controllers(self, application, ext_mgr, mapper): """ Return a dict of ResponseExtensionController objects by collection """ response_ext_controllers = {} for resp_ext in ext_mgr.get_response_extensions(): - if not resp_ext.url_route in response_ext_controllers.keys(): + if not resp_ext.key in response_ext_controllers.keys(): controller = ResponseExtensionController(application) - response_ext_controllers[resp_ext.url_route] = controller + mapper.connect(resp_ext.url_route + '.:(format)', + action='process', + controller=controller, + conditions=resp_ext.conditions) + + mapper.connect(resp_ext.url_route, + action='process', + controller=controller, + conditions=resp_ext.conditions) + response_ext_controllers[resp_ext.key] = controller + return response_ext_controllers def __init__(self, application, ext_mgr=None): @@ -159,38 +179,20 @@ class ExtensionMiddleware(wsgi.Middleware): parent_resource=resource.parent) # extended actions - action_controllers = self._actions_by_collection(application, ext_mgr) + action_controllers = self._action_ext_controllers(application, ext_mgr, + mapper) for action in ext_mgr.get_actions(): - LOG.debug(_('Extended collection/action: %s/%s'), - action.collection, - action.action_name) + LOG.debug(_('Extended action: %s'), action.action_name) controller = action_controllers[action.collection] controller.add_action(action.action_name, action.handler) - mapper.connect("/%s/:(id)/action.:(format)" % action.collection, - action='action', - controller=controller, - conditions=dict(method=['POST'])) - mapper.connect("/%s/:(id)/action" % action.collection, - action='action', - controller=controller, - conditions=dict(method=['POST'])) - # extended responses - resp_controllers = self._responses_by_collection(application, ext_mgr) + resp_controllers = self._response_ext_controllers(application, ext_mgr, + mapper) for response_ext in ext_mgr.get_response_extensions(): - LOG.debug(_('Extended response: %s'), response_ext.url_route) - controller = resp_controllers[response_ext.url_route] + LOG.debug(_('Extended response: %s'), response_ext.key) + controller = resp_controllers[response_ext.key] controller.add_handler(response_ext.handler) - mapper.connect(response_ext.url_route + '.:(format)', - action='process', - controller=controller, - conditions=response_ext.conditions) - - mapper.connect(response_ext.url_route, - action='process', - controller=controller, - conditions=response_ext.conditions) self._router = routes.middleware.RoutesMiddleware(self._dispatch, mapper) @@ -322,10 +324,11 @@ class ResponseExtension(object): core nova OpenStack API controllers. """ - def __init__(self, url_route, method, handler): + def __init__(self, method, url_route, handler): self.url_route = url_route - self.conditions = dict(method=[method]) self.handler = handler + self.conditions = dict(method=[method]) + self.key = "%s-%s" % (method, url_route) class ActionExtension(object): diff --git a/nova/tests/api/openstack/extensions/foxinsocks.py b/nova/tests/api/openstack/extensions/foxinsocks.py index 09a328273..2e93d8a55 100644 --- a/nova/tests/api/openstack/extensions/foxinsocks.py +++ b/nova/tests/api/openstack/extensions/foxinsocks.py @@ -56,7 +56,7 @@ class Foxinsocks(object): data['flavor']['googoose'] = "Gooey goo for chewy chewing!" return data - resp_ext = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', + resp_ext = extensions.ResponseExtension('GET', '/flavors/:(id)', _resp_handler) response_exts.append(resp_ext) return response_exts diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 0f99dec55..149f1973e 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -31,6 +31,8 @@ import nova.wsgi FLAGS = flags.FLAGS +response_body = "Try to say this Mr. Knox, sir..." + class StubController(nova.wsgi.Controller): @@ -92,11 +94,9 @@ class ExtensionControllerTest(unittest.TestCase): response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) -response_body = "Try to say this Mr. Knox, sir..." class ResourceExtensionTest(unittest.TestCase): - def test_no_extension_present(self): manager = StubExtensionManager(None) app = openstack.APIRouter() @@ -205,7 +205,8 @@ class ResponseExtensionTest(unittest.TestCase): data['flavor']['googoose'] = test_resp return data - resp_ext = extensions.ResponseExtension('/v1.0/flavors/:(id)', 'GET', + resp_ext = extensions.ResponseExtension('GET', + '/v1.0/flavors/:(id)', _resp_handler) manager = StubExtensionManager(None, None, resp_ext) app = fakes.wsgi_app() -- cgit From f91cd67c403d7de54600eea0d91c223af0493788 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 16 Mar 2011 14:09:29 -0400 Subject: Comment update. --- nova/api/openstack/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 4adfcfc5b..557b12fd9 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -345,7 +345,7 @@ class ActionExtension(object): class ResourceExtension(object): """ - ResourceExtension objects can be used to add add top level resources + ResourceExtension objects can be used to add top level resources to the OpenStack API in nova. """ -- cgit From 7b21275c0c05a9a7dbb23463f2b90623d79645ec Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 16 Mar 2011 14:22:29 -0400 Subject: Revert commit that modified CA/openssl.cnf.tmpl. --- CA/openssl.cnf.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CA/openssl.cnf.tmpl b/CA/openssl.cnf.tmpl index cf8bac828..dd81f1c2b 100644 --- a/CA/openssl.cnf.tmpl +++ b/CA/openssl.cnf.tmpl @@ -43,7 +43,7 @@ policy = policy_match [ policy_match ] countryName = match -stateOrProvinceName = optional +stateOrProvinceName = match organizationName = optional organizationalUnitName = optional commonName = supplied -- cgit From 4d057c9c2df77816ead6f30fa2795148aa8148d3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 11:44:40 -0700 Subject: Refactored ZoneRedirect into ZoneChildHelper so ZoneManager can use this too. --- nova/api/zone_redirect.py | 79 +++++++++++++++++++++++------------------- nova/compute/api.py | 8 ++--- nova/scheduler/api.py | 40 ++++++++++++++++----- nova/scheduler/zone_manager.py | 2 +- 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index fec1b1af3..0adf94046 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -28,6 +28,7 @@ import urllib from nova import exception from nova import log as logging from nova import wsgi +from nova.scheduler import api import novaclient.client as client import novaclient.exceptions as osexceptions @@ -41,6 +42,43 @@ except ImportError: LOG = logging.getLogger('server') +class RequestForwarder(api.ChildZoneHelper): + + def __init__(self, resource, method, body): + self.resource = resource + self.method = method + self.body = body + + def process(self, client, zone): + api_url = zone.api_url + LOG.debug(_("Zone redirect to: %(api_url)s, " % locals())) + try: + if self.method == 'GET': + response, body = client.get(self.resource, body=self.body) + elif self.method == 'POST': + response, body = client.post(self.resource, body=self.body) + elif self.method == 'PUT': + response, body = client.put(self.resource, body=self.body) + elif self.method == 'DELETE': + response, body = client.delete(self.resource, body=self.body) + except osexceptions.OpenStackException, e: + LOG.info(_("Zone returned error: %s ('%s', '%s')"), + e.code, e.message, e.details) + res = webob.Response() + res.status = "404" + return res + + status = response.status + LOG.debug(_("Zone %(api_url)s response: " + "%(response)s [%(status)s]/ %(body)s") % + locals()) + res = webob.Response() + res.status = response['status'] + res.content_type = response['content-type'] + res.body = json.dumps(body) + return res + + class ZoneRedirectMiddleware(wsgi.Middleware): """Catches Zone Routing exceptions and delegates the call to child zones.""" @@ -57,10 +95,8 @@ class ZoneRedirectMiddleware(wsgi.Middleware): # Todo(sandy): This only works for OpenStack API currently. # Needs to be broken out into a driver. - new_req = req.copy() - scheme, netloc, path, query, frag = \ - urlparse.urlsplit(new_req.path_qs) + urlparse.urlsplit(req.path_qs) query = urlparse.parse_qsl(query) query = [(key, value) for key, value in query if key != 'fresh'] query = urllib.urlencode(query) @@ -69,38 +105,11 @@ class ZoneRedirectMiddleware(wsgi.Middleware): m = re.search('/v\d+\.\d+/(.+)', url) resource = m.group(1) - for zone in e.zones: - LOG.debug(_("Zone redirect to:[url:%(api_url)s, " - "username:%(username)s]" - % dict(api_url=zone.api_url, - username=zone.username))) - - nova = client.OpenStackClient(zone.username, zone.password, - zone.api_url) - nova.authenticate() - try: - if req.method == 'GET': - response, body = nova.get(resource, body=new_req.body) - elif req.method == 'POST': - response, body = nova.post(resource, body=new_req.body) - elif req.method == 'PUT': - response, body = nova.put(resource, body=new_req.body) - elif req.method == 'DELETE': - response, body = nova.delete(resource, - body=new_req.body) - except osexceptions.OpenStackException, e: - LOG.info(_("Zone returned error: %s ('%s', '%s')"), - e.code, e.message, e.details) - continue - - LOG.debug(_("Zone Response: %s [%s]/ %s"), response, - response.status, body) - if response.status == 200: - res = webob.Response() - res.status = response['status'] - res.content_type = response['content-type'] - res.body = json.dumps(body) - return res + forwarder = RequestForwarder(resource, req.method, req.body) + for result in forwarder.start(e.zones): + # Todo(sandy): We need to aggregate multiple successes. + if result.status_int == 200: + return result LOG.debug(_("Zone Redirect Middleware returning 404 ...")) res = webob.Response() diff --git a/nova/compute/api.py b/nova/compute/api.py index 1185b9964..215257217 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -385,9 +385,7 @@ class API(base.Base): if not host: instance = self.get(context, instance_id) host = instance['host'] - #queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - queue = self.scheduler_api.get_queue_for_instance(context, - FLAGS.compute_topic, host) + queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) params['instance_id'] = instance_id kwargs = {'method': method, 'args': params} rpc.cast(context, queue, kwargs) @@ -406,9 +404,7 @@ class API(base.Base): if not host: instance = self.get(context, instance_id) host = instance["host"] - #queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - queue = self.scheduler_api.get_queue_for_instance(context, - FLAGS.compute_topic, host) + queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) params['instance_id'] = instance_id kwargs = {'method': method, 'args': params} return rpc.call(context, queue, kwargs) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 48da5bcfc..073784f31 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -23,6 +23,10 @@ from nova import flags from nova import log as logging from nova import rpc +import novaclient.client as client + +from eventlet import greenpool + FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.api') @@ -76,6 +80,8 @@ class API(object): @classmethod def get_instance_or_reroute(cls, context, instance_id): + """Return an instance from the db or throw a ZoneRouteException + if not found.""" try: instance = db.instance_get(context, instance_id) return instance @@ -88,12 +94,30 @@ class API(object): zones = db.zone_get_all(context) raise exception.ZoneRouteException(zones) - @classmethod - def get_queue_for_instance(cls, context, service, instance_id): - instance = db.instance_get(context, instance_id) - zone = db.get_zone(instance.zone.id) - if cls._is_current_zone(zone): - return db.queue_get_for(context, service, instance['host']) - # Throw a reroute Exception for the middleware to pick up. - raise exception.ZoneRouteException(zone) +def _wrap_method(function, self): + def _wrap(*args, **kwargs): + return function(self, *args, **kwargs) + return _wrap + + +def _process(self, zone): + nova = client.OpenStackClient(zone.username, zone.password, + zone.api_url) + nova.authenticate() + return self.process(nova, zone) + + +class ChildZoneHelper(object): + """Delegate a call to a set of Child Zones and wait for their + responses. Could be used for Zone Redirect or by the Scheduler + plug-ins to query the children.""" + + def start(self, zone_list): + self.green_pool = greenpool.GreenPool() + return [ result for result in self.green_pool.imap( + _wrap_method(_process, self), zone_list)] + + def process(self, client, zone): + """Derived class must override.""" + pass diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index c1a50dbc3..d32cc2e8f 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -104,7 +104,7 @@ class ZoneManager(object): """Keeps the zone states updated.""" def __init__(self): self.last_zone_db_check = datetime.min - self.zone_states = {} + self.zone_states = {} # { : ZoneState } self.service_states = {} # { : { : { cap k : v }}} self.green_pool = greenpool.GreenPool() -- cgit From af754e3bba9b2ee93147a3533319ac5a5e199f45 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Wed, 16 Mar 2011 21:51:32 +0300 Subject: libvirt template and libvirt_conn.spawn modified in way that was proposed for xenapi multinic support --- nova/virt/libvirt.xml.template | 21 ++-- nova/virt/libvirt_conn.py | 121 +++++++++++++++------ plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 3 +- 3 files changed, 101 insertions(+), 44 deletions(-) diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index 88bfbc668..43324c34b 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -69,21 +69,24 @@ #end if #end if + +#for $nic in $nics - - + + - - - -#if $getVar('extra_params', False) - ${extra_params} + + + +#if $getVar('nic.extra_params', False) + ${nic.extra_params} #end if -#if $getVar('ra_server', False) - +#if $getVar('nic.ra_server', False) + #end if +#end for diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7994e9547..c122ac8d4 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -412,16 +412,18 @@ class LibvirtConnection(object): # the normal xml file, we can just call reboot here self.reboot(instance) + # NOTE(ilyaalekseyev): Implementation like in multinics + # for xenapi(tr3buchet) @exception.wrap_exception - def spawn(self, instance): - xml = self.to_xml(instance) + def spawn(self, instance, network_info=None): + xml = self.to_xml(instance, network_info) db.instance_set_state(context.get_admin_context(), instance['id'], power_state.NOSTATE, 'launching') - self.firewall_driver.setup_basic_filtering(instance) - self.firewall_driver.prepare_instance_filter(instance) - self._create_image(instance, xml) + self.firewall_driver.setup_basic_filtering(instance, network_info) + self.firewall_driver.prepare_instance_filter(instance, network_info) + self._create_image(instance, xml, network_info) self._conn.createXML(xml, 0) LOG.debug(_("instance %s: is running"), instance['name']) self.firewall_driver.apply_instance_filter(instance) @@ -578,7 +580,8 @@ class LibvirtConnection(object): utils.execute('truncate', target, '-s', "%dG" % local_gb) # TODO(vish): should we format disk by default? - def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): + def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None, + network_info=None): # syntactic nicety def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, @@ -690,17 +693,7 @@ class LibvirtConnection(object): if FLAGS.libvirt_type == 'uml': utils.execute('sudo', 'chown', 'root', basepath('disk')) - def to_xml(self, instance, rescue=False): - # TODO(termie): cache? - LOG.debug(_('instance %s: starting toXML method'), instance['name']) - network = db.network_get_by_instance(context.get_admin_context(), - instance['id']) - # FIXME(vish): stick this in db - instance_type = instance['instance_type'] - # instance_type = test.INSTANCE_TYPES[instance_type] - instance_type = instance_types.get_instance_type(instance_type) - ip_address = db.instance_get_fixed_address(context.get_admin_context(), - instance['id']) + def _get_nic_for_xml(self, instance_id, network, mapping): # Assume that the gateway also acts as the dhcp server. dhcp_server = network['gateway'] ra_server = network['ra_server'] @@ -728,6 +721,75 @@ class LibvirtConnection(object): (net, mask) else: extra_params = "\n" + + result = { + 'id': mapping['mac'].replace(':', ''), + 'bridge_name': network['bridge'], + 'mac_address': mapping['mac'], + 'ip_address': mapping['ips'][0]['ip'], + 'dhcp_server': dhcp_server, + 'extra_params': extra_params, + } + + if ra_server: + result['ra_server'] = ra_server + "/128" + + return result + + def to_xml(self, instance, rescue=False, network_info=None): + admin_context = context.get_admin_context() + + # TODO(termie): cache? + LOG.debug(_('instance %s: starting toXML method'), instance['name']) + + ip_addresses = db.fixed_ip_get_all_by_instance(admin_context, + instance['id']) + + networks = db.network_get_all_by_instance(admin_context, + instance['id']) + + #TODO(ilyaalekseyev) remove network_info creation code + # when multinics will be completed + if network_info is None: + network_info = [] + + def ip_dict(ip): + return { + "ip": ip.address, + "netmask": network["netmask"], + "enabled": "1"} + + def ip6_dict(ip6): + return { + "ip": ip6.addressV6, + "netmask": ip6.netmaskV6, + "gateway": ip6.gatewayV6, + "enabled": "1"} + + for network in networks: + network_ips = [ip for ip in ip_addresses + if ip.network_id == network.id] + + mapping = { + 'label': network['label'], + 'gateway': network['gateway'], + 'mac': instance.mac_address, + 'dns': [network['dns']], + 'ips': [ip_dict(ip) for ip in network_ips], + 'ip6s': [ip6_dict(ip) for ip in network_ips]} + + network_info.append((network, mapping)) + + nics = [] + for (network, mapping) in network_info: + nics.append(self._get_nic_for_xml(instance['id'], + network, + mapping)) + # FIXME(vish): stick this in db + instance_type = instance['instance_type'] + # instance_type = test.INSTANCE_TYPES[instance_type] + instance_type = instance_types.get_instance_type(instance_type) + if FLAGS.use_cow_images: driver_type = 'qcow2' else: @@ -739,17 +801,11 @@ class LibvirtConnection(object): instance['name']), 'memory_kb': instance_type['memory_mb'] * 1024, 'vcpus': instance_type['vcpus'], - 'bridge_name': network['bridge'], - 'mac_address': instance['mac_address'], - 'ip_address': ip_address, - 'dhcp_server': dhcp_server, - 'extra_params': extra_params, 'rescue': rescue, 'local': instance_type['local_gb'], - 'driver_type': driver_type} + 'driver_type': driver_type, + 'nics': nics} - if ra_server: - xml_info['ra_server'] = ra_server + "/128" if not rescue: if instance['kernel_id']: xml_info['kernel'] = xml_info['basepath'] + "/kernel" @@ -762,7 +818,6 @@ class LibvirtConnection(object): xml = str(Template(self.libvirt_xml, searchList=[xml_info])) LOG.debug(_('instance %s: finished toXML method'), instance['name']) - return xml def get_info(self, instance_name): @@ -1251,7 +1306,7 @@ class LibvirtConnection(object): class FirewallDriver(object): - def prepare_instance_filter(self, instance): + def prepare_instance_filter(self, instance, network_info=None): """Prepare filters for the instance. At this point, the instance isn't running yet.""" @@ -1285,7 +1340,7 @@ class FirewallDriver(object): the security group.""" raise NotImplementedError() - def setup_basic_filtering(self, instance): + def setup_basic_filtering(self, instance, network_info=None): """Create rules to block spoofing and allow dhcp. This gets called when spawning an instance, before @@ -1390,7 +1445,7 @@ class NWFilterFirewall(FirewallDriver): ''' - def setup_basic_filtering(self, instance): + def setup_basic_filtering(self, instance, network_info=None): """Set up basic filtering (MAC, IP, and ARP spoofing protection)""" logging.info('called setup_basic_filtering in nwfilter') @@ -1495,7 +1550,7 @@ class NWFilterFirewall(FirewallDriver): # Nothing to do pass - def prepare_instance_filter(self, instance): + def prepare_instance_filter(self, instance, network_info=None): """ Creates an NWFilter for the given instance. In the process, it makes sure the filters for the security groups as well as @@ -1598,9 +1653,9 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.ipv4['filter'].add_chain('sg-fallback') self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') - def setup_basic_filtering(self, instance): + def setup_basic_filtering(self, instance, network_info=None): """Use NWFilter from libvirt for this.""" - return self.nwfilter.setup_basic_filtering(instance) + return self.nwfilter.setup_basic_filtering(instance, network_info) def apply_instance_filter(self, instance): """No-op. Everything is done in prepare_instance_filter""" @@ -1614,7 +1669,7 @@ class IptablesFirewallDriver(FirewallDriver): LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) - def prepare_instance_filter(self, instance): + def prepare_instance_filter(self, instance, network_info=None): self.instances[instance['id']] = instance self.add_filters_for_instance(instance) self.iptables.apply() diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index c996f6ef4..0a45f3873 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -216,8 +216,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, os_type): 'x-image-meta-status': 'queued', 'x-image-meta-disk-format': 'vhd', 'x-image-meta-container-format': 'ovf', - 'x-image-meta-property-os-type': os_type - } + 'x-image-meta-property-os-type': os_type} for header, value in headers.iteritems(): conn.putheader(header, value) -- cgit From 703e680aa6d0da1953ec6f8ae3a6aa66dc9fad7e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 16 Mar 2011 16:13:24 -0700 Subject: Fix the errors that pylint was reporting on this file This was meant more as a test of whether pylint was now returning false-positives. It looks like the bugs it's reporting are at least partially real. --- nova/api/openstack/servers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 3ecd4fb01..dfaf35128 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -25,8 +25,9 @@ from nova import compute from nova import exception from nova import flags from nova import log as logging -from nova import wsgi +from nova import quota from nova import utils +from nova import wsgi from nova.api.openstack import common from nova.api.openstack import faults from nova.auth import manager as auth_manager @@ -188,7 +189,7 @@ class Controller(wsgi.Controller): key_data=key_data, metadata=metadata, injected_files=injected_files) - except QuotaError as error: + except quota.QuotaError as error: self._handle_quota_error(error) server = _translate_keys(instances[0]) @@ -238,7 +239,7 @@ class Controller(wsgi.Controller): injected_files.append((path, contents)) return injected_files - def _handle_quota_errors(self, error): + def _handle_quota_error(self, error): """ Reraise quota errors as api-specific http exceptions """ -- cgit From a1e2959312b51757653447de3e8c9e92029da6fd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 16 Mar 2011 16:23:31 -0700 Subject: Fix a few of the more obvious non-errors while we're in here --- nova/api/openstack/servers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index dfaf35128..42cf693de 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -15,11 +15,10 @@ import base64 import hashlib -import json import traceback -from xml.dom import minidom from webob import exc +from xml.dom import minidom from nova import compute from nova import exception @@ -33,7 +32,6 @@ from nova.api.openstack import faults from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state -import nova.api.openstack LOG = logging.getLogger('server') @@ -270,7 +268,7 @@ class Controller(wsgi.Controller): update_dict['admin_pass'] = inst_dict['server']['adminPass'] try: self.compute_api.set_admin_password(ctxt, id) - except exception.TimeoutException, e: + except exception.TimeoutException: return exc.HTTPRequestTimeout() if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] -- cgit From c9158dfcf4efd2cf22df9aed7b1bb01e037e8eb2 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 19:04:27 -0700 Subject: moved scheduler API check into db.api decorator --- nova/api/zone_redirect.py | 5 ++++- nova/compute/api.py | 10 ++-------- nova/compute/manager.py | 5 +---- nova/db/api.py | 35 ++++++++++++++++++++++++++++++++++- nova/exception.py | 4 +++- nova/scheduler/api.py | 23 ++++++----------------- 6 files changed, 50 insertions(+), 32 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 0adf94046..4fe255c99 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -43,7 +43,7 @@ LOG = logging.getLogger('server') class RequestForwarder(api.ChildZoneHelper): - + """Worker for sending an OpenStack Request to each child zone.""" def __init__(self, resource, method, body): self.resource = resource self.method = method @@ -98,10 +98,13 @@ class ZoneRedirectMiddleware(wsgi.Middleware): scheme, netloc, path, query, frag = \ urlparse.urlsplit(req.path_qs) query = urlparse.parse_qsl(query) + # Remove any cache busters from old novaclient calls ... query = [(key, value) for key, value in query if key != 'fresh'] query = urllib.urlencode(query) url = urlparse.urlunsplit((scheme, netloc, path, query, frag)) + # Strip off the API version, since this is given when the + # child zone was added. m = re.search('/v\d+\.\d+/(.+)', url) resource = m.group(1) diff --git a/nova/compute/api.py b/nova/compute/api.py index 215257217..f4bfe720c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -34,7 +34,6 @@ from nova import rpc from nova import utils from nova import volume from nova.compute import instance_types -from nova.scheduler import api as scheduler from nova.db import base FLAGS = flags.FLAGS @@ -51,7 +50,7 @@ class API(base.Base): def __init__(self, image_service=None, network_api=None, volume_api=None, hostname_factory=generate_default_hostname, - scheduler_api=None, **kwargs): + **kwargs): if not image_service: image_service = utils.import_object(FLAGS.image_service) self.image_service = image_service @@ -61,9 +60,6 @@ class API(base.Base): if not volume_api: volume_api = volume.API() self.volume_api = volume_api - if not scheduler_api: - scheduler_api = scheduler.API() - self.scheduler_api = scheduler_api self.hostname_factory = hostname_factory super(API, self).__init__(**kwargs) @@ -347,8 +343,7 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - rv = self.scheduler_api.get_instance_or_reroute(context, instance_id) - #rv = self.db.instance_get(context, instance_id) + rv = self.db.instance_get(context, instance_id) return dict(rv.iteritems()) def get_all(self, context, project_id=None, reservation_id=None, @@ -513,7 +508,6 @@ class API(base.Base): def get_ajax_console(self, context, instance_id): """Get a url to an AJAX Console""" - instance = self.get(context, instance_id) output = self._call_compute_message('get_ajax_console', context, instance_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 499b212e2..ce60c6b43 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -48,7 +48,6 @@ from nova import scheduler_manager from nova import rpc from nova import utils from nova.compute import power_state -from nova.scheduler import api as scheduler_api FLAGS = flags.FLAGS flags.DEFINE_string('instances_path', '$state_path/instances', @@ -523,9 +522,7 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): """Pause an instance on this server.""" context = context.elevated() LOG.debug(_('*** instance %s: starting pause'), instance_id) - instance_ref = scheduler_api.get_instance_or_reroute(context, - instance_id) - #instance_ref = self.db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) LOG.audit(_('instance %s: pausing'), instance_id, context=context) self.db.instance_set_state(context, instance_id, diff --git a/nova/db/api.py b/nova/db/api.py index 2ecfc0211..6298e16ad 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -34,6 +34,7 @@ The underlying driver is loaded as a :class:`LazyPluggable`. from nova import exception from nova import flags +from nova import log as logging from nova import utils @@ -52,6 +53,9 @@ IMPL = utils.LazyPluggable(FLAGS['db_backend'], sqlalchemy='nova.db.sqlalchemy.api') +LOG = logging.getLogger('server') + + class NoMoreAddresses(exception.Error): """No more available addresses.""" pass @@ -71,6 +75,34 @@ class NoMoreTargets(exception.Error): """No more available blades""" pass + +################### + + +def reroute_if_not_found(key_args_index=None): + """Decorator used to indicate that the method should throw + a RouteRedirectException if the query can't find anything. + """ + def wrap(f): + def wrapped_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except exception.InstanceNotFound, e: + context = args[0] + key = None + if key_args_index: + key = args[key_args_index] + LOG.debug(_("Instance %(key)s not found locally: '%(e)s'" % + locals())) + + # Throw a reroute Exception for the middleware to pick up. + LOG.debug("Firing ZoneRouteException") + zones = zone_get_all(context) + raise exception.ZoneRouteException(zones, e) + return wrapped_f + return wrap + + ################### @@ -367,7 +399,8 @@ def instance_destroy(context, instance_id): return IMPL.instance_destroy(context, instance_id) -def instance_get(context, instance_id): +@reroute_if_not_found(key_args_index=1) +def instance_get(context, instance_id, reroute=True): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) diff --git a/nova/exception.py b/nova/exception.py index d0baa2e29..cfed32a72 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -93,8 +93,10 @@ class TimeoutException(Error): class ZoneRouteException(Error): - def __init__(self, zones, *args, **kwargs): + """Thrown in API to reroute request to child zones.""" + def __init__(self, zones, original_exception, *args, **kwargs): self.zones = zones + self.original_exception = original_exception super(ZoneRouteException, self).__init__(*args, **kwargs) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 073784f31..2da2dabfe 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -78,30 +78,16 @@ class API(object): capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) - @classmethod - def get_instance_or_reroute(cls, context, instance_id): - """Return an instance from the db or throw a ZoneRouteException - if not found.""" - try: - instance = db.instance_get(context, instance_id) - return instance - except exception.InstanceNotFound, e: - LOG.debug(_("Instance %(instance_id)s not found locally: '%(e)s'" % - locals())) - - # Throw a reroute Exception for the middleware to pick up. - LOG.debug("Firing ZoneRouteException") - zones = db.zone_get_all(context) - raise exception.ZoneRouteException(zones) - def _wrap_method(function, self): + """Wrap method to supply 'self'.""" def _wrap(*args, **kwargs): return function(self, *args, **kwargs) return _wrap def _process(self, zone): + """Worker stub for green thread pool""" nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() @@ -114,10 +100,13 @@ class ChildZoneHelper(object): plug-ins to query the children.""" def start(self, zone_list): + """Spawn a green thread for each child zone, calling the + derived classes process() method as the worker. Returns + a list of HTTP Responses. 1 per child.""" self.green_pool = greenpool.GreenPool() return [ result for result in self.green_pool.imap( _wrap_method(_process, self), zone_list)] def process(self, client, zone): - """Derived class must override.""" + """Worker Method. Derived class must override.""" pass -- cgit From cfe77c1236b68aa96dd85503582e08a07a23f77f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 19:21:32 -0700 Subject: cleanup --- nova/api/openstack/servers.py | 1 - nova/compute/manager.py | 1 - nova/db/api.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ffcbe628c..85999764f 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -343,7 +343,6 @@ class Controller(wsgi.Controller): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: - LOG.debug(_("*** Compute.api::pause %s"), id) self.compute_api.pause(ctxt, id) except: readable = traceback.format_exc() diff --git a/nova/compute/manager.py b/nova/compute/manager.py index ce60c6b43..ebe1ce6f0 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -521,7 +521,6 @@ class ComputeManager(scheduler_manager.SchedulerDependentManager): def pause_instance(self, context, instance_id): """Pause an instance on this server.""" context = context.elevated() - LOG.debug(_('*** instance %s: starting pause'), instance_id) instance_ref = self.db.instance_get(context, instance_id) LOG.audit(_('instance %s: pausing'), instance_id, context=context) self.db.instance_set_state(context, diff --git a/nova/db/api.py b/nova/db/api.py index 6298e16ad..d56d6f404 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -400,7 +400,7 @@ def instance_destroy(context, instance_id): @reroute_if_not_found(key_args_index=1) -def instance_get(context, instance_id, reroute=True): +def instance_get(context, instance_id): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) -- cgit From 609a912fa8a816c1f47140489dcc1131356cd67c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 19:26:54 -0700 Subject: pep8 --- nova/api/zone_redirect.py | 8 ++++---- nova/scheduler/api.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 4fe255c99..7ebae1401 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -48,7 +48,7 @@ class RequestForwarder(api.ChildZoneHelper): self.resource = resource self.method = method self.body = body - + def process(self, client, zone): api_url = zone.api_url LOG.debug(_("Zone redirect to: %(api_url)s, " % locals())) @@ -89,12 +89,12 @@ class ZoneRedirectMiddleware(wsgi.Middleware): return req.get_response(self.application) except exception.ZoneRouteException as e: if not e.zones: - exc = webob.exc.HTTPInternalServerError(explanation= - _("No zones to reroute to.")) + exc = webob.exc.HTTPInternalServerError(explanation=_( + "No zones to reroute to.")) return faults.Fault(exc) # Todo(sandy): This only works for OpenStack API currently. - # Needs to be broken out into a driver. + # Needs to be broken out into a driver. scheme, netloc, path, query, frag = \ urlparse.urlsplit(req.path_qs) query = urlparse.parse_qsl(query) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 2da2dabfe..f0b645c09 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -77,7 +77,7 @@ class API(object): args=dict(service_name=service_name, host=host, capabilities=capabilities)) return rpc.fanout_cast(context, 'scheduler', kwargs) - + def _wrap_method(function, self): """Wrap method to supply 'self'.""" @@ -92,7 +92,7 @@ def _process(self, zone): zone.api_url) nova.authenticate() return self.process(nova, zone) - + class ChildZoneHelper(object): """Delegate a call to a set of Child Zones and wait for their @@ -104,9 +104,9 @@ class ChildZoneHelper(object): derived classes process() method as the worker. Returns a list of HTTP Responses. 1 per child.""" self.green_pool = greenpool.GreenPool() - return [ result for result in self.green_pool.imap( + return [result for result in self.green_pool.imap( _wrap_method(_process, self), zone_list)] - + def process(self, client, zone): """Worker Method. Derived class must override.""" pass -- cgit From 2f72127478405f5d87a40d799cc04e77e744f35b Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 23:31:06 -0300 Subject: removed dead method --- nova/scheduler/api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 804dade6b..71d211fe9 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -49,10 +49,6 @@ def _call_scheduler(method, context, params=None): class API(object): """API for interacting with the scheduler.""" - @classmethod - def _is_current_zone(cls, zone): - return True - @classmethod def get_zone_list(cls, context): """Return a list of zones assoicated with this zone.""" -- cgit From 82a65107ba3e3df8ec52984bb835b71fe4283b4c Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 16 Mar 2011 23:33:32 -0300 Subject: fix up copyright --- nova/api/zone_redirect.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/api/zone_redirect.py b/nova/api/zone_redirect.py index 7ebae1401..c600b9ab5 100644 --- a/nova/api/zone_redirect.py +++ b/nova/api/zone_redirect.py @@ -1,6 +1,4 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may -- cgit From abc6c82449dfc46a33dcd8190840e51f44b5b930 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Mar 2011 07:30:22 -0700 Subject: Replaced capability flags with List --- nova/api/openstack/zones.py | 4 ++-- nova/flags.py | 5 +++-- nova/tests/api/openstack/test_zones.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 547920901..ebfc7743c 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -71,9 +71,9 @@ class Controller(wsgi.Controller): items = api.API.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) - caps = FLAGS.zone_capabilities.split(';') + caps = FLAGS.zone_capabilities for cap in caps: - key_values = cap.split(':') + key_values = cap.split('=') zone[key_values[0]] = key_values[1] for item, (min_value, max_value) in items.iteritems(): zone[item] = "%s,%s" % (min_value, max_value) diff --git a/nova/flags.py b/nova/flags.py index c05cef373..3a8ec1a39 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -358,5 +358,6 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'hypervisor:xenserver;os:linux', - 'Key/Value tags which represent capabilities of this zone') +DEFINE_list('zone_capabilities', + ['hypervisor=xenserver;kvm', 'os=linux;windows'], + 'Key/Multi-value list representng capabilities of this zone') diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 5e3aee4a7..12d39fd29 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -178,7 +178,7 @@ class ZonesTest(test.TestCase): def test_zone_info(self): FLAGS.zone_name = 'darksecret' - FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' + FLAGS.zone_capabilities = ['cap1=a;b', 'cap2=c;d'] self.stubs.Set(api, '_call_scheduler', zone_caps) body = dict(zone=dict(username='zeb', password='sneaky')) @@ -188,5 +188,5 @@ class ZonesTest(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) self.assertEqual(res_dict['zone']['name'], 'darksecret') - self.assertEqual(res_dict['zone']['cap1'], 'a,b') - self.assertEqual(res_dict['zone']['cap2'], 'c,d') + self.assertEqual(res_dict['zone']['cap1'], 'a;b') + self.assertEqual(res_dict['zone']['cap2'], 'c;d') -- cgit From ba0160cacac1c7db71eadd6624ee75a014c18378 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Thu, 17 Mar 2011 18:06:05 +0300 Subject: refactored: network_info creation extracted to method --- nova/virt/libvirt_conn.py | 74 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index c122ac8d4..90bd5421c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -154,6 +154,45 @@ def _get_ip_version(cidr): return int(net.version()) +def _get_network_info(instance): + admin_context = context.get_admin_context() + + ip_addresses = db.fixed_ip_get_all_by_instance(admin_context, + instance['id']) + + networks = db.network_get_all_by_instance(admin_context, + instance['id']) + network_info = [] + + def ip_dict(ip): + return { + "ip": ip.address, + "netmask": network["netmask"], + "enabled": "1"} + + def ip6_dict(ip6): + return { + "ip": ip6.addressV6, + "netmask": ip6.netmaskV6, + "gateway": ip6.gatewayV6, + "enabled": "1"} + + for network in networks: + network_ips = [ip for ip in ip_addresses + if ip.network_id == network.id] + + mapping = { + 'label': network['label'], + 'gateway': network['gateway'], + 'mac': instance.mac_address, + 'dns': [network['dns']], + 'ips': [ip_dict(ip) for ip in network_ips], + 'ip6s': [ip6_dict(ip) for ip in network_ips]} + + network_info.append((network, mapping)) + return network_info + + class LibvirtConnection(object): def __init__(self, read_only): @@ -742,43 +781,10 @@ class LibvirtConnection(object): # TODO(termie): cache? LOG.debug(_('instance %s: starting toXML method'), instance['name']) - ip_addresses = db.fixed_ip_get_all_by_instance(admin_context, - instance['id']) - - networks = db.network_get_all_by_instance(admin_context, - instance['id']) - #TODO(ilyaalekseyev) remove network_info creation code # when multinics will be completed if network_info is None: - network_info = [] - - def ip_dict(ip): - return { - "ip": ip.address, - "netmask": network["netmask"], - "enabled": "1"} - - def ip6_dict(ip6): - return { - "ip": ip6.addressV6, - "netmask": ip6.netmaskV6, - "gateway": ip6.gatewayV6, - "enabled": "1"} - - for network in networks: - network_ips = [ip for ip in ip_addresses - if ip.network_id == network.id] - - mapping = { - 'label': network['label'], - 'gateway': network['gateway'], - 'mac': instance.mac_address, - 'dns': [network['dns']], - 'ips': [ip_dict(ip) for ip in network_ips], - 'ip6s': [ip6_dict(ip) for ip in network_ips]} - - network_info.append((network, mapping)) + network_info = _get_network_info(instance) nics = [] for (network, mapping) in network_info: -- cgit From 720ba5d5c9ad2c24d9f0275fb783f191836a75f3 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Thu, 17 Mar 2011 18:22:48 +0300 Subject: bugfix --- nova/virt/libvirt_conn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 90bd5421c..b26955d42 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -155,6 +155,8 @@ def _get_ip_version(cidr): def _get_network_info(instance): + #TODO(ilyaalekseyev) If we will keep this function + # we should cache network_info admin_context = context.get_admin_context() ip_addresses = db.fixed_ip_get_all_by_instance(admin_context, @@ -189,7 +191,7 @@ def _get_network_info(instance): 'ips': [ip_dict(ip) for ip in network_ips], 'ip6s': [ip6_dict(ip) for ip in network_ips]} - network_info.append((network, mapping)) + network_info.append((network, mapping)) return network_info -- cgit From 41c097000c1eeb4f1532b22f136c383b8174e6cc Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 17 Mar 2011 11:35:32 -0400 Subject: Add tests and code to handle multiple ResponseExtension objects. --- nova/api/openstack/extensions.py | 13 ++++++++++++- nova/tests/api/openstack/extensions/foxinsocks.py | 17 ++++++++++++++--- nova/tests/api/openstack/test_extensions.py | 17 ++++++++++++++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 557b12fd9..23181d2a6 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -66,9 +66,20 @@ class ResponseExtensionController(wsgi.Controller): def process(self, req, *args, **kwargs): res = req.get_response(self.application) + content_type = req.best_match_content_type() # currently response handlers are un-ordered for handler in self.handlers: - return handler(res) + res=handler(res) + try: + body = res.body + headers = res.headers + except AttributeError: + body = self._serialize(res, content_type) + headers={"Content-Type": content_type} + res = webob.Response() + res.body = body + res.headers = headers + return res class ExtensionController(wsgi.Controller): diff --git a/nova/tests/api/openstack/extensions/foxinsocks.py b/nova/tests/api/openstack/extensions/foxinsocks.py index 2e93d8a55..fa979c7b5 100644 --- a/nova/tests/api/openstack/extensions/foxinsocks.py +++ b/nova/tests/api/openstack/extensions/foxinsocks.py @@ -49,16 +49,27 @@ class Foxinsocks(object): def get_response_extensions(self): response_exts = [] - def _resp_handler(res): + def _goose_handler(res): #NOTE: This only handles JSON responses. # You can use content type header to test for XML. data = json.loads(res.body) data['flavor']['googoose'] = "Gooey goo for chewy chewing!" return data - resp_ext = extensions.ResponseExtension('GET', '/flavors/:(id)', - _resp_handler) + resp_ext = extensions.ResponseExtension('GET', '/v1.0/flavors/:(id)', + _goose_handler) response_exts.append(resp_ext) + + def _bands_handler(res): + #NOTE: This only handles JSON responses. + # You can use content type header to test for XML. + data = json.loads(res.body) + data['big_bands'] = 'Pig Bands!' + return data + + resp_ext2 = extensions.ResponseExtension('GET', '/v1.0/flavors/:(id)', + _bands_handler) + response_exts.append(resp_ext2) return response_exts def _add_tweedle(self, input_dict, req, id): diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 149f1973e..11ed61e0d 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -195,7 +195,7 @@ class ResponseExtensionTest(unittest.TestCase): fakes.stub_out_auth(self.stubs) self.context = context.get_admin_context() - def test_get_resources(self): + def test_get_resources_with_stub_mgr(self): test_resp = "Gooey goo for chewy chewing!" @@ -208,6 +208,7 @@ class ResponseExtensionTest(unittest.TestCase): resp_ext = extensions.ResponseExtension('GET', '/v1.0/flavors/:(id)', _resp_handler) + manager = StubExtensionManager(None, None, resp_ext) app = fakes.wsgi_app() ext_midware = extensions.ExtensionMiddleware(app, manager) @@ -216,3 +217,17 @@ class ResponseExtensionTest(unittest.TestCase): self.assertEqual(200, response.status_int) response_data = json.loads(response.body) self.assertEqual(test_resp, response_data['flavor']['googoose']) + + def test_get_resources_with_mgr(self): + + test_resp = "Gooey goo for chewy chewing!" + + app = fakes.wsgi_app() + ext_midware = extensions.ExtensionMiddleware(app) + request = webob.Request.blank("/v1.0/flavors/1") + response = request.get_response(ext_midware) + self.assertEqual(200, response.status_int) + response_data = json.loads(response.body) + print response_data + self.assertEqual(test_resp, response_data['flavor']['googoose']) + self.assertEqual("Pig Bands!", response_data['big_bands']) -- cgit From 1f99a95b8615e55c9828eb36e12b9aaa762470bb Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Thu, 17 Mar 2011 20:48:22 +0300 Subject: fixed IpTablesFirewal --- nova/virt/libvirt_conn.py | 88 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b26955d42..fcaf8d879 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1663,6 +1663,8 @@ class IptablesFirewallDriver(FirewallDriver): def setup_basic_filtering(self, instance, network_info=None): """Use NWFilter from libvirt for this.""" + if not network_info: + network_info = _get_network_info(instance) return self.nwfilter.setup_basic_filtering(instance, network_info) def apply_instance_filter(self, instance): @@ -1678,28 +1680,47 @@ class IptablesFirewallDriver(FirewallDriver): 'filtered'), instance['id']) def prepare_instance_filter(self, instance, network_info=None): + if not network_info: + network_info = _get_network_info(instance) self.instances[instance['id']] = instance - self.add_filters_for_instance(instance) + self.add_filters_for_instance(instance, network_info) self.iptables.apply() - def add_filters_for_instance(self, instance): + def add_filters_for_instance(self, instance, network_info=None): + if not network_info: + network_info = _get_network_info(instance) chain_name = self._instance_chain_name(instance) self.iptables.ipv4['filter'].add_chain(chain_name) - ipv4_address = self._ip_for_instance(instance) - self.iptables.ipv4['filter'].add_rule('local', - '-d %s -j $%s' % - (ipv4_address, chain_name)) + + if network_info: + ips_v4 = [] + for (_n, mapping) in network_info: + for ip in mapping['ips']: + ips_v4.append(ip['ip']) + else: + ips_v4 = [self._ip_for_instance(instance)] + + for ipv4_address in ips_v4: + self.iptables.ipv4['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv4_address, chain_name)) if FLAGS.use_ipv6: self.iptables.ipv6['filter'].add_chain(chain_name) - ipv6_address = self._ip_for_instance_v6(instance) - self.iptables.ipv6['filter'].add_rule('local', - '-d %s -j $%s' % - (ipv6_address, - chain_name)) + if network_info: + ips_v6 = [ip['ip'] for ip in mapping['ip6s'] for (_n, mapping) + in network_info] + else: + ips_v6 = [self._ip_for_instance_v6(instance)] + + for ipv6_address in ips_v6: + self.iptables.ipv6['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv6_address, + chain_name)) - ipv4_rules, ipv6_rules = self.instance_rules(instance) + ipv4_rules, ipv6_rules = self.instance_rules(instance, network_info) for rule in ipv4_rules: self.iptables.ipv4['filter'].add_rule(chain_name, rule) @@ -1715,7 +1736,9 @@ class IptablesFirewallDriver(FirewallDriver): if FLAGS.use_ipv6: self.iptables.ipv6['filter'].remove_chain(chain_name) - def instance_rules(self, instance): + def instance_rules(self, instance, network_info=None): + if not network_info: + network_info = _get_network_info(instance) ctxt = context.get_admin_context() ipv4_rules = [] @@ -1729,28 +1752,49 @@ class IptablesFirewallDriver(FirewallDriver): ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] - dhcp_server = self._dhcp_server_for_instance(instance) - ipv4_rules += ['-s %s -p udp --sport 67 --dport 68 ' - '-j ACCEPT' % (dhcp_server,)] + if network_info: + dhcp_servers = [network['gateway'] for (network, _m) + in network_info] + else: + dhcp_servers = [self._dhcp_server_for_instance(instance)] + + for dhcp_server in dhcp_servers: + ipv4_rules += ['-s %s -p udp --sport 67 --dport 68 ' + '-j ACCEPT' % (dhcp_server,)] #Allow project network traffic if FLAGS.allow_project_net_traffic: - cidr = self._project_cidr_for_instance(instance) - ipv4_rules += ['-s %s -j ACCEPT' % (cidr,)] + if network_info: + cidrs = [network['cidr'] for (network, _m) in network_info] + else: + cidrs = [self._project_cidr_for_instance(instance)] + for cidr in cidrs: + ipv4_rules += ['-s %s -j ACCEPT' % (cidr,)] # We wrap these in FLAGS.use_ipv6 because they might cause # a DB lookup. The other ones are just list operations, so # they're not worth the clutter. if FLAGS.use_ipv6: # Allow RA responses - ra_server = self._ra_server_for_instance(instance) - if ra_server: + if network_info: + ra_servers = [network['ra_server'] for (network, _m) + in network_info] + else: + ra_servers = [self._ra_server_for_instance(instance)] + + for ra_server in ra_servers: ipv6_rules += ['-s %s/128 -p icmpv6 -j ACCEPT' % (ra_server,)] #Allow project network traffic if FLAGS.allow_project_net_traffic: - cidrv6 = self._project_cidrv6_for_instance(instance) - ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] + if network_info: + cidrv6s = [network['cidr_v6'] for (network, _m) + in network_info] + else: + cidrv6s = [self._project_cidrv6_for_instance(instance)] + + for cidrv6 in cidrv6s: + ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] security_groups = db.security_group_get_by_instance(ctxt, instance['id']) -- cgit From 31388f18f8c0ebe3cae58ebd2a46e2bedb376fd4 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Thu, 17 Mar 2011 20:56:25 +0300 Subject: fixes for NWFilterFirewall and net injection --- nova/virt/interfaces.template | 18 +++++++------- nova/virt/libvirt_conn.py | 58 +++++++++++++++++++++++++++++-------------- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/nova/virt/interfaces.template b/nova/virt/interfaces.template index 87b92b84a..7d40a0f69 100644 --- a/nova/virt/interfaces.template +++ b/nova/virt/interfaces.template @@ -5,13 +5,13 @@ auto lo iface lo inet loopback -# The primary network interface -auto eth0 -iface eth0 inet static - address %(address)s - netmask %(netmask)s - broadcast %(broadcast)s - gateway %(gateway)s - dns-nameservers %(dns)s - +#for $ifc in $interfaces +auto ${ifc.name} +iface ${ifc.name} inet static + address ${ifc.address} + netmask ${ifc.netmask} + broadcast ${ifc.broadcast} + gateway ${ifc.gateway} + dns-nameservers ${ifc.dns} +#end for diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b26955d42..bc6c9f37d 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -623,6 +623,9 @@ class LibvirtConnection(object): def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None, network_info=None): + if network_info is None: + network_info = _get_network_info(inst) + # syntactic nicety def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, @@ -698,21 +701,32 @@ class LibvirtConnection(object): key = str(inst['key_data']) net = None - network_ref = db.network_get_by_instance(context.get_admin_context(), - inst['id']) - if network_ref['injected']: - admin_context = context.get_admin_context() - address = db.instance_get_fixed_address(admin_context, inst['id']) - ra_server = network_ref['ra_server'] - if not ra_server: - ra_server = "fd00::" - with open(FLAGS.injected_network_template) as f: - net = f.read() % {'address': address, - 'netmask': network_ref['netmask'], - 'gateway': network_ref['gateway'], - 'broadcast': network_ref['broadcast'], - 'dns': network_ref['dns'], - 'ra_server': ra_server} + #network_ref = db.network_get_by_instance(context.get_admin_context(), + # inst['id']) + + nets = [] + ifc_template = open(FLAGS.injected_network_template).read() + ifc_num = -1 + for (network_ref, _m) in network_info: + ifc_num += 1 + if network_ref['injected']: + admin_context = context.get_admin_context() + address = db.instance_get_fixed_address( + admin_context, inst['id']) + ra_server = network_ref['ra_server'] + if not ra_server: + ra_server = "fd00::" + net_info = {'name': 'eth%d' % ifc_num, + 'address': address, + 'netmask': network_ref['netmask'], + 'gateway': network_ref['gateway'], + 'broadcast': network_ref['broadcast'], + 'dns': network_ref['dns'], + 'ra_server': ra_server} + nets.append(net_info) + + net = str(Template(ifc_template, searchList=[{'interfaces': nets}])) + if key or net: inst_name = inst['name'] img_id = inst.image_id @@ -738,6 +752,7 @@ class LibvirtConnection(object): # Assume that the gateway also acts as the dhcp server. dhcp_server = network['gateway'] ra_server = network['ra_server'] + mac_id = mapping['mac'].replace(':', '') if FLAGS.allow_project_net_traffic: if FLAGS.use_ipv6: @@ -764,7 +779,7 @@ class LibvirtConnection(object): extra_params = "\n" result = { - 'id': mapping['mac'].replace(':', ''), + 'id': mac_id, 'bridge_name': network['bridge'], 'mac_address': mapping['mac'], 'ip_address': mapping['ips'][0]['ip'], @@ -1362,6 +1377,11 @@ class FirewallDriver(object): instance['id']) return network['ra_server'] + def _all_ra_servers_for_instance(selfself, instance): + networks = db.network_get_all_by_instance(context.get_admin_context(), + instance['id']) + return [network['ra_server'] for network in networks] + class NWFilterFirewall(FirewallDriver): """ @@ -1576,8 +1596,10 @@ class NWFilterFirewall(FirewallDriver): 'nova-base-ipv6', 'nova-allow-dhcp-server'] if FLAGS.use_ipv6: - ra_server = self._ra_server_for_instance(instance) - if ra_server: + #ra_server = self._ra_server_for_instance(instance) + ra_servers = self._all_ra_servers_for_instance(instance) + #if ra_server: + if len(ra_servers) != 0: instance_secgroup_filter_children += ['nova-allow-ra-server'] ctxt = context.get_admin_context() -- cgit From 25199b6b93d17ff7dc192306e44932969846d9b6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Mar 2011 19:55:55 -0700 Subject: decorator more generic now --- nova/scheduler/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 0e2c69f75..190eb363e 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -99,6 +99,7 @@ def child_zone_helper(zone_list, func): def _issue_novaclient_command(nova, zone, method_name, instance_id): server = None try: + manager = getattr(nova, "servers") if isinstance(instance_id, int) or instance_id.isdigit(): server = manager.get(int(instance_id)) else: -- cgit From 0bc393bd1a0b722b08a2834873a8a825b86035c2 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 06:38:02 -0700 Subject: enable_zone_routing flag --- nova/compute/api.py | 20 ++++++++--------- nova/scheduler/api.py | 60 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 9fb4c8ae2..96538dd00 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -343,7 +343,7 @@ class API(base.Base): rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) - #@scheduler_api.reroute_if_not_found("delete") + @scheduler_api.reroute("delete") def delete(self, context, instance_id): LOG.debug(_("Going to try to terminate %s"), instance_id) try: @@ -373,12 +373,10 @@ class API(base.Base): def get(self, context, instance_id): """Get a single instance with the given ID.""" - LOG.debug("*** COMPUTE.API::GET") rv = self.db.instance_get(context, instance_id) - LOG.debug("*** COMPUTE.API::GET OUT CLEAN") return dict(rv.iteritems()) - @scheduler_api.reroute_if_not_found("get") + @scheduler_api.reroute_compute("get") def routing_get(self, context, instance_id): """Use this method instead of get() if this is the only operation you intend to to. It will route to novaclient.get @@ -502,17 +500,17 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, }},) - #@scheduler_api.reroute_if_not_found("pause") + @scheduler_api.reroute_compute("pause") def pause(self, context, instance_id): """Pause the given instance.""" self._cast_compute_message('pause_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("unpause") + @scheduler_api.reroute_compute("unpause") def unpause(self, context, instance_id): """Unpause the given instance.""" self._cast_compute_message('unpause_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("diagnostics") + @scheduler_api.reroute_compute("diagnostics") def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for the given instance.""" return self._call_compute_message( @@ -524,22 +522,22 @@ class API(base.Base): """Retrieve actions for the given instance.""" return self.db.instance_get_actions(context, instance_id) - #@scheduler_api.reroute_if_not_found("suspend") + @scheduler_api.reroute_compute("suspend") def suspend(self, context, instance_id): """suspend the instance with instance_id""" self._cast_compute_message('suspend_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("resume") + @scheduler_api.reroute_compute("resume") def resume(self, context, instance_id): """resume the instance with instance_id""" self._cast_compute_message('resume_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("rescue") + @scheduler_api.reroute_compute("rescue") def rescue(self, context, instance_id): """Rescue the given instance.""" self._cast_compute_message('rescue_instance', context, instance_id) - #@scheduler_api.reroute_if_not_found("unrescue") + @scheduler_api.reroute_compute("unrescue") def unrescue(self, context, instance_id): """Unrescue the given instance.""" self._cast_compute_message('unrescue_instance', context, instance_id) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 190eb363e..90b92d7ed 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -28,6 +28,10 @@ import novaclient.client as client from eventlet import greenpool FLAGS = flags.FLAGS +flags.DEFINE_bool('enable_zone_routing', + False, + 'When True, routing to child zones will occur.') + LOG = logging.getLogger('nova.scheduler.api') @@ -83,7 +87,8 @@ def _wrap_method(function, self): def _process(func, zone): - """Worker stub for green thread pool""" + """Worker stub for green thread pool. Give the worker + an authenticated nova client and zone info.""" nova = client.OpenStackClient(zone.username, zone.password, zone.api_url) nova.authenticate() @@ -91,36 +96,42 @@ def _process(func, zone): def child_zone_helper(zone_list, func): + """Fire off a command to each zone in the list.""" green_pool = greenpool.GreenPool() return [result for result in green_pool.imap( _wrap_method(_process, func), zone_list)] -def _issue_novaclient_command(nova, zone, method_name, instance_id): - server = None +def _issue_novaclient_command(nova, zone, collection, method_name, \ + item_id): + """Use novaclient to issue command to a single child zone. + One of these will be run in parallel for each child zone.""" + item = None try: - manager = getattr(nova, "servers") - if isinstance(instance_id, int) or instance_id.isdigit(): - server = manager.get(int(instance_id)) + manager = getattr(nova, collection) + if isinstance(item_id, int) or item_id.isdigit(): + item = manager.get(int(item_id)) else: - server = manager.find(name=instance_id) + item = manager.find(name=item_id) except novaclient.NotFound: url = zone.api_url - LOG.debug(_("Instance %(instance_id)s not found on '%(url)s'" % + LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % locals())) return - return getattr(server, method_name)() + return getattr(item, method_name)() -def wrap_novaclient_function(f, method_name, instance_id): +def wrap_novaclient_function(f, collection, method_name, item_id): + """Appends collection, method_name and item_id to the incoming + (nova, zone) call from child_zone_helper.""" def inner(nova, zone): - return f(nova, zone, method_name, instance_id) + return f(nova, zone, collection, method_name, item_id) return inner -class reroute_if_not_found(object): +class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query can't find anything. @@ -130,19 +141,32 @@ class reroute_if_not_found(object): def __call__(self, f): def wrapped_f(*args, **kwargs): - LOG.debug("***REROUTE-3: %s / %s" % (args, kwargs)) - context = args[1] - instance_id = args[2] + collection, context, item_id = \ + self.get_collection_context_and_id() try: return f(*args, **kwargs) except exception.InstanceNotFound, e: - LOG.debug(_("Instance %(instance_id)s not found " + LOG.debug(_("Instance %(item_id)s not found " "locally: '%(e)s'" % locals())) + if not FLAGS.enable_zone_routing: + raise + zones = db.zone_get_all(context) + if not zones: + raise + result = child_zone_helper(zones, wrap_novaclient_function(_issue_novaclient_command, - self.method_name, instance_id)) + collection, self.method_name, item_id)) LOG.debug("***REROUTE: %s" % result) - return result + return self.unmarshall_result(result) return wrapped_f + + def get_collection_context_and_id(self, args): + """Returns a tuple of (novaclient collection name, security + context and resource id. Derived class should override this.""" + return ("servers", args[1], args[2]) + + def unmarshall_result(self, result): + return result -- cgit From 70e8b431334989ad067f0a5543aea408b7186c5c Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 18 Mar 2011 10:34:08 -0400 Subject: Fixed 'Undefined variable' errors generated by pylint (E0602). --- nova/api/openstack/accounts.py | 7 ++++--- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 5 +++-- nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py | 2 -- nova/virt/fake.py | 2 +- nova/virt/libvirt_conn.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 2510ffb61..86066fa20 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -14,6 +14,7 @@ # under the License. import common +import webob.exc from nova import exception from nova import flags @@ -51,10 +52,10 @@ class Controller(wsgi.Controller): raise exception.NotAuthorized(_("Not admin user.")) def index(self, req): - raise faults.Fault(exc.HTTPNotImplemented()) + raise faults.Fault(webob.exc.HTTPNotImplemented()) def detail(self, req): - raise faults.Fault(exc.HTTPNotImplemented()) + raise faults.Fault(webob.exc.HTTPNotImplemented()) def show(self, req, id): """Return data about the given account id""" @@ -69,7 +70,7 @@ class Controller(wsgi.Controller): def create(self, req): """We use update with create-or-update semantics because the id comes from an external source""" - raise faults.Fault(exc.HTTPNotImplemented()) + raise faults.Fault(webob.exc.HTTPNotImplemented()) def update(self, req, id): """This is really create or update.""" diff --git a/nova/compute/api.py b/nova/compute/api.py index 32577af82..058b600bf 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -479,7 +479,7 @@ class API(base.Base): self._cast_compute_message('confirm_resize', context, instance_id, migration_ref['source_compute'], params=params) - self.db.migration_update(context, migration_id, + self.db.migration_update(context, migration_ref['id'], {'status': 'confirmed'}) self.db.instance_update(context, instance_id, {'host': migration_ref['dest_compute'], }) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 9d9b86c1d..98e6f938a 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2204,7 +2204,7 @@ def migration_get(context, id, session=None): filter_by(id=id).first() if not result: raise exception.NotFound(_("No migration found with id %s") - % migration_id) + % id) return result @@ -2216,7 +2216,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") - % migration_id) + % id) return result @@ -2427,6 +2427,7 @@ def zone_create(context, values): @require_admin_context def zone_update(context, zone_id, values): + session = get_session() zone = session.query(models.Zone).filter_by(id=zone_id).first() if not zone: raise exception.NotFound(_("No zone with id %(zone_id)s") % locals()) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py b/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py index 66609054e..8b962bf7f 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py @@ -55,7 +55,6 @@ def upgrade(migrate_engine): try: instance_types.create() except Exception: - logging.info(repr(table)) logging.exception('Exception while creating instance_types table') raise @@ -76,7 +75,6 @@ def upgrade(migrate_engine): 'local_gb': values["local_gb"], 'flavorid': values["flavorid"]}) except Exception: - logging.info(repr(table)) logging.exception('Exception while seeding instance_types table') raise diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 3a06284a1..451760721 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -323,7 +323,7 @@ class FakeConnection(object): Note that this function takes an instance ID, not a compute.service.Instance, so that it can be called by compute.monitor. """ - return [0L, 0L, 0L, 0L, null] + return [0L, 0L, 0L, 0L, None] def interface_stats(self, instance_name, iface_id): """ diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e80b9fbdf..8a59c5bba 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -46,7 +46,7 @@ import time import uuid from xml.dom import minidom - +from eventlet import greenthread from eventlet import tpool from eventlet import semaphore -- cgit From 047bff904817838279199a7099023b505e35343f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 07:43:42 -0700 Subject: whoopsy --- nova/compute/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 96538dd00..270600664 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -343,7 +343,7 @@ class API(base.Base): rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) - @scheduler_api.reroute("delete") + @scheduler_api.reroute_compute("delete") def delete(self, context, instance_id): LOG.debug(_("Going to try to terminate %s"), instance_id) try: -- cgit From 930d7bf1987c1b270ec0e456f982efb70527ed15 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 07:47:23 -0700 Subject: whoopsy2 --- nova/scheduler/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 90b92d7ed..b639ae786 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -142,7 +142,7 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): collection, context, item_id = \ - self.get_collection_context_and_id() + self.get_collection_context_and_id(args) try: return f(*args, **kwargs) except exception.InstanceNotFound, e: -- cgit From 12ffa884c07b55c982a1ad60a94e72c955db81c3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 09:02:36 -0700 Subject: fixed up novaclient usage to include managers --- nova/scheduler/api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index b639ae786..7efc28072 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -17,14 +17,14 @@ Handles all requests relating to schedulers. """ +import novaclient + from nova import db from nova import exception from nova import flags from nova import log as logging from nova import rpc -import novaclient.client as client - from eventlet import greenpool FLAGS = flags.FLAGS @@ -80,7 +80,7 @@ class API(object): def _wrap_method(function, self): - """Wrap method to supply 'self'.""" + """Wrap method to supply self.""" def _wrap(*args, **kwargs): return function(self, *args, **kwargs) return _wrap @@ -89,8 +89,7 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" - nova = client.OpenStackClient(zone.username, zone.password, - zone.api_url) + nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) nova.authenticate() return func(nova, zone) @@ -134,8 +133,7 @@ def wrap_novaclient_function(f, collection, method_name, item_id): class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query - can't find anything. - """ + can't find anything.""" def __init__(self, method_name): self.method_name = method_name -- cgit From ef33d6bde27276fb4c93ed6bbcb972977f03a370 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 09:21:08 -0700 Subject: results --- nova/api/openstack/servers.py | 2 +- nova/scheduler/api.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 9f14a6b82..49f714d47 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -130,7 +130,7 @@ class Controller(wsgi.Controller): try: LOG.debug(_("***SHOW")) instance = self.compute_api.routing_get(req.environ['nova.context'], id) - LOG.debug(_("***SHOW")) + LOG.debug(_("***SHOW OUT %s" % instance)) return _translate_detail_keys(instance) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 7efc28072..6b0f804f9 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -118,7 +118,10 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ locals())) return - return getattr(item, method_name)() + LOG.debug("***CALLING CHILD ZONE") + result = getattr(item, method_name)() + LOG.debug("***CHILD ZONE GAVE %s", result) + return result def wrap_novaclient_function(f, collection, method_name, item_id): -- cgit From 705020cc4acded862633aa5e02d5bb46c88dbc51 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 11:46:27 -0700 Subject: api decorator --- nova/api/openstack/servers.py | 2 ++ nova/scheduler/api.py | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 49f714d47..17d620562 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -33,6 +33,7 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack +from nova.scheduler import api as scheduler_api LOG = logging.getLogger('server') @@ -125,6 +126,7 @@ class Controller(wsgi.Controller): res = [entity_maker(inst)['server'] for inst in limited_list] return dict(servers=res) + @scheduler_api.redirect_handler def show(self, req, id): """ Returns server details by server id """ try: diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6b0f804f9..f5df446b3 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -105,22 +105,25 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ item_id): """Use novaclient to issue command to a single child zone. One of these will be run in parallel for each child zone.""" - item = None + result = None try: manager = getattr(nova, collection) if isinstance(item_id, int) or item_id.isdigit(): - item = manager.get(int(item_id)) + result = manager.get(int(item_id)) else: - item = manager.find(name=item_id) + result = manager.find(name=item_id) except novaclient.NotFound: url = zone.api_url LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % locals())) return - LOG.debug("***CALLING CHILD ZONE") - result = getattr(item, method_name)() - LOG.debug("***CHILD ZONE GAVE %s", result) + if method_name.lower() not in ['get', 'find']: + LOG.debug("***CALLING CHILD ZONE") + m = getattr(item, method_name) + LOG.debug("***METHOD ATTR %s" % m) + result = getattr(item, method_name)() + LOG.debug("***CHILD ZONE GAVE %s", result) return result @@ -133,6 +136,14 @@ def wrap_novaclient_function(f, collection, method_name, item_id): return inner +class RedirectResult(exception.Error): + """Used to the HTTP API know that these results are pre-cooked + and they can be returned to the caller directly.""" + def __init__(self, results): + self.results = results + super(RedirectResult, self).__init__( + message=_("Uncaught Zone redirection exception")) + class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query @@ -161,7 +172,7 @@ class reroute_compute(object): wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) LOG.debug("***REROUTE: %s" % result) - return self.unmarshall_result(result) + raise RedirectResult(self.unmarshall_result(result)) return wrapped_f def get_collection_context_and_id(self, args): @@ -170,4 +181,14 @@ class reroute_compute(object): return ("servers", args[1], args[2]) def unmarshall_result(self, result): - return result + return [server.__dict__ for server in result] + + +def redirect_handler(f): + def new_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except RedirectResult, e: + LOG.debug("***CAUGHT REROUTE: %s" % e.results) + return e.results + return new_f -- cgit From 37f2c3036890f9bbfd88a369dfd590744256aaf9 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 12:00:35 -0700 Subject: works again. woo hoo --- nova/scheduler/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index f5df446b3..8b8457e8d 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -181,7 +181,13 @@ class reroute_compute(object): return ("servers", args[1], args[2]) def unmarshall_result(self, result): - return [server.__dict__ for server in result] + server = result[0].__dict__ + + for k in server.keys(): + if k[0] == '_' or k == 'manager': + del server[k] + + return dict(server=server) def redirect_handler(f): -- cgit From feb5c82e29303285d3f914c37116a59538fec28f Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 18 Mar 2011 12:23:57 -0700 Subject: fix ups --- nova/api/openstack/servers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 17d620562..86414fab2 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -137,6 +137,7 @@ class Controller(wsgi.Controller): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) + @scheduler_api.redirect_handler def delete(self, req, id): """ Destroys a server """ try: @@ -258,6 +259,7 @@ class Controller(wsgi.Controller): # if the original error is okay, just reraise it raise error + @scheduler_api.redirect_handler def update(self, req, id): """ Updates the server name or password """ if len(req.body) == 0: @@ -283,6 +285,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() + @scheduler_api.redirect_handler def action(self, req, id): """Multi-purpose method used to reboot, rebuild, or resize a server""" @@ -348,6 +351,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def lock(self, req, id): """ lock the instance with id @@ -363,6 +367,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def unlock(self, req, id): """ unlock the instance with id @@ -378,6 +383,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def get_lock(self, req, id): """ return the boolean state of (instance with id)'s lock @@ -392,6 +398,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def reset_network(self, req, id): """ Reset networking on an instance (admin only). @@ -406,6 +413,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def inject_network_info(self, req, id): """ Inject network info for an instance (admin only). @@ -420,6 +428,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def pause(self, req, id): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] @@ -431,6 +440,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def unpause(self, req, id): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] @@ -442,6 +452,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def suspend(self, req, id): """permit admins to suspend the server""" context = req.environ['nova.context'] @@ -453,6 +464,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def resume(self, req, id): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] @@ -464,6 +476,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def rescue(self, req, id): """Permit users to rescue the server.""" context = req.environ["nova.context"] @@ -475,6 +488,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def unrescue(self, req, id): """Permit users to unrescue the server.""" context = req.environ["nova.context"] @@ -486,6 +500,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def get_ajax_console(self, req, id): """ Returns a url to an instance's ajaxterm console. """ try: @@ -495,6 +510,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() + @scheduler_api.redirect_handler def diagnostics(self, req, id): """Permit Admins to retrieve server diagnostics.""" ctxt = req.environ["nova.context"] -- cgit -- cgit From 8f0b60f598c28b2f558f3ecdaa2f9604926393e6 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 07:49:58 -0700 Subject: remove scheduler.api.API. naming changes. --- nova/api/openstack/zones.py | 4 ++-- nova/manager.py | 2 +- nova/rpc.py | 2 +- nova/scheduler/api.py | 43 +++++++++++++++------------------- nova/tests/api/openstack/test_zones.py | 8 +++---- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index ebfc7743c..d129cf34f 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -53,7 +53,7 @@ class Controller(wsgi.Controller): """Return all zones in brief""" # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... - items = api.API.get_zone_list(req.environ['nova.context']) + items = api.get_zone_list(req.environ['nova.context']) if not items: items = db.zone_get_all(req.environ['nova.context']) @@ -68,7 +68,7 @@ class Controller(wsgi.Controller): def info(self, req): """Return name and capabilities for this zone.""" - items = api.API.get_zone_capabilities(req.environ['nova.context']) + items = api.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) caps = FLAGS.zone_capabilities diff --git a/nova/manager.py b/nova/manager.py index f384e3f0f..508f133ca 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -96,7 +96,7 @@ class SchedulerDependentManager(Manager): """Pass data back to the scheduler at a periodic interval""" if self.last_capabilities: logging.debug(_("Notifying Schedulers of capabilities ...")) - api.API.update_service_capabilities(context, self.service_name, + api.update_service_capabilities(context, self.service_name, self.host, self.last_capabilities) super(SchedulerDependentManager, self).periodic_tasks(context) diff --git a/nova/rpc.py b/nova/rpc.py index 4918c0b95..2e3cd9057 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -244,7 +244,7 @@ class FanoutPublisher(Publisher): self.exchange = "%s_fanout" % topic self.queue = "%s_fanout" % topic self.durable = False - LOG.info(_("Writing to '%(exchange)s' fanout exchange"), + LOG.info(_("Creating '%(exchange)s' fanout exchange"), dict(exchange=self.exchange)) super(FanoutPublisher, self).__init__(connection=connection) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index b6d27dacc..e2cf3b6a3 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -40,30 +40,25 @@ def _call_scheduler(method, context, params=None): return rpc.call(context, queue, kwargs) -class API(object): - """API for interacting with the scheduler.""" +def get_zone_list(context): + """Return a list of zones assoicated with this zone.""" + items = _call_scheduler('get_zone_list', context) + for item in items: + item['api_url'] = item['api_url'].replace('\\/', '/') + return items - @classmethod - def get_zone_list(cls, context): - """Return a list of zones assoicated with this zone.""" - items = _call_scheduler('get_zone_list', context) - for item in items: - item['api_url'] = item['api_url'].replace('\\/', '/') - return items - @classmethod - def get_zone_capabilities(cls, context, service=None): - """Returns a dict of key, value capabilities for this zone, - or for a particular class of services running in this zone.""" - return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) +def get_zone_capabilities(context, service=None): + """Returns a dict of key, value capabilities for this zone, + or for a particular class of services running in this zone.""" + return _call_scheduler('get_zone_capabilities', context=context, + params=dict(service=service)) - @classmethod - def update_service_capabilities(cls, context, service_name, host, - capabilities): - """Send an update to all the scheduler services informing them - of the capabilities of this service.""" - kwargs = dict(method='update_service_capabilities', - args=dict(service_name=service_name, host=host, - capabilities=capabilities)) - return rpc.fanout_cast(context, 'scheduler', kwargs) + +def update_service_capabilities(context, service_name, host, capabilities): + """Send an update to all the scheduler services informing them + of the capabilities of this service.""" + kwargs = dict(method='update_service_capabilities', + args=dict(service_name=service_name, host=host, + capabilities=capabilities)) + return rpc.fanout_cast(context, 'scheduler', kwargs) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 12d39fd29..a3f191aaa 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -75,7 +75,7 @@ def zone_get_all_db(context): ] -def zone_caps(method, context, params): +def zone_capabilities(method, context, params): return dict() @@ -98,13 +98,13 @@ class ZonesTest(test.TestCase): self.stubs.Set(nova.db, 'zone_delete', zone_delete) self.old_zone_name = FLAGS.zone_name - self.old_zone_caps = FLAGS.zone_capabilities + self.old_zone_capabilities = FLAGS.zone_capabilities def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin FLAGS.zone_name = self.old_zone_name - FLAGS.zone_capabilities = self.old_zone_caps + FLAGS.zone_capabilities = self.old_zone_capabilities super(ZonesTest, self).tearDown() def test_get_zone_list_scheduler(self): @@ -179,7 +179,7 @@ class ZonesTest(test.TestCase): def test_zone_info(self): FLAGS.zone_name = 'darksecret' FLAGS.zone_capabilities = ['cap1=a;b', 'cap2=c;d'] - self.stubs.Set(api, '_call_scheduler', zone_caps) + self.stubs.Set(api, '_call_scheduler', zone_capabilities) body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/info') -- cgit From 012c94e5304b9e00477409b92bf73f4316b19260 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 21 Mar 2011 13:56:55 -0400 Subject: Fix limit unit tests (reconciles w/ trunk changes). --- nova/tests/api/openstack/test_extensions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 11ed61e0d..d1f8c659e 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -190,8 +190,6 @@ class ResponseExtensionTest(unittest.TestCase): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} - fakes.stub_out_networking(self.stubs) - fakes.stub_out_rate_limiting(self.stubs) fakes.stub_out_auth(self.stubs) self.context = context.get_admin_context() -- cgit From f988df6c6f29d6c885d44c6768297aaf489faf34 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 21 Mar 2011 13:58:39 -0400 Subject: Fix pep8 issues in nova/api/openstack/extensions.py. --- nova/api/openstack/extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 23181d2a6..f881dbde7 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -69,13 +69,13 @@ class ResponseExtensionController(wsgi.Controller): content_type = req.best_match_content_type() # currently response handlers are un-ordered for handler in self.handlers: - res=handler(res) + res = handler(res) try: body = res.body headers = res.headers except AttributeError: body = self._serialize(res, content_type) - headers={"Content-Type": content_type} + headers = {"Content-Type": content_type} res = webob.Response() res.body = body res.headers = headers -- cgit From fbd94f236adaa906fcc9c90de94e491e3d75653b Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 21 Mar 2011 13:59:26 -0400 Subject: Added copyright header. --- nova/tests/api/openstack/extensions/foxinsocks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nova/tests/api/openstack/extensions/foxinsocks.py b/nova/tests/api/openstack/extensions/foxinsocks.py index fa979c7b5..249dd81bf 100644 --- a/nova/tests/api/openstack/extensions/foxinsocks.py +++ b/nova/tests/api/openstack/extensions/foxinsocks.py @@ -1,3 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import json from nova import wsgi -- cgit From ff2d6dc656c03b8aeab5e50c5d39ca9dcde9b9b1 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 21 Mar 2011 14:00:39 -0400 Subject: Updated comment per the extension naming convention we actually use. --- nova/api/openstack/extensions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index f881dbde7..9d98d849a 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -303,9 +303,9 @@ class ExtensionManager(object): def _load_extensions(self): """ Load extensions from the configured path. The extension name is - constructed from the camel cased module_name + 'Extension'. If your - extension module was named widgets.py the extension class within that - module should be 'WidgetsExtension'. + constructed from the module_name. If your extension module was named + widgets.py the extension class within that module should be + 'Widgets'. See nova/tests/api/openstack/extensions/foxinsocks.py for an example extension implementation. -- cgit From 7976fb08d89a8e8b6bf8c276a50e30ae11584ce3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 11:01:34 -0700 Subject: more robust extraction of arguments --- nova/scheduler/api.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index c7acd3548..935e7b366 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -149,7 +149,7 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): collection, context, item_id = \ - self.get_collection_context_and_id(args) + self.get_collection_context_and_id(args, kwargs) try: return f(*args, **kwargs) except exception.InstanceNotFound, e: @@ -170,10 +170,16 @@ class reroute_compute(object): raise RedirectResult(self.unmarshall_result(result)) return wrapped_f - def get_collection_context_and_id(self, args): + def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" - return ("servers", args[1], args[2]) + context = kwargs.get('context', None) + instance_id = kwargs.get('instance_id', None) + if len(args) > 0 and not context: + context = args[1] + if len(args) > 1 and not instance_id: + context = args[2] + return ("servers", context, instance_id) def unmarshall_result(self, result): server = result[0].__dict__ -- cgit From b1def6b2b104a143b7491cef9a01babe9ab3e75d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 11:07:19 -0700 Subject: pep8 --- nova/scheduler/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 935e7b366..aebfe1770 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -127,7 +127,7 @@ def wrap_novaclient_function(f, collection, method_name, item_id): (nova, zone) call from child_zone_helper.""" def inner(nova, zone): return f(nova, zone, collection, method_name, item_id) - + return inner @@ -139,6 +139,7 @@ class RedirectResult(exception.Error): super(RedirectResult, self).__init__( message=_("Uncaught Zone redirection exception")) + class reroute_compute(object): """Decorator used to indicate that the method should delegate the call the child zones if the db query @@ -158,7 +159,7 @@ class reroute_compute(object): if not FLAGS.enable_zone_routing: raise - + zones = db.zone_get_all(context) if not zones: raise -- cgit From ffd2bc759af4f53019838bf20a4f016a566fbbd6 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 21 Mar 2011 13:21:26 -0500 Subject: Added XenAPI rescue unit tests --- nova/tests/test_xenapi.py | 12 ++++++++++++ nova/tests/xenapi/stubs.py | 19 +++++++++++++++++++ nova/virt/xenapi/vmops.py | 11 ++++++++--- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 66a973a78..e54ffe712 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -186,6 +186,7 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_stream_disk(self.stubs) stubs.stubout_is_vdi_pv(self.stubs) self.stubs.Set(VMOps, 'reset_network', reset_network) + stubs.stub_out_vm_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) self.conn = xenapi_conn.get_connection(False) @@ -369,6 +370,17 @@ class XenAPIVMTestCase(test.TestCase): self.assertEquals(vif_rec['qos_algorithm_params']['kbps'], str(4 * 1024)) + def test_rescue(self): + instance = self._create_instance() + conn = xenapi_conn.get_connection(False) + conn.rescue(instance, None) + + def test_unrescue(self): + instance = self._create_instance() + conn = xenapi_conn.get_connection(False) + # Ensure that it will not unrescue a non-rescued instance. + self.assertRaises(Exception, conn.unrescue, instance, None) + def tearDown(self): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 70d46a1fb..a0370a2ec 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -185,6 +185,25 @@ class FakeSessionForVMTests(fake.SessionBase): pass +def stub_out_vm_methods(stubs): + def fake_shutdown(self, inst, vm, method="clean"): + pass + + def fake_acquire_bootlock(self, vm): + pass + + def fake_release_bootlock(self, vm): + pass + + def fake_spawn_rescue(self, inst): + pass + + stubs.Set(vmops.VMOps, "_shutdown", fake_shutdown) + stubs.Set(vmops.VMOps, "_acquire_bootlock", fake_acquire_bootlock) + stubs.Set(vmops.VMOps, "_release_bootlock", fake_release_bootlock) + stubs.Set(vmops.VMOps, "spawn_rescue", fake_spawn_rescue) + + class FakeSessionForVolumeTests(fake.SessionBase): """ Stubs out a XenAPISession for Volume tests """ def __init__(self, uri): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 29f162ad1..18eec9544 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -85,6 +85,11 @@ class VMOps(object): vdi_uuid = self.create_disk(instance) self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) + def spawn_rescue(self, instance): + """Break rescue's spawn into separate method for unit tests""" + vdi_uuid = self.create_disk(instance) + self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) + def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" instance_name = instance.name @@ -600,7 +605,7 @@ class VMOps(object): """ rescue_vm_ref = VMHelper.lookup(self._session, - instance.name + "-rescue") + str(instance.name) + "-rescue") if rescue_vm_ref: raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) @@ -610,7 +615,7 @@ class VMOps(object): self._acquire_bootlock(vm_ref) instance._rescue = True - self.spawn(instance) + self.spawn_rescue(instance) rescue_vm_ref = self._get_vm_opaque_ref(instance) vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] @@ -628,7 +633,7 @@ class VMOps(object): """ rescue_vm_ref = VMHelper.lookup(self._session, - instance.name + "-rescue") + str(instance.name) + "-rescue") if not rescue_vm_ref: raise exception.NotFound(_( -- cgit From 7cc28482a4ebeeb5dfa44c9e1c37bb135c1c66be Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 21 Mar 2011 17:00:08 -0500 Subject: Remove dupe'd code --- nova/virt/xenapi/vmops.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 18eec9544..a5f43200d 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -86,9 +86,8 @@ class VMOps(object): self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) def spawn_rescue(self, instance): - """Break rescue's spawn into separate method for unit tests""" - vdi_uuid = self.create_disk(instance) - self._spawn_with_disk(instance, vdi_uuid=vdi_uuid) + """Spawn a rescue instance""" + self.spawn(instance) def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" -- cgit From 8db9e359d85cbf8e9afab2260759543b1717c3f9 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 21 Mar 2011 17:56:30 -0500 Subject: Remove _get_vm_opaque_ref() calls in rescue/unrescue --- nova/virt/xenapi/vmops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index a5f43200d..c2ead3f57 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -609,13 +609,13 @@ class VMOps(object): raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) - vm_ref = self._get_vm_opaque_ref(instance) + vm_ref = VMHelper.lookup(self._session, instance.name) self._shutdown(instance, vm_ref) self._acquire_bootlock(vm_ref) instance._rescue = True self.spawn_rescue(instance) - rescue_vm_ref = self._get_vm_opaque_ref(instance) + rescue_vm_ref = VMHelper.lookup(self._session, instance.name) vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] vdi_ref = self._session.get_xenapi().VBD.get_record(vbd_ref)["VDI"] @@ -638,7 +638,7 @@ class VMOps(object): raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) - original_vm_ref = self._get_vm_opaque_ref(instance) + original_vm_ref = VMHelper.lookup(self._session, instance.name) vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) instance._rescue = False -- cgit From 08d06d1219a00b90ae211fb44fc7e33ba71c7a76 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 18:16:35 -0700 Subject: better comments. First redirect test --- nova/scheduler/api.py | 49 +++++++++++++++++++++++-------- nova/tests/test_scheduler.py | 70 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 12 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index aebfe1770..ff7e21679 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -84,13 +84,18 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" + LOG.debug("*** PROCESS %s/%s" % (func, zone)) nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) nova.authenticate() return func(nova, zone) def child_zone_helper(zone_list, func): - """Fire off a command to each zone in the list.""" + """Fire off a command to each zone in the list. + The return is [novaclient return objects] from each child zone. + For example, if you are calling server.pause(), the list will + be whatever the response from server.pause() is. One entry + per child zone called.""" green_pool = greenpool.GreenPool() return [result for result in green_pool.imap( _wrap_method(_process, func), zone_list)] @@ -103,6 +108,7 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ result = None try: manager = getattr(nova, collection) + LOG.debug("***MANAGER %s" % manager) if isinstance(item_id, int) or item_id.isdigit(): result = manager.get(int(item_id)) else: @@ -115,9 +121,9 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ if method_name.lower() not in ['get', 'find']: LOG.debug("***CALLING CHILD ZONE") - m = getattr(item, method_name) + m = getattr(result, method_name) LOG.debug("***METHOD ATTR %s" % m) - result = getattr(item, method_name)() + result = getattr(result, method_name)() LOG.debug("***CHILD ZONE GAVE %s", result) return result @@ -152,6 +158,7 @@ class reroute_compute(object): collection, context, item_id = \ self.get_collection_context_and_id(args, kwargs) try: + # Call the original function ... return f(*args, **kwargs) except exception.InstanceNotFound, e: LOG.debug(_("Instance %(item_id)s not found " @@ -164,32 +171,50 @@ class reroute_compute(object): if not zones: raise + # Ask the children to provide an answer ... result = child_zone_helper(zones, wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) LOG.debug("***REROUTE: %s" % result) + # Scrub the results and raise another exception + # so the API layers can bail out gracefully ... raise RedirectResult(self.unmarshall_result(result)) return wrapped_f def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" + LOG.debug("***COLLECT: %s/%s" % (args, kwargs)) context = kwargs.get('context', None) instance_id = kwargs.get('instance_id', None) if len(args) > 0 and not context: context = args[1] if len(args) > 1 and not instance_id: - context = args[2] + instance_id = args[2] return ("servers", context, instance_id) - def unmarshall_result(self, result): - server = result[0].__dict__ - - for k in server.keys(): - if k[0] == '_' or k == 'manager': - del server[k] - - return dict(server=server) + def unmarshall_result(self, zone_responses): + """Result is a list of responses from each child zone. + Each decorator derivation is responsible to turning this + into a format expected by the calling method. For + example, this one is expected to return a single Server + dict {'server':{k:v}}. Others may return a list of them, like + {'servers':[{k,v}]}""" + reduced_response = [] + for zone_response in zone_responses: + if not zone_response: + continue + + server = zone_response.__dict__ + + for k in server.keys(): + if k[0] == '_' or k == 'manager': + del server[k] + + reduced_response.append(dict(server=server)) + if reduced_response: + return reduced_response[0] # first for now. + return {} def redirect_handler(f): diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 244e43bd9..50e2429ba 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -21,6 +21,8 @@ Tests For Scheduler import datetime import mox +import stubout +import webob from mox import IgnoreArg from nova import context @@ -32,6 +34,7 @@ from nova import test from nova import rpc from nova import utils from nova.auth import manager as auth_manager +from nova.scheduler import api from nova.scheduler import manager from nova.scheduler import driver from nova.compute import power_state @@ -937,3 +940,70 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) + + +class FakeZone(object): + def __init__(self, api_url, username, password): + self.api_url = api_url + self.username = username + self.password = password + +def zone_get_all(context): + return [ + FakeZone('http://example.com', 'bob', 'xxx'), + ] + + +def go_boom(self, context, instance): + raise exception.InstanceNotFound("boom message", instance) + + +def fake_openstack_init(self, username, password, api): + servers=[] + + +def fake_auth(self): + pass + +class FakeServer: + def foo(self): + pass + +class FakeManager: + def get(self, id): + return FakeServer() + +class FakeOpenStack: + + def __init__(self, username, api, auth): + self.servers = FakeManager() + + def authenticate(self): + pass + + +class ZoneRedirectTest(test.TestCase): + def setUp(self): + super(ZoneRedirectTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + + self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) + self.stubs.Set(db, 'zone_get_all', zone_get_all) + + self.enable_zone_routing = FLAGS.enable_zone_routing + FLAGS.enable_zone_routing = True + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.enable_zone_routing = self.enable_zone_routing + super(ZoneRedirectTest, self).tearDown() + + def test_trap_found_locally(self): + decorator = api.reroute_compute("foo") + try: + wrapper = decorator(go_boom) + result = wrapper(None, None, 1) # self, context, id + except api.RedirectResult, e: + pass + + -- cgit From 380731ce71e8909615da6138bb7d5e7226e375ac Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 18:56:59 -0700 Subject: better comments. First redirect test --- nova/tests/test_scheduler.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 50e2429ba..6d55cad04 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -958,23 +958,23 @@ def go_boom(self, context, instance): raise exception.InstanceNotFound("boom message", instance) -def fake_openstack_init(self, username, password, api): - servers=[] +class FakeServer(object): + def __init__(self): + self.name = 'myserver' + self.kvm = 'kvm' + self.manager = 100 + self._hidden = True - -def fake_auth(self): - pass - -class FakeServer: def foo(self): - pass + return None + -class FakeManager: +class FakeManager(object): def get(self, id): return FakeServer() -class FakeOpenStack: +class FakeOpenStack: def __init__(self, username, api, auth): self.servers = FakeManager() @@ -987,7 +987,6 @@ class ZoneRedirectTest(test.TestCase): super(ZoneRedirectTest, self).setUp() self.stubs = stubout.StubOutForTesting() - self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) self.stubs.Set(db, 'zone_get_all', zone_get_all) self.enable_zone_routing = FLAGS.enable_zone_routing @@ -999,11 +998,12 @@ class ZoneRedirectTest(test.TestCase): super(ZoneRedirectTest, self).tearDown() def test_trap_found_locally(self): + self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) decorator = api.reroute_compute("foo") try: wrapper = decorator(go_boom) result = wrapper(None, None, 1) # self, context, id except api.RedirectResult, e: - pass + self.assertTrue(e.results, {}) -- cgit From 8303d0f280a7bfbc5c5fb128465549b03badc1f1 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 21:41:41 -0700 Subject: routing test coverage --- nova/scheduler/api.py | 18 +++---- nova/tests/test_scheduler.py | 121 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 105 insertions(+), 34 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index ff7e21679..4f189fe37 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -84,7 +84,6 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" - LOG.debug("*** PROCESS %s/%s" % (func, zone)) nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) nova.authenticate() return func(nova, zone) @@ -108,7 +107,6 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ result = None try: manager = getattr(nova, collection) - LOG.debug("***MANAGER %s" % manager) if isinstance(item_id, int) or item_id.isdigit(): result = manager.get(int(item_id)) else: @@ -117,14 +115,10 @@ def _issue_novaclient_command(nova, zone, collection, method_name, \ url = zone.api_url LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" % locals())) - return + return None if method_name.lower() not in ['get', 'find']: - LOG.debug("***CALLING CHILD ZONE") - m = getattr(result, method_name) - LOG.debug("***METHOD ATTR %s" % m) result = getattr(result, method_name)() - LOG.debug("***CHILD ZONE GAVE %s", result) return result @@ -172,19 +166,22 @@ class reroute_compute(object): raise # Ask the children to provide an answer ... - result = child_zone_helper(zones, + result = self._call_child_zones(zones, wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) - LOG.debug("***REROUTE: %s" % result) # Scrub the results and raise another exception # so the API layers can bail out gracefully ... raise RedirectResult(self.unmarshall_result(result)) return wrapped_f + def _call_child_zones(self, zones, function): + """Ask the child zones to perform this operation. + Broken out for testing.""" + return child_zone_helper(zones, function) + def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" - LOG.debug("***COLLECT: %s/%s" % (args, kwargs)) context = kwargs.get('context', None) instance_id = kwargs.get('instance_id', None) if len(args) > 0 and not context: @@ -222,6 +219,5 @@ def redirect_handler(f): try: return f(*args, **kwargs) except RedirectResult, e: - LOG.debug("***CAUGHT REROUTE: %s" % e.results) return e.results return new_f diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 6d55cad04..0aebd0380 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -21,6 +21,7 @@ Tests For Scheduler import datetime import mox +import novaclient.exceptions import stubout import webob @@ -954,34 +955,33 @@ def zone_get_all(context): ] -def go_boom(self, context, instance): - raise exception.InstanceNotFound("boom message", instance) +class FakeRerouteCompute(api.reroute_compute): + def _call_child_zones(self, zones, function): + return [ ] + + def get_collection_context_and_id(self, args, kwargs): + return ("servers", None, 1) + def unmarshall_result(self, zone_responses): + return dict(magic="found me") -class FakeServer(object): - def __init__(self): - self.name = 'myserver' - self.kvm = 'kvm' - self.manager = 100 - self._hidden = True - def foo(self): - return None +def go_boom(self, context, instance): + raise exception.InstanceNotFound("boom message", instance) -class FakeManager(object): - def get(self, id): - return FakeServer() +def found_instance(self, context, instance): + return dict(name='myserver') -class FakeOpenStack: - def __init__(self, username, api, auth): - self.servers = FakeManager() +class FakeResource(object): + def __init__(self, attribute_dict): + for k, v in attribute_dict.iteritems(): + setattr(self, k, v) - def authenticate(self): + def pause(self): pass - class ZoneRedirectTest(test.TestCase): def setUp(self): super(ZoneRedirectTest, self).setUp() @@ -998,12 +998,87 @@ class ZoneRedirectTest(test.TestCase): super(ZoneRedirectTest, self).tearDown() def test_trap_found_locally(self): - self.stubs.Set(api.novaclient, 'OpenStack', FakeOpenStack) - decorator = api.reroute_compute("foo") + decorator = FakeRerouteCompute("foo") try: - wrapper = decorator(go_boom) - result = wrapper(None, None, 1) # self, context, id + result = decorator(found_instance)(None, None, 1) except api.RedirectResult, e: - self.assertTrue(e.results, {}) + self.fail(_("Successful database hit should succeed")) + def test_trap_not_found_locally(self): + decorator = FakeRerouteCompute("foo") + try: + result = decorator(go_boom)(None, None, 1) + except api.RedirectResult, e: + self.assertEquals(e.results['magic'], 'found me') + def test_get_collection_context_and_id(self): + decorator = api.reroute_compute("foo") + self.assertEquals(decorator.get_collection_context_and_id( + (None, 10, 20), {}), ("servers", 10, 20)) + self.assertEquals(decorator.get_collection_context_and_id( + (None, 11,), dict(instance_id=21)), ("servers", 11, 21)) + self.assertEquals(decorator.get_collection_context_and_id( + (None,), dict(context=12, instance_id=22)), ("servers", 12, 22)) + + def test_unmarshal_single_server(self): + decorator = api.reroute_compute("foo") + self.assertEquals(decorator.unmarshall_result([]), {}) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(a=1, b=2)),]), + dict(server=dict(a=1, b=2))) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(a=1, _b=2)),]), + dict(server=dict(a=1,))) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(a=1, manager=2)),]), + dict(server=dict(a=1,))) + self.assertEquals(decorator.unmarshall_result( + [FakeResource(dict(_a=1, manager=2)),]), + dict(server={})) + +class FakeServerCollection(object): + def get(self, instance_id): + return FakeResource(dict(a=10, b=20)) + + def find(self, name): + return FakeResource(dict(a=11, b=22)) + +class FakeEmptyServerCollection(object): + def get(self, f): + raise novaclient.NotFound(1) + + def find(self, name): + raise novaclient.NotFound(2) + +class FakeNovaClient(object): + def __init__(self, collection): + self.servers = collection + +class DynamicNovaClientTest(test.TestCase): + def test_issue_novaclient_command_found(self): + zone = FakeZone('http://example.com', 'bob', 'xxx') + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeServerCollection()), + zone, "servers", "get", 100).a, 10) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeServerCollection()), + zone, "servers", "find", "name").b, 22) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeServerCollection()), + zone, "servers", "pause", 100), None) + + def test_issue_novaclient_command_not_found(self): + zone = FakeZone('http://example.com', 'bob', 'xxx') + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeEmptyServerCollection()), + zone, "servers", "get", 100), None) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeEmptyServerCollection()), + zone, "servers", "find", "name"), None) + + self.assertEquals(api._issue_novaclient_command( + FakeNovaClient(FakeEmptyServerCollection()), + zone, "servers", "any", "name"), None) -- cgit From e74482f30c602530313faf15e0d429acefee7bde Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 21:47:58 -0700 Subject: routing test coverage --- nova/tests/test_scheduler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 0aebd0380..8434f5a43 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -1008,9 +1008,18 @@ class ZoneRedirectTest(test.TestCase): decorator = FakeRerouteCompute("foo") try: result = decorator(go_boom)(None, None, 1) + self.assertFail(_("Should have rerouted.")) except api.RedirectResult, e: self.assertEquals(e.results['magic'], 'found me') + def test_routing_flags(self): + FLAGS.enable_zone_routing = False + decorator = FakeRerouteCompute("foo") + try: + result = decorator(go_boom)(None, None, 1) + except exception.InstanceNotFound, e: + self.assertEquals(e.message, 'boom message') + def test_get_collection_context_and_id(self): decorator = api.reroute_compute("foo") self.assertEquals(decorator.get_collection_context_and_id( -- cgit From 65482f5d9513c3dda64171d0460001e299be9673 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 21 Mar 2011 21:51:14 -0700 Subject: added zone routing flag test --- nova/tests/test_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 8434f5a43..277ffe367 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -1017,6 +1017,7 @@ class ZoneRedirectTest(test.TestCase): decorator = FakeRerouteCompute("foo") try: result = decorator(go_boom)(None, None, 1) + self.assertFail(_("Should have thrown exception.")) except exception.InstanceNotFound, e: self.assertEquals(e.message, 'boom message') -- cgit From 97e8f300af824145c8b92949ccbdfe81c0d7ca95 Mon Sep 17 00:00:00 2001 From: Josh Kleinpeter Date: Tue, 22 Mar 2011 12:33:34 -0500 Subject: Changed default for disabled on service_get_all to None. Changed calls to service_get_all so that the results should still be as they previously were. --- bin/nova-manage | 2 +- nova/api/ec2/admin.py | 2 +- nova/api/ec2/cloud.py | 4 ++-- nova/db/api.py | 2 +- nova/db/sqlalchemy/api.py | 13 ++++++++----- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 69cbf6f95..c5a4bea7e 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -610,7 +610,7 @@ class ServiceCommands(object): args: [host] [service]""" ctxt = context.get_admin_context() now = datetime.datetime.utcnow() - services = db.service_get_all(ctxt) + db.service_get_all(ctxt, True) + services = db.service_get_all(ctxt) if host: services = [s for s in services if s['host'] == host] if service: diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index d9a4ef999..3ae29d8ce 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -299,7 +299,7 @@ class AdminController(object): * Volume (up, down, None) * Volume Count """ - services = db.service_get_all(context) + services = db.service_get_all(context, False) now = datetime.datetime.utcnow() hosts = [] rv = [] diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index e257e44e7..2afcea77c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -196,7 +196,7 @@ class CloudController(object): def _describe_availability_zones(self, context, **kwargs): ctxt = context.elevated() - enabled_services = db.service_get_all(ctxt) + enabled_services = db.service_get_all(ctxt, False) disabled_services = db.service_get_all(ctxt, True) available_zones = [] for zone in [service.availability_zone for service @@ -221,7 +221,7 @@ class CloudController(object): rv = {'availabilityZoneInfo': [{'zoneName': 'nova', 'zoneState': 'available'}]} - services = db.service_get_all(context) + services = db.service_get_all(context, False) now = datetime.datetime.utcnow() hosts = [] for host in [service['host'] for service in services]: diff --git a/nova/db/api.py b/nova/db/api.py index add5bd83e..b3ca861e2 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -89,7 +89,7 @@ def service_get_by_host_and_topic(context, host, topic): return IMPL.service_get_by_host_and_topic(context, host, topic) -def service_get_all(context, disabled=False): +def service_get_all(context, disabled=None): """Get all services.""" return IMPL.service_get_all(context, disabled) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 3bf4f5eb8..321efe0e5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -143,12 +143,15 @@ def service_get(context, service_id, session=None): @require_admin_context -def service_get_all(context, disabled=False): +def service_get_all(context, disabled=None): session = get_session() - return session.query(models.Service).\ - filter_by(deleted=can_read_deleted(context)).\ - filter_by(disabled=disabled).\ - all() + query = session.query(models.Service).\ + filter_by(deleted=can_read_deleted(context)) + + if disabled is not None: + query = query.filter_by(disabled=disabled) + + return query.all() @require_admin_context -- cgit From 2a38aa7583be37ece6c42ba9307c2db0232dbed3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 22 Mar 2011 10:37:56 -0700 Subject: Whoops --- nova/api/openstack/servers.py | 5 +++-- nova/scheduler/api.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index be423c572..199d89c6d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -92,9 +92,10 @@ class Controller(wsgi.Controller): """ Returns server details by server id """ try: LOG.debug(_("***SHOW")) - instance = self.compute_api.get(req.environ['nova.context'], id) - builder = servers_views.get_view_builder(req) + instance = self.compute_api.routing_get( + req.environ['nova.context'], id) LOG.debug(_("***SHOW OUT %s" % instance)) + builder = servers_views.get_view_builder(req) return builder.build(instance, is_detail=True) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 4f189fe37..bd64f9b9b 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -149,8 +149,10 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): + LOG.debug(_("IN DECORATOR ...")) collection, context, item_id = \ self.get_collection_context_and_id(args, kwargs) + LOG.debug(_("IN DECORATOR 2...")) try: # Call the original function ... return f(*args, **kwargs) @@ -166,6 +168,7 @@ class reroute_compute(object): raise # Ask the children to provide an answer ... + LOG.debug(_("Asking child zones ...")) result = self._call_child_zones(zones, wrap_novaclient_function(_issue_novaclient_command, collection, self.method_name, item_id)) -- cgit From 209da18033a49062bbcfaf7739db5959be87b142 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 22 Mar 2011 20:36:49 -0700 Subject: pep8 and fixed up zone-list --- nova/api/openstack/servers.py | 2 -- nova/api/openstack/zones.py | 18 +++++++----------- nova/db/api.py | 4 ---- nova/scheduler/api.py | 24 ++++++++++++++++++++---- nova/scheduler/zone_manager.py | 5 +++-- nova/tests/test_scheduler.py | 40 +++++++++++++++++++++++----------------- 6 files changed, 53 insertions(+), 40 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 199d89c6d..db6a1de97 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -91,10 +91,8 @@ class Controller(wsgi.Controller): def show(self, req, id): """ Returns server details by server id """ try: - LOG.debug(_("***SHOW")) instance = self.compute_api.routing_get( req.environ['nova.context'], id) - LOG.debug(_("***SHOW OUT %s" % instance)) builder = servers_views.get_view_builder(req) return builder.build(instance, is_detail=True) except exception.NotFound: diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d129cf34f..6ce27e9a9 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -15,7 +15,6 @@ import common -from nova import db from nova import flags from nova import log as logging from nova import wsgi @@ -39,7 +38,8 @@ def _exclude_keys(item, keys): def _scrub_zone(zone): - return _filter_keys(zone, ('id', 'api_url')) + return _exclude_keys(zone, ('username', 'password', 'created_at', + 'deleted', 'deleted_at', 'updated_at')) class Controller(wsgi.Controller): @@ -54,12 +54,8 @@ class Controller(wsgi.Controller): # Ask the ZoneManager in the Scheduler for most recent data, # or fall-back to the database ... items = api.get_zone_list(req.environ['nova.context']) - if not items: - items = db.zone_get_all(req.environ['nova.context']) - items = common.limited(items, req) - items = [_exclude_keys(item, ['username', 'password']) - for item in items] + items = [_scrub_zone(item) for item in items] return dict(zones=items) def detail(self, req): @@ -82,23 +78,23 @@ class Controller(wsgi.Controller): def show(self, req, id): """Return data about the given zone id""" zone_id = int(id) - zone = db.zone_get(req.environ['nova.context'], zone_id) + zone = api.zone_get(req.environ['nova.context'], zone_id) return dict(zone=_scrub_zone(zone)) def delete(self, req, id): zone_id = int(id) - db.zone_delete(req.environ['nova.context'], zone_id) + api.zone_delete(req.environ['nova.context'], zone_id) return {} def create(self, req): context = req.environ['nova.context'] env = self._deserialize(req.body, req.get_content_type()) - zone = db.zone_create(context, env["zone"]) + zone = api.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id): context = req.environ['nova.context'] env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) - zone = db.zone_update(context, zone_id, env["zone"]) + zone = api.zone_update(context, zone_id, env["zone"]) return dict(zone=_scrub_zone(zone)) diff --git a/nova/db/api.py b/nova/db/api.py index a4cdb2ae2..7aedaa772 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -34,7 +34,6 @@ The underlying driver is loaded as a :class:`LazyPluggable`. from nova import exception from nova import flags -from nova import log as logging from nova import utils @@ -53,9 +52,6 @@ IMPL = utils.LazyPluggable(FLAGS['db_backend'], sqlalchemy='nova.db.sqlalchemy.api') -LOG = logging.getLogger('server') - - class NoMoreAddresses(exception.Error): """No more available addresses.""" pass diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index bd64f9b9b..c1417dfe4 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -55,9 +55,27 @@ def get_zone_list(context): items = _call_scheduler('get_zone_list', context) for item in items: item['api_url'] = item['api_url'].replace('\\/', '/') + if not items: + items = db.zone_get_all(context) return items +def zone_get(context, zone_id): + return db.zone_get(context, zone_id) + + +def zone_delete(context, zone_id): + return db.zone_delete(context, zone_id) + + +def zone_create(context, data): + return db.zone_create(context, data) + + +def zone_update(context, zone_id, data): + return db.zone_update(context, zone_id, data) + + def get_zone_capabilities(context, service=None): """Returns a dict of key, value capabilities for this zone, or for a particular class of services running in this zone.""" @@ -149,10 +167,8 @@ class reroute_compute(object): def __call__(self, f): def wrapped_f(*args, **kwargs): - LOG.debug(_("IN DECORATOR ...")) collection, context, item_id = \ self.get_collection_context_and_id(args, kwargs) - LOG.debug(_("IN DECORATOR 2...")) try: # Call the original function ... return f(*args, **kwargs) @@ -181,7 +197,7 @@ class reroute_compute(object): """Ask the child zones to perform this operation. Broken out for testing.""" return child_zone_helper(zones, function) - + def get_collection_context_and_id(self, args, kwargs): """Returns a tuple of (novaclient collection name, security context and resource id. Derived class should override this.""" @@ -212,7 +228,7 @@ class reroute_compute(object): del server[k] reduced_response.append(dict(server=server)) - if reduced_response: + if reduced_response: return reduced_response[0] # first for now. return {} diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index d32cc2e8f..198f9d4cc 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -58,8 +58,9 @@ class ZoneState(object): child zone.""" self.last_seen = datetime.now() self.attempt = 0 - self.name = zone_metadata["name"] - self.capabilities = zone_metadata["capabilities"] + self.name = zone_metadata.get("name", "n/a") + self.capabilities = ", ".join(["%s=%s" % (k, v) + for k, v in zone_metadata.iteritems() if k != 'name']) self.is_active = True def to_dict(self): diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 277ffe367..6df74dd61 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -949,6 +949,7 @@ class FakeZone(object): self.username = username self.password = password + def zone_get_all(context): return [ FakeZone('http://example.com', 'bob', 'xxx'), @@ -957,8 +958,8 @@ def zone_get_all(context): class FakeRerouteCompute(api.reroute_compute): def _call_child_zones(self, zones, function): - return [ ] - + return [] + def get_collection_context_and_id(self, args, kwargs): return ("servers", None, 1) @@ -982,6 +983,7 @@ class FakeResource(object): def pause(self): pass + class ZoneRedirectTest(test.TestCase): def setUp(self): super(ZoneRedirectTest, self).setUp() @@ -1024,27 +1026,28 @@ class ZoneRedirectTest(test.TestCase): def test_get_collection_context_and_id(self): decorator = api.reroute_compute("foo") self.assertEquals(decorator.get_collection_context_and_id( - (None, 10, 20), {}), ("servers", 10, 20)) + (None, 10, 20), {}), ("servers", 10, 20)) self.assertEquals(decorator.get_collection_context_and_id( - (None, 11,), dict(instance_id=21)), ("servers", 11, 21)) + (None, 11,), dict(instance_id=21)), ("servers", 11, 21)) self.assertEquals(decorator.get_collection_context_and_id( (None,), dict(context=12, instance_id=22)), ("servers", 12, 22)) def test_unmarshal_single_server(self): decorator = api.reroute_compute("foo") - self.assertEquals(decorator.unmarshall_result([]), {}) + self.assertEquals(decorator.unmarshall_result([]), {}) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(a=1, b=2)),]), - dict(server=dict(a=1, b=2))) + [FakeResource(dict(a=1, b=2)), ]), + dict(server=dict(a=1, b=2))) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(a=1, _b=2)),]), - dict(server=dict(a=1,))) + [FakeResource(dict(a=1, _b=2)), ]), + dict(server=dict(a=1,))) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(a=1, manager=2)),]), - dict(server=dict(a=1,))) + [FakeResource(dict(a=1, manager=2)), ]), + dict(server=dict(a=1,))) self.assertEquals(decorator.unmarshall_result( - [FakeResource(dict(_a=1, manager=2)),]), - dict(server={})) + [FakeResource(dict(_a=1, manager=2)), ]), + dict(server={})) + class FakeServerCollection(object): def get(self, instance_id): @@ -1053,6 +1056,7 @@ class FakeServerCollection(object): def find(self, name): return FakeResource(dict(a=11, b=22)) + class FakeEmptyServerCollection(object): def get(self, f): raise novaclient.NotFound(1) @@ -1060,10 +1064,12 @@ class FakeEmptyServerCollection(object): def find(self, name): raise novaclient.NotFound(2) + class FakeNovaClient(object): def __init__(self, collection): self.servers = collection + class DynamicNovaClientTest(test.TestCase): def test_issue_novaclient_command_found(self): zone = FakeZone('http://example.com', 'bob', 'xxx') @@ -1078,17 +1084,17 @@ class DynamicNovaClientTest(test.TestCase): self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeServerCollection()), zone, "servers", "pause", 100), None) - + def test_issue_novaclient_command_not_found(self): zone = FakeZone('http://example.com', 'bob', 'xxx') self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "get", 100), None) + zone, "servers", "get", 100), None) self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "find", "name"), None) + zone, "servers", "find", "name"), None) self.assertEquals(api._issue_novaclient_command( FakeNovaClient(FakeEmptyServerCollection()), - zone, "servers", "any", "name"), None) + zone, "servers", "any", "name"), None) -- cgit From 02db94dc33d72182201fd78651e5e5e82ab411c2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 01:22:11 -0700 Subject: Earlier versions of the python libvirt binding had getVersion in the libvirt namespace, not on the connection object. Check both. --- nova/virt/libvirt_conn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index f264cf619..0fabec4d0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -981,7 +981,11 @@ class LibvirtConnection(object): """ - return self._conn.getVersion() + # NOTE(justinsb): getVersion moved between libvirt versions + method = getattr(self._conn, 'getVersion', None) # Newer location + if method is None: + method = getattr(libvirt, 'getVersion') # Older location + return method() def get_cpu_info(self): """Get cpuinfo information. -- cgit From 327938fd67bb033597945bdabddaa155ae4bced6 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Wed, 23 Mar 2011 09:19:15 -0400 Subject: id -> instance_id --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 98e6f938a..6f08307be 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -2216,7 +2216,7 @@ def migration_get_by_instance_and_status(context, instance_id, status): filter_by(status=status).first() if not result: raise exception.NotFound(_("No migration found with instance id %s") - % id) + % instance_id) return result -- cgit From f5dada1e0193f9fff89735f169aafffbac1cbd4a Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Wed, 23 Mar 2011 19:22:51 +0300 Subject: review comments fixed --- nova/virt/interfaces.template | 4 ++++ nova/virt/libvirt_conn.py | 45 ++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/nova/virt/interfaces.template b/nova/virt/interfaces.template index 7d40a0f69..a946a1000 100644 --- a/nova/virt/interfaces.template +++ b/nova/virt/interfaces.template @@ -7,11 +7,15 @@ iface lo inet loopback #for $ifc in $interfaces auto ${ifc.name} +#if $getVar('ifc.address', None) iface ${ifc.name} inet static address ${ifc.address} netmask ${ifc.netmask} broadcast ${ifc.broadcast} gateway ${ifc.gateway} dns-nameservers ${ifc.dns} +#else +iface ${ifc.name} inet dhcp +#end if #end for diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 579e2960a..7353d1909 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -623,7 +623,7 @@ class LibvirtConnection(object): def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None, network_info=None): - if network_info is None: + if not network_info: network_info = _get_network_info(inst) # syntactic nicety @@ -707,15 +707,16 @@ class LibvirtConnection(object): nets = [] ifc_template = open(FLAGS.injected_network_template).read() ifc_num = -1 - for (network_ref, _m) in network_info: + admin_context = context.get_admin_context() + for (network_ref, _) in network_info: ifc_num += 1 - if network_ref['injected']: - admin_context = context.get_admin_context() + + if not 'injected' in network_ref: + net_info = {'name': 'eth%d' % ifc_num} + else: address = db.instance_get_fixed_address( admin_context, inst['id']) - ra_server = network_ref['ra_server'] - if not ra_server: - ra_server = "fd00::" + ra_server = network_ref.get('ra_server', "fd00::") net_info = {'name': 'eth%d' % ifc_num, 'address': address, 'netmask': network_ref['netmask'], @@ -800,7 +801,7 @@ class LibvirtConnection(object): #TODO(ilyaalekseyev) remove network_info creation code # when multinics will be completed - if network_info is None: + if not network_info: network_info = _get_network_info(instance) nics = [] @@ -809,9 +810,8 @@ class LibvirtConnection(object): network, mapping)) # FIXME(vish): stick this in db - instance_type = instance['instance_type'] - # instance_type = test.INSTANCE_TYPES[instance_type] - instance_type = instance_types.get_instance_type(instance_type) + instance_type_name = instance['instance_type'] + instance_type = instance_types.get_instance_type(instance_type_name) if FLAGS.use_cow_images: driver_type = 'qcow2' @@ -1608,10 +1608,8 @@ class NWFilterFirewall(FirewallDriver): 'nova-base-ipv6', 'nova-allow-dhcp-server'] if FLAGS.use_ipv6: - #ra_server = self._ra_server_for_instance(instance) ra_servers = self._all_ra_servers_for_instance(instance) - #if ra_server: - if len(ra_servers) != 0: + if ra_servers: instance_secgroup_filter_children += ['nova-allow-ra-server'] ctxt = context.get_admin_context() @@ -1729,10 +1727,8 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.ipv4['filter'].add_chain(chain_name) - ips_v4 = [] - for (_n, mapping) in network_info: - for ip in mapping['ips']: - ips_v4.append(ip['ip']) + ips_v4 = [ip['ip'] for (_, mapping) in network_info + for ip in mapping['ips']] for ipv4_address in ips_v4: self.iptables.ipv4['filter'].add_rule('local', @@ -1741,8 +1737,8 @@ class IptablesFirewallDriver(FirewallDriver): if FLAGS.use_ipv6: self.iptables.ipv6['filter'].add_chain(chain_name) - ips_v6 = [ip['ip'] for ip in mapping['ip6s'] for (_n, mapping) - in network_info] + ips_v6 = [ip['ip'] for (_, mapping) in network_info + for ip in mapping['ip6s']] for ipv6_address in ips_v6: self.iptables.ipv6['filter'].add_rule('local', @@ -1785,14 +1781,14 @@ class IptablesFirewallDriver(FirewallDriver): dhcp_servers = [network['gateway'] for (network, _m) in network_info] for dhcp_server in dhcp_servers: - ipv4_rules += ['-s %s -p udp --sport 67 --dport 68 ' - '-j ACCEPT' % (dhcp_server,)] + ipv4_rules.append('-s %s -p udp --sport 67 --dport 68 ' + '-j ACCEPT' % (dhcp_server,)) #Allow project network traffic if FLAGS.allow_project_net_traffic: cidrs = [network['cidr'] for (network, _m) in network_info] for cidr in cidrs: - ipv4_rules += ['-s %s -j ACCEPT' % (cidr,)] + ipv4_rules.append('-s %s -j ACCEPT' % (cidr,)) # We wrap these in FLAGS.use_ipv6 because they might cause # a DB lookup. The other ones are just list operations, so @@ -1803,7 +1799,8 @@ class IptablesFirewallDriver(FirewallDriver): in network_info] for ra_server in ra_servers: - ipv6_rules += ['-s %s/128 -p icmpv6 -j ACCEPT' % (ra_server,)] + ipv6_rules.append('-s %s/128 -p icmpv6 -j ACCEPT' + % (ra_server,)) #Allow project network traffic if FLAGS.allow_project_net_traffic: -- cgit From a8a345630bd90a74bae00e11dbaf013c60dc7d84 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Wed, 23 Mar 2011 21:44:58 +0300 Subject: pep8 fixed --- nova/virt/libvirt_conn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 5a5f2b14b..5c9e48864 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -717,7 +717,7 @@ class LibvirtConnection(object): if not 'injected' in network_ref: continue - + address = mapping['ips'][0]['ip'] address_v6 = None if FLAGS.use_ipv6: @@ -1924,4 +1924,3 @@ class IptablesFirewallDriver(FirewallDriver): network = db.network_get_by_instance(context.get_admin_context(), instance['id']) return network['cidr_v6'] - -- cgit From 683fcb5da6e742e2b9f1750939dc6a17776d59de Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Wed, 23 Mar 2011 21:56:24 +0300 Subject: xml template fixed --- nova/virt/libvirt.xml.template | 2 +- nova/virt/libvirt_conn.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index ca03900a8..d74a9e85b 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -82,7 +82,7 @@ ${nic.extra_params} #end if #if $getVar('nic.gateway_v6', False) - + #end if diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 5c9e48864..bed348332 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -797,7 +797,7 @@ class LibvirtConnection(object): } if gateway_v6: - xml_info['gateway_v6'] = gateway_v6 + "/128" + result['gateway_v6'] = gateway_v6 + "/128" return result -- cgit From 514e748e3000f97a9d1c03ba3b5ab6faff79abfd Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Wed, 23 Mar 2011 22:08:22 +0300 Subject: one more minor fix --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index bed348332..6fb7c06bd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1814,7 +1814,7 @@ class IptablesFirewallDriver(FirewallDriver): in network_info] for cidrv6 in cidrv6s: - ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] + ipv6_rules.append('-s %s -j ACCEPT' % (cidrv6,)) security_groups = db.security_group_get_by_instance(ctxt, instance['id']) -- cgit From 05e6f82aa971606f7d33fb1de8f2c1c170d030de Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Wed, 23 Mar 2011 12:31:15 -0700 Subject: indenting cleanup --- nova/api/openstack/zones.py | 5 ++--- nova/compute/manager.py | 2 +- nova/flags.py | 2 +- nova/manager.py | 5 ++++- nova/rpc.py | 4 ++-- nova/scheduler/api.py | 2 +- nova/tests/test_zones.py | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d129cf34f..d4a59993b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -17,7 +17,6 @@ import common from nova import db from nova import flags -from nova import log as logging from nova import wsgi from nova.scheduler import api @@ -73,8 +72,8 @@ class Controller(wsgi.Controller): zone = dict(name=FLAGS.zone_name) caps = FLAGS.zone_capabilities for cap in caps: - key_values = cap.split('=') - zone[key_values[0]] = key_values[1] + key, value = cap.split('=') + zone[key] = value for item, (min_value, max_value) in items.iteritems(): zone[item] = "%s,%s" % (min_value, max_value) return dict(zone=zone) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index eae1fee68..289c91d8a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -118,7 +118,7 @@ class ComputeManager(manager.SchedulerDependentManager): self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(service_name="compute", - *args, **kwargs) + *args, **kwargs) def init_host(self): """Do any initialization that needs to be run if this is a diff --git a/nova/flags.py b/nova/flags.py index 3a8ec1a39..bf83b8e0f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -360,4 +360,4 @@ DEFINE_string('node_availability_zone', 'nova', DEFINE_string('zone_name', 'nova', 'name of this zone') DEFINE_list('zone_capabilities', ['hypervisor=xenserver;kvm', 'os=linux;windows'], - 'Key/Multi-value list representng capabilities of this zone') + 'Key/Multi-value list representng capabilities of this zone') diff --git a/nova/manager.py b/nova/manager.py index 508f133ca..804a50479 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -59,6 +59,8 @@ from nova.scheduler import api FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.manager') + class Manager(base.Base): def __init__(self, host=None, db_driver=None): @@ -83,6 +85,7 @@ class SchedulerDependentManager(Manager): should derive from this class. Otherwise they can derive from manager.Manager directly. Updates are only sent after update_service_capabilities is called with non-None values.""" + def __init__(self, host=None, db_driver=None, service_name="undefined"): self.last_capabilities = None self.service_name = service_name @@ -95,7 +98,7 @@ class SchedulerDependentManager(Manager): def periodic_tasks(self, context=None): """Pass data back to the scheduler at a periodic interval""" if self.last_capabilities: - logging.debug(_("Notifying Schedulers of capabilities ...")) + LOG.debug(_("Notifying Schedulers of capabilities ...")) api.update_service_capabilities(context, self.service_name, self.host, self.last_capabilities) diff --git a/nova/rpc.py b/nova/rpc.py index 6ddaea092..388f78d69 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -219,8 +219,8 @@ class FanoutAdapterConsumer(AdapterConsumer): self.queue = "%s_fanout_%s" % (topic, unique) self.durable = False LOG.info(_("Created '%(exchange)s' fanout exchange " - "with '%(key)s' routing key"), - dict(exchange=self.exchange, key=self.routing_key)) + "with '%(key)s' routing key"), + dict(exchange=self.exchange, key=self.routing_key)) super(FanoutAdapterConsumer, self).__init__(connection=connection, topic=topic, proxy=proxy) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index e2cf3b6a3..19a05b716 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -52,7 +52,7 @@ def get_zone_capabilities(context, service=None): """Returns a dict of key, value capabilities for this zone, or for a particular class of services running in this zone.""" return _call_scheduler('get_zone_capabilities', context=context, - params=dict(service=service)) + params=dict(service=service)) def update_service_capabilities(context, service_name, host, capabilities): diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py index 48e1442cf..688dc704d 100644 --- a/nova/tests/test_zones.py +++ b/nova/tests/test_zones.py @@ -96,7 +96,7 @@ class ZoneManagerTestCase(test.TestCase): zm.update_service_capabilities("svc10", "host1", dict(a=99, b=99)) caps = zm.get_zone_capabilities(self, None) self.assertEquals(caps, dict(svc1_a=(2, 20), svc1_b=(3, 30), - svc10_a=(99, 99), svc10_b=(99, 99))) + svc10_a=(99, 99), svc10_b=(99, 99))) zm.update_service_capabilities("svc1", "host3", dict(c=5)) caps = zm.get_zone_capabilities(self, None) -- cgit From 05c4257545fb598222cb472d59d9b8be7ba9535a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 13:29:29 -0700 Subject: Give the user a nicer error message if they're using the Lucid libvirt --- nova/virt/libvirt_conn.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0fabec4d0..26d34b367 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -982,10 +982,14 @@ class LibvirtConnection(object): """ # NOTE(justinsb): getVersion moved between libvirt versions - method = getattr(self._conn, 'getVersion', None) # Newer location + # Trying to do be compatible with older versions is a lost cause + # But ... we can at least give the user a nice message + method = getattr(self._conn, 'getVersion', None) if method is None: - method = getattr(libvirt, 'getVersion') # Older location - return method() + raise exception.Error(_("libvirt version is too old" + " (does not support getVersion)")) + + return self._conn.getVersion() def get_cpu_info(self): """Get cpuinfo information. -- cgit From d966b1989224b8ba7bf580a3f3f8fc0f04b9a566 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 13:34:56 -0700 Subject: Keep the fallback code - we may want to do better version checking in future --- nova/virt/libvirt_conn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 26d34b367..ba794cfd8 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -988,8 +988,11 @@ class LibvirtConnection(object): if method is None: raise exception.Error(_("libvirt version is too old" " (does not support getVersion)")) + # NOTE(justinsb): If we wanted to get the version, we could: + # method = getattr(libvirt, 'getVersion', None) + # NOTE(justinsb): This would then rely on a proper version check - return self._conn.getVersion() + return method() def get_cpu_info(self): """Get cpuinfo information. -- cgit From 3c0fcc47be08ac4f3d508fd46f3b95036899aaad Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Mar 2011 13:39:01 -0700 Subject: fix utils.execute retries for osx also some minor misc cleanups --- nova/network/linux_net.py | 13 ++++++------- nova/tests/test_volume.py | 4 ++-- nova/utils.py | 9 +++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0a273588f..46158bbc0 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -1,3 +1,5 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -212,10 +214,7 @@ class IptablesManager(object): """ def __init__(self, execute=None): if not execute: - if FLAGS.fake_network: - self.execute = lambda *args, **kwargs: ('', '') - else: - self.execute = utils.execute + self.execute = _execute else: self.execute = execute @@ -361,9 +360,6 @@ class IptablesManager(object): return new_filter -iptables_manager = IptablesManager() - - def metadata_forward(): """Create forwarding rule for metadata""" iptables_manager.ipv4['nat'].add_rule("PREROUTING", @@ -776,3 +772,6 @@ def _ip_bridge_cmd(action, params, device): cmd.extend(params) cmd.extend(['dev', device]) return cmd + + +iptables_manager = IptablesManager() diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index 5d68ca2ae..d71b75f3f 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -356,8 +356,8 @@ class ISCSITestCase(DriverTestCase): tid = db.volume_get_iscsi_target_num(self.context, volume_id_list[0]) self.mox.StubOutWithMock(self.volume.driver, '_execute') self.volume.driver._execute("sudo", "ietadm", "--op", "show", - "--tid=%(tid)d" % locals() - ).AndRaise(exception.ProcessExecutionError()) + "--tid=%(tid)d" % locals()).AndRaise( + exception.ProcessExecutionError()) self.mox.ReplayAll() self.assertRaises(exception.ProcessExecutionError, diff --git a/nova/utils.py b/nova/utils.py index 499af2039..249470636 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -170,10 +170,6 @@ def execute(*cmd, **kwargs): stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess - # call clean something up in between calls, without - # it two execute calls in a row hangs the second one - greenthread.sleep(0) return result except ProcessExecutionError: if not attempts: @@ -182,6 +178,11 @@ def execute(*cmd, **kwargs): LOG.debug(_("%r failed. Retrying."), cmd) if delay_on_retry: greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) def ssh_execute(ssh, cmd, process_input=None, -- cgit From 95fa499f1a7718694e37a747a6a5a0e309ce877d Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Thu, 24 Mar 2011 00:36:07 +0300 Subject: migration gateway_v6 to network_info --- nova/tests/test_virt.py | 3 ++- nova/utils.py | 14 ++++++++----- nova/virt/libvirt_conn.py | 53 +++++++---------------------------------------- 3 files changed, 19 insertions(+), 51 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index b214f5ce7..98bb11526 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -785,7 +785,8 @@ class NWFilterTestCase(test.TestCase): instance_ref = db.instance_create(self.context, {'user_id': 'fake', - 'project_id': 'fake'}) + 'project_id': 'fake', + 'mac_address': '00:A0:C9:14:C8:29'}) inst_id = instance_ref['id'] ip = '10.11.12.13' diff --git a/nova/utils.py b/nova/utils.py index 499af2039..44234813f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -309,11 +309,15 @@ def get_my_linklocal(interface): def to_global_ipv6(prefix, mac): - mac64 = netaddr.EUI(mac).eui64().words - int_addr = int(''.join(['%02x' % i for i in mac64]), 16) - mac64_addr = netaddr.IPAddress(int_addr) - maskIP = netaddr.IPNetwork(prefix).ip - return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).format() + try: + mac64 = netaddr.EUI(mac).eui64().words + int_addr = int(''.join(['%02x' % i for i in mac64]), 16) + mac64_addr = netaddr.IPAddress(int_addr) + maskIP = netaddr.IPNetwork(prefix).ip + return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).\ + format() + except TypeError: + raise TypeError(_("Bad mac for to_global_ipv6: %s" % mac)) def to_mac(ipv6_address): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 6fb7c06bd..dd2439e42 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -757,7 +757,7 @@ class LibvirtConnection(object): if FLAGS.libvirt_type == 'uml': utils.execute('sudo', 'chown', 'root', basepath('disk')) - def _get_nic_for_xml(self, instance_id, network, mapping): + def _get_nic_for_xml(self, network, mapping): # Assume that the gateway also acts as the dhcp server. dhcp_server = network['gateway'] gateway_v6 = network['gateway_v6'] @@ -802,8 +802,6 @@ class LibvirtConnection(object): return result def to_xml(self, instance, rescue=False, network_info=None): - admin_context = context.get_admin_context() - # TODO(termie): cache? LOG.debug(_('instance %s: starting toXML method'), instance['name']) @@ -814,8 +812,7 @@ class LibvirtConnection(object): nics = [] for (network, mapping) in network_info: - nics.append(self._get_nic_for_xml(instance['id'], - network, + nics.append(self._get_nic_for_xml(network, mapping)) # FIXME(vish): stick this in db instance_type_name = instance['instance_type'] @@ -1392,16 +1389,6 @@ class FirewallDriver(object): """ raise NotImplementedError() - def _gateway_v6_for_instance(self, instance): - network = db.network_get_by_instance(context.get_admin_context(), - instance['id']) - return network['gateway_v6'] - - def _all_gateway_v6_for_instance(self, instance): - networks = db.network_get_all_by_instance(context.get_admin_context(), - instance['id']) - return [network['gateway_v6'] for network in networks] - class NWFilterFirewall(FirewallDriver): """ @@ -1604,6 +1591,8 @@ class NWFilterFirewall(FirewallDriver): it makes sure the filters for the security groups as well as the base filter are all in place. """ + if not network_info: + network_info = _get_network_info(instance) if instance['image_id'] == FLAGS.vpn_image_id: base_filter = 'nova-vpn' else: @@ -1616,7 +1605,8 @@ class NWFilterFirewall(FirewallDriver): 'nova-base-ipv6', 'nova-allow-dhcp-server'] if FLAGS.use_ipv6: - gateways_v6 = self._all_gateway_v6_for_instance(instance) + gateways_v6 = [network['gateway_v6'] for (network, _) in + network_info] if gateways_v6: instance_secgroup_filter_children += ['nova-allow-ra-server'] @@ -1803,7 +1793,8 @@ class IptablesFirewallDriver(FirewallDriver): # they're not worth the clutter. if FLAGS.use_ipv6: # Allow RA responses - gateways_v6 = self._all_gateway_v6_for_instance(instance) + gateways_v6 = [network['gateway_v6'] for (network, _) in + network_info] for gateway_v6 in gateways_v6: ipv6_rules.append( '-s %s/128 -p icmpv6 -j ACCEPT' % (gateway_v6,)) @@ -1896,31 +1887,3 @@ class IptablesFirewallDriver(FirewallDriver): def _instance_chain_name(self, instance): return 'inst-%s' % (instance['id'],) - - def _ip_for_instance(self, instance): - return db.instance_get_fixed_address(context.get_admin_context(), - instance['id']) - - def _ip_for_instance_v6(self, instance): - return db.instance_get_fixed_address_v6(context.get_admin_context(), - instance['id']) - - def _dhcp_server_for_instance(self, instance): - network = db.network_get_by_instance(context.get_admin_context(), - instance['id']) - return network['gateway'] - - def _gateway_v6_for_instance(self, instance): - network = db.network_get_by_instance(context.get_admin_context(), - instance['id']) - return network['gateway_v6'] - - def _project_cidr_for_instance(self, instance): - network = db.network_get_by_instance(context.get_admin_context(), - instance['id']) - return network['cidr'] - - def _project_cidrv6_for_instance(self, instance): - network = db.network_get_by_instance(context.get_admin_context(), - instance['id']) - return network['cidr_v6'] -- cgit From 5170e8b5dd96cf8c7bb91e84203cfaebb099af46 Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Thu, 24 Mar 2011 00:56:56 +0300 Subject: small fix --- nova/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/utils.py b/nova/utils.py index 44234813f..fabc01532 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -317,7 +317,7 @@ def to_global_ipv6(prefix, mac): return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).\ format() except TypeError: - raise TypeError(_("Bad mac for to_global_ipv6: %s" % mac)) + raise TypeError(_("Bad mac for to_global_ipv6: %s") % mac) def to_mac(ipv6_address): -- cgit From 16372d3bc0181a57958ce185e89f1f21126b9e77 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Mar 2011 20:21:44 -0700 Subject: Don't try to parse a datetime if it is the empty string (or None) --- nova/image/glance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 171b28fde..9984a3ba1 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -73,7 +73,7 @@ class GlanceImageService(service.BaseImageService): Returns image with known timestamp fields converted to datetime objects """ for attr in ['created_at', 'updated_at', 'deleted_at']: - if image.get(attr) is not None: + if image.get(attr): image[attr] = self._parse_glance_iso8601_timestamp(image[attr]) return image -- cgit From 107c3f75d91dcb7aadf3136e964d1feb6c505dc7 Mon Sep 17 00:00:00 2001 From: Muneyuki Noguchi Date: Thu, 24 Mar 2011 16:21:50 +0900 Subject: Declare libvirt_type to avoid AttributeError in live_migration --- bin/nova-manage | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-manage b/bin/nova-manage index 69cbf6f95..6712fbadb 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -97,6 +97,7 @@ flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') flags.DECLARE('images_path', 'nova.image.local') +flags.DECLARE('libvirt_type', 'nova.virt.libvirt_conn') flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) -- cgit From f640d32bd8698fc2c30b2ca0454672d691f9b296 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 24 Mar 2011 05:02:54 -0700 Subject: fix based on sirp's comments --- nova/scheduler/api.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index c1417dfe4..a4f304c62 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -118,16 +118,15 @@ def child_zone_helper(zone_list, func): _wrap_method(_process, func), zone_list)] -def _issue_novaclient_command(nova, zone, collection, method_name, \ - item_id): +def _issue_novaclient_command(nova, zone, collection, method_name, item_id): """Use novaclient to issue command to a single child zone. One of these will be run in parallel for each child zone.""" + manager = getattr(nova, collection) result = None try: - manager = getattr(nova, collection) - if isinstance(item_id, int) or item_id.isdigit(): + try: result = manager.get(int(item_id)) - else: + except ValueError, e: result = manager.find(name=item_id) except novaclient.NotFound: url = zone.api_url -- cgit From 4e5b511b422501167161c3bbe4dd755c0370c93f Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Thu, 24 Mar 2011 16:53:32 +0300 Subject: couple of bugs fixed --- nova/tests/test_virt.py | 3 +- nova/virt/libvirt_conn.py | 75 ++++++++++++++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 98bb11526..12f97383e 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -803,7 +803,8 @@ class NWFilterTestCase(test.TestCase): 'instance_id': instance_ref['id']}) def _ensure_all_called(): - instance_filter = 'nova-instance-%s' % instance_ref['name'] + instance_filter = 'nova-instance-%s-%s' % (instance_ref['name'], + '00A0C914C829') secgroup_filter = 'nova-secgroup-%s' % self.security_group['id'] for required in [secgroup_filter, 'allow-dhcp-server', 'no-arp-spoofing', 'no-ip-spoofing', diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index dd2439e42..bbb5699e9 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -189,8 +189,10 @@ def _get_network_info(instance): 'gateway': network['gateway'], 'mac': instance.mac_address, 'dns': [network['dns']], - 'ips': [ip_dict(ip) for ip in network_ips], - 'ip6s': [ip6_dict(ip) for ip in network_ips]} + 'ips': [ip_dict(ip) for ip in network_ips]} + + if FLAGS.use_ipv6: + mapping['ip6s'] = [ip6_dict(ip) for ip in network_ips] network_info.append((network, mapping)) return network_info @@ -632,6 +634,8 @@ class LibvirtConnection(object): if not network_info: network_info = _get_network_info(inst) + if not suffix: + suffix = '' # syntactic nicety def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, @@ -1484,6 +1488,9 @@ class NWFilterFirewall(FirewallDriver): """Set up basic filtering (MAC, IP, and ARP spoofing protection)""" logging.info('called setup_basic_filtering in nwfilter') + if not network_info: + network_info = _get_network_info(instance) + if self.handle_security_groups: # No point in setting up a filter set that we'll be overriding # anyway. @@ -1492,9 +1499,11 @@ class NWFilterFirewall(FirewallDriver): logging.info('ensuring static filters') self._ensure_static_filters() - instance_filter_name = self._instance_filter_name(instance) - self._define_filter(self._filter_container(instance_filter_name, - ['nova-base'])) + for (network, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + self._define_filter(self._filter_container(instance_filter_name, + ['nova-base'])) def _ensure_static_filters(self): if self.static_filters_configured: @@ -1598,38 +1607,47 @@ class NWFilterFirewall(FirewallDriver): else: base_filter = 'nova-base' - instance_filter_name = self._instance_filter_name(instance) - instance_secgroup_filter_name = '%s-secgroup' % (instance_filter_name,) - instance_filter_children = [base_filter, instance_secgroup_filter_name] + ctxt = context.get_admin_context() + + instance_secgroup_filter_name = \ + '%s-secgroup' % (self._instance_filter_name(instance)) + #% (instance_filter_name,) + instance_secgroup_filter_children = ['nova-base-ipv4', 'nova-base-ipv6', 'nova-allow-dhcp-server'] - if FLAGS.use_ipv6: - gateways_v6 = [network['gateway_v6'] for (network, _) in - network_info] - if gateways_v6: - instance_secgroup_filter_children += ['nova-allow-ra-server'] - ctxt = context.get_admin_context() - - if FLAGS.allow_project_net_traffic: - instance_filter_children += ['nova-project'] - if FLAGS.use_ipv6: - instance_filter_children += ['nova-project-v6'] - - for security_group in db.security_group_get_by_instance(ctxt, - instance['id']): + for security_group in \ + db.security_group_get_by_instance(ctxt, instance['id']): self.refresh_security_group_rules(security_group['id']) instance_secgroup_filter_children += [('nova-secgroup-%s' % - security_group['id'])] + security_group['id'])] - self._define_filter( + self._define_filter( self._filter_container(instance_secgroup_filter_name, instance_secgroup_filter_children)) - self._define_filter( + for (network, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + instance_filter_children = \ + [base_filter, instance_secgroup_filter_name] + + if FLAGS.use_ipv6: + gateway_v6 = network['gateway_v6'] + + if gateway_v6: + instance_secgroup_filter_children += \ + ['nova-allow-ra-server'] + + if FLAGS.allow_project_net_traffic: + instance_filter_children += ['nova-project'] + if FLAGS.use_ipv6: + instance_filter_children += ['nova-project-v6'] + + self._define_filter( self._filter_container(instance_filter_name, instance_filter_children)) @@ -1677,8 +1695,11 @@ class NWFilterFirewall(FirewallDriver): xml += "chain='ipv4'>%s" % rule_xml return xml - def _instance_filter_name(self, instance): - return 'nova-instance-%s' % instance['name'] + def _instance_filter_name(self, instance, nic_id=None): + if not nic_id: + return 'nova-instance-%s' % (instance['name']) + + return 'nova-instance-%s-%s' % (instance['name'], nic_id) class IptablesFirewallDriver(FirewallDriver): -- cgit From fa582b2e10e43bbb81e0e5c3baf4560300711271 Mon Sep 17 00:00:00 2001 From: Eldar Nugaev Date: Thu, 24 Mar 2011 17:24:44 +0300 Subject: pep8 clearing --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index bbb5699e9..181de4540 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -636,6 +636,7 @@ class LibvirtConnection(object): if not suffix: suffix = '' + # syntactic nicety def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, @@ -1698,7 +1699,6 @@ class NWFilterFirewall(FirewallDriver): def _instance_filter_name(self, instance, nic_id=None): if not nic_id: return 'nova-instance-%s' % (instance['name']) - return 'nova-instance-%s-%s' % (instance['name'], nic_id) -- cgit From cf70a1a76dfa0456ea6230eaa014fa98e7ddd464 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 24 Mar 2011 10:30:09 -0500 Subject: Small refactor --- nova/utils.py | 7 ++----- nova/virt/xenapi/vmops.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 03a6e8095..6042a5332 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -336,11 +336,8 @@ utcnow.override_time = None def is_older_than(before, seconds): - """Return True if before is older than 'seconds'""" - if utcnow() - before > datetime.timedelta(seconds=seconds): - return True - else: - return False + """Return True if before is older than seconds""" + return utcnow() - before > datetime.timedelta(seconds=seconds) def utcnow_ts(): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 84cf836ac..6c1f6424a 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -528,7 +528,7 @@ class VMOps(object): vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) for vbd_ref in vbd_refs: vbd_rec = self._session.get_xenapi().VBD.get_record(vbd_ref) - if vbd_rec["userdevice"] == "1": # primary VBD is always 1 + if vbd_rec.get("userdevice", None) == "1": # VBD is always 1 VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) @@ -712,18 +712,18 @@ class VMOps(object): in rescue mode for >= the provided timeout """ last_ran = self.poll_rescue_last_ran - if last_ran: - if not utils.is_older_than(last_ran, timeout): - # Do not run. Let's bail. - return - else: - # Update the time tracker and proceed. - self.poll_rescue_last_ran = utils.utcnow() - else: + if not last_ran: # We need a base time to start tracking. self.poll_rescue_last_ran = utils.utcnow() return + if not utils.is_older_than(last_ran, timeout): + # Do not run. Let's bail. + return + + # Update the time tracker and proceed. + self.poll_rescue_last_ran = utils.utcnow() + rescue_vms = [] for instance in self.list_instances(): if instance.endswith("-rescue"): -- cgit From 12184874da4369891b2eae49982623fc6c9315e3 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 24 Mar 2011 13:26:57 -0400 Subject: Updated to use new APIRouterV11 class in tests. --- nova/tests/api/openstack/test_extensions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index d1f8c659e..4bc823d15 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -81,14 +81,14 @@ class StubExtensionManager(object): class ExtensionControllerTest(unittest.TestCase): def test_index(self): - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app) request = webob.Request.blank("/extensions") response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) def test_get_by_alias(self): - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app) request = webob.Request.blank("/extensions/FOXNSOX") response = request.get_response(ext_midware) @@ -99,7 +99,7 @@ class ResourceExtensionTest(unittest.TestCase): def test_no_extension_present(self): manager = StubExtensionManager(None) - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/blah") response = request.get_response(ext_midware) @@ -109,7 +109,7 @@ class ResourceExtensionTest(unittest.TestCase): res_ext = extensions.ResourceExtension('tweedles', StubController(response_body)) manager = StubExtensionManager(res_ext) - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/tweedles") response = request.get_response(ext_midware) @@ -120,7 +120,7 @@ class ResourceExtensionTest(unittest.TestCase): res_ext = extensions.ResourceExtension('tweedles', StubController(response_body)) manager = StubExtensionManager(res_ext) - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app, manager) request = webob.Request.blank("/tweedles") response = request.get_response(ext_midware) @@ -137,7 +137,7 @@ class ExtensionManagerTest(unittest.TestCase): "extensions") def test_get_resources(self): - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app) request = webob.Request.blank("/foxnsocks") response = request.get_response(ext_midware) @@ -152,7 +152,7 @@ class ActionExtensionTest(unittest.TestCase): "extensions") def _send_server_action_request(self, url, body): - app = openstack.APIRouter() + app = openstack.APIRouterV11() ext_midware = extensions.ExtensionMiddleware(app) request = webob.Request.blank(url) request.method = 'POST' -- cgit From e5069f27cd9e6551a6b035d6fff1b02a6bf0b492 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 24 Mar 2011 13:56:25 -0400 Subject: Reconcile tests with latest trunk merges. --- etc/api-paste.ini | 2 +- nova/tests/api/openstack/extensions/foxinsocks.py | 4 ++-- nova/tests/api/openstack/test_extensions.py | 13 +++++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/etc/api-paste.ini b/etc/api-paste.ini index 65c92b553..35d4a8391 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -86,7 +86,7 @@ paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory paste.filter_factory = nova.api.openstack.limits:RateLimitingMiddleware.factory [filter:extensions] -paste.filter_factory = nova.api.openstack.extensions:ExtensionsRouter.factory +paste.filter_factory = nova.api.openstack.extensions:ExtensionMiddleware.factory [app:osapiapp10] paste.app_factory = nova.api.openstack:APIRouterV10.factory diff --git a/nova/tests/api/openstack/extensions/foxinsocks.py b/nova/tests/api/openstack/extensions/foxinsocks.py index 249dd81bf..0860b51ac 100644 --- a/nova/tests/api/openstack/extensions/foxinsocks.py +++ b/nova/tests/api/openstack/extensions/foxinsocks.py @@ -73,7 +73,7 @@ class Foxinsocks(object): data['flavor']['googoose'] = "Gooey goo for chewy chewing!" return data - resp_ext = extensions.ResponseExtension('GET', '/v1.0/flavors/:(id)', + resp_ext = extensions.ResponseExtension('GET', '/v1.1/flavors/:(id)', _goose_handler) response_exts.append(resp_ext) @@ -84,7 +84,7 @@ class Foxinsocks(object): data['big_bands'] = 'Pig Bands!' return data - resp_ext2 = extensions.ResponseExtension('GET', '/v1.0/flavors/:(id)', + resp_ext2 = extensions.ResponseExtension('GET', '/v1.1/flavors/:(id)', _bands_handler) response_exts.append(resp_ext2) return response_exts diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 4bc823d15..481d34ed1 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -193,6 +193,10 @@ class ResponseExtensionTest(unittest.TestCase): fakes.stub_out_auth(self.stubs) self.context = context.get_admin_context() + def tearDown(self): + self.stubs.UnsetAll() + super(ResponseExtensionTest, self).tearDown() + def test_get_resources_with_stub_mgr(self): test_resp = "Gooey goo for chewy chewing!" @@ -204,13 +208,14 @@ class ResponseExtensionTest(unittest.TestCase): return data resp_ext = extensions.ResponseExtension('GET', - '/v1.0/flavors/:(id)', + '/v1.1/flavors/:(id)', _resp_handler) manager = StubExtensionManager(None, None, resp_ext) app = fakes.wsgi_app() ext_midware = extensions.ExtensionMiddleware(app, manager) - request = webob.Request.blank("/v1.0/flavors/1") + request = webob.Request.blank("/v1.1/flavors/1") + request.environ['api.version'] = '1.1' response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) response_data = json.loads(response.body) @@ -222,10 +227,10 @@ class ResponseExtensionTest(unittest.TestCase): app = fakes.wsgi_app() ext_midware = extensions.ExtensionMiddleware(app) - request = webob.Request.blank("/v1.0/flavors/1") + request = webob.Request.blank("/v1.1/flavors/1") + request.environ['api.version'] = '1.1' response = request.get_response(ext_midware) self.assertEqual(200, response.status_int) response_data = json.loads(response.body) - print response_data self.assertEqual(test_resp, response_data['flavor']['googoose']) self.assertEqual("Pig Bands!", response_data['big_bands']) -- cgit From c7ccbd7a16a546cbd0717427772691ce7d8b4da6 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:46 -0700 Subject: support volume and network in the direct api --- bin/nova-direct-api | 9 +++++++-- nova/api/ec2/cloud.py | 20 +++++++++++--------- nova/compute/api.py | 9 +++++---- nova/tests/test_direct.py | 14 ++++++++++++-- nova/volume/api.py | 3 ++- 5 files changed, 37 insertions(+), 18 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index a2c9f1557..1a78fb0c0 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -34,12 +34,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import compute from nova import flags from nova import log as logging +from nova import network from nova import utils +from nova import volume from nova import wsgi from nova.api import direct -from nova.compute import api as compute_api FLAGS = flags.FLAGS @@ -50,12 +52,15 @@ flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) logging.setup() - direct.register_service('compute', compute_api.API()) + direct.register_service('compute', compute.API()) + direct.register_service('volume', volume.API()) + direct.register_service('network', network.API()) direct.register_service('reflect', direct.Reflection()) router = direct.Router() with_json = direct.JsonParamsMiddleware(router) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 2afcea77c..5d31d71d3 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -541,7 +541,7 @@ class CloudController(object): volumes = [] for ec2_id in volume_id: internal_id = ec2utils.ec2_id_to_id(ec2_id) - volume = self.volume_api.get(context, internal_id) + volume = self.volume_api.get(context, volume_id=internal_id) volumes.append(volume) else: volumes = self.volume_api.get_all(context) @@ -585,9 +585,11 @@ class CloudController(object): def create_volume(self, context, size, **kwargs): LOG.audit(_("Create volume of %s GB"), size, context=context) - volume = self.volume_api.create(context, size, - kwargs.get('display_name'), - kwargs.get('display_description')) + volume = self.volume_api.create( + context, + size=size, + name=kwargs.get('display_name'), + description=kwargs.get('display_description')) # TODO(vish): Instance should be None at db layer instead of # trying to lazy load, but for now we turn it into # a dict to avoid an error. @@ -606,7 +608,7 @@ class CloudController(object): if field in kwargs: changes[field] = kwargs[field] if changes: - self.volume_api.update(context, volume_id, kwargs) + self.volume_api.update(context, volume_id=volume_id, fields=changes) return True def attach_volume(self, context, volume_id, instance_id, device, **kwargs): @@ -619,7 +621,7 @@ class CloudController(object): instance_id=instance_id, volume_id=volume_id, device=device) - volume = self.volume_api.get(context, volume_id) + volume = self.volume_api.get(context, volume_id=volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], 'instanceId': ec2utils.id_to_ec2_id(instance_id), @@ -630,7 +632,7 @@ class CloudController(object): def detach_volume(self, context, volume_id, **kwargs): volume_id = ec2utils.ec2_id_to_id(volume_id) LOG.audit(_("Detach volume %s"), volume_id, context=context) - volume = self.volume_api.get(context, volume_id) + volume = self.volume_api.get(context, volume_id=volume_id) instance = self.compute_api.detach_volume(context, volume_id=volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], @@ -768,7 +770,7 @@ class CloudController(object): def release_address(self, context, public_ip, **kwargs): LOG.audit(_("Release address %s"), public_ip, context=context) - self.network_api.release_floating_ip(context, public_ip) + self.network_api.release_floating_ip(context, address=public_ip) return {'releaseResponse': ["Address released."]} def associate_address(self, context, instance_id, public_ip, **kwargs): @@ -782,7 +784,7 @@ class CloudController(object): def disassociate_address(self, context, public_ip, **kwargs): LOG.audit(_("Disassociate address %s"), public_ip, context=context) - self.network_api.disassociate_floating_ip(context, public_ip) + self.network_api.disassociate_floating_ip(context, address=public_ip) return {'disassociateResponse': ["Address disassociated."]} def run_instances(self, context, **kwargs): diff --git a/nova/compute/api.py b/nova/compute/api.py index 309847156..f4aab97de 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -636,7 +636,7 @@ class API(base.Base): if not re.match("^/dev/[a-z]d[a-z]+$", device): raise exception.ApiError(_("Invalid device specified: %s. " "Example device: /dev/vdb") % device) - self.volume_api.check_attach(context, volume_id) + self.volume_api.check_attach(context, volume_id=volume_id) instance = self.get(context, instance_id) host = instance['host'] rpc.cast(context, @@ -650,7 +650,7 @@ class API(base.Base): instance = self.db.volume_get_instance(context.elevated(), volume_id) if not instance: raise exception.ApiError(_("Volume isn't attached to anything!")) - self.volume_api.check_detach(context, volume_id) + self.volume_api.check_detach(context, volume_id=volume_id) host = instance['host'] rpc.cast(context, self.db.queue_get_for(context, FLAGS.compute_topic, host), @@ -661,5 +661,6 @@ class API(base.Base): def associate_floating_ip(self, context, instance_id, address): instance = self.get(context, instance_id) - self.network_api.associate_floating_ip(context, address, - instance['fixed_ip']) + self.network_api.associate_floating_ip(context, + floating_ip=address, + fixed_ip=instance['fixed_ip']) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 80e4d2e1f..001246fc4 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -25,7 +25,9 @@ import webob from nova import compute from nova import context from nova import exception +from nova import network from nova import test +from nova import volume from nova import utils from nova.api import direct from nova.tests import test_cloud @@ -93,12 +95,20 @@ class DirectTestCase(test.TestCase): class DirectCloudTestCase(test_cloud.CloudTestCase): def setUp(self): super(DirectCloudTestCase, self).setUp() - compute_handle = compute.API(network_api=self.cloud.network_api, - volume_api=self.cloud.volume_api) + compute_handle = compute.API(image_service=self.cloud.image_service) + volume_handle = volume.API() + network_handle = network.API() direct.register_service('compute', compute_handle) + direct.register_service('volume', volume_handle) + direct.register_service('network', network_handle) + self.router = direct.JsonParamsMiddleware(direct.Router()) proxy = direct.Proxy(self.router) self.cloud.compute_api = proxy.compute + self.cloud.volume_api = proxy.volume + self.cloud.network_api = proxy.network + compute_handle.volume_api = proxy.volume + compute_handle.network_api = proxy.network def tearDown(self): super(DirectCloudTestCase, self).tearDown() diff --git a/nova/volume/api.py b/nova/volume/api.py index 2f4494845..4b4bb9dc5 100644 --- a/nova/volume/api.py +++ b/nova/volume/api.py @@ -82,7 +82,8 @@ class API(base.Base): self.db.volume_update(context, volume_id, fields) def get(self, context, volume_id): - return self.db.volume_get(context, volume_id) + rv = self.db.volume_get(context, volume_id) + return dict(rv.iteritems()) def get_all(self, context): if context.is_admin: -- cgit From ac44b8a9c5ed6a761793e1fa997768bd00a6c2da Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:46 -0700 Subject: improve the formatting of the stack tool --- bin/stack | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/stack b/bin/stack index 25caca06f..d84a82e27 100755 --- a/bin/stack +++ b/bin/stack @@ -59,11 +59,21 @@ USAGE = """usage: stack [options] [arg1=value arg2=value] def format_help(d): """Format help text, keys are labels and values are descriptions.""" + MAX_INDENT = 30 indent = max([len(k) for k in d]) + if indent > MAX_INDENT: + indent = MAX_INDENT - 6 + out = [] for k, v in d.iteritems(): - t = textwrap.TextWrapper(initial_indent=' %s ' % k.ljust(indent), - subsequent_indent=' ' * (indent + 6)) + if (len(k) + 6) > MAX_INDENT: + out.extend([' %s' % k]) + initial_indent = ' ' * (indent + 6) + else: + initial_indent = ' %s ' % k.ljust(indent) + subsequent_indent = ' ' * (indent + 6) + t = textwrap.TextWrapper(initial_indent=initial_indent, + subsequent_indent=subsequent_indent) out.extend(t.wrap(v)) return out -- cgit From ef5c9e11595a00de468783adbb60cfbc2cbbf13d Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:46 -0700 Subject: add Limited, an API limiting/versioning wrapper --- bin/nova-direct-api | 7 +++++++ nova/api/direct.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 1a78fb0c0..ac0b5b51c 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -53,12 +53,19 @@ flags.DEFINE_flag(flags.HelpXMLFlag()) +class ReadOnlyCompute(direct.Limited): + """Read-only Compute API.""" + + _allowed = ['get', 'get_all', 'get_console_output'] + + if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) logging.setup() direct.register_service('compute', compute.API()) + direct.register_service('compute-readonly', ReadOnlyCompute(compute.API())) direct.register_service('volume', volume.API()) direct.register_service('network', network.API()) direct.register_service('reflect', direct.Reflection()) diff --git a/nova/api/direct.py b/nova/api/direct.py index dfca250e0..1011091a6 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -211,6 +211,42 @@ class ServiceWrapper(wsgi.Controller): return result +class Limited(object): + __notdoc = """Limit the available methods on a given object. + + (Not a docstring so that the docstring can be conditionally overriden.) + + Useful when defining a public API that only exposes a subset of an + internal API. + + Expected usage of this class is to define a subclass that lists the allowed + methods in the 'allowed' variable. + + Additionally where appropriate methods can be added or overwritten, for + example to provide backwards compatibility. + + """ + + _allowed = None + + def __init__(self, proxy): + self._proxy = proxy + if not self.__doc__: + self.__doc__ = proxy.__doc__ + if not self._allowed: + self._allowed = [] + + def __getattr__(self, key): + """Only return methods that are named in self._allowed.""" + if key not in self._allowed: + raise AttributeError() + return getattr(self._proxy, key) + + def __dir__(self): + """Only return methods that are named in self._allowed.""" + return [x for x in dir(self._proxy) if x in self._allowed] + + class Proxy(object): """Pretend a Direct API endpoint is an object.""" def __init__(self, app, prefix=None): -- cgit From 5c03ade2ee82350d845c8306d5aab9eda3073137 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:47 -0700 Subject: add some more docs to direct.py --- nova/api/direct.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nova/api/direct.py b/nova/api/direct.py index 1011091a6..2e158e89e 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -225,6 +225,10 @@ class Limited(object): Additionally where appropriate methods can be added or overwritten, for example to provide backwards compatibility. + The wrapping approach has been chosen so that the wrapped API can maintain + its own internal consistency, for example if it calls "self.create" it + should get its own create method rather than anything we do here. + """ _allowed = None -- cgit From a7863c026819a9369cecaa42778a10ab54e798ba Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:47 -0700 Subject: add an example of a versioned api --- bin/nova-direct-api | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index ac0b5b51c..0f7589871 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -53,11 +53,19 @@ flags.DEFINE_flag(flags.HelpXMLFlag()) +# An example of an API that only exposes read-only methods. class ReadOnlyCompute(direct.Limited): """Read-only Compute API.""" _allowed = ['get', 'get_all', 'get_console_output'] +# An example of an API that provides a backwards compatibility layer. +class VolumeVersionOne(direct.Limited): + _allowed = ['create', 'delete', 'update', 'get'] + + def create(self, context, size, name): + self.proxy.create(context, size, name, description=None) + if __name__ == '__main__': utils.default_flagfile() @@ -65,10 +73,11 @@ if __name__ == '__main__': logging.setup() direct.register_service('compute', compute.API()) - direct.register_service('compute-readonly', ReadOnlyCompute(compute.API())) direct.register_service('volume', volume.API()) direct.register_service('network', network.API()) direct.register_service('reflect', direct.Reflection()) + direct.register_service('compute-readonly', ReadOnlyCompute(compute.API())) + direct.register_service('volume-v1', VolumeVersionOne(volume.API())) router = direct.Router() with_json = direct.JsonParamsMiddleware(router) with_req = direct.PostParamsMiddleware(with_json) -- cgit From a1bde64e91a8b76fd0e69c3bdfc51e4e85adf6f0 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:47 -0700 Subject: add some more docs and make it more obvious which parts are examples --- bin/nova-direct-api | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 0f7589871..bb3aa8ae7 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -54,12 +54,19 @@ flags.DEFINE_flag(flags.HelpXMLFlag()) # An example of an API that only exposes read-only methods. +# In this case we're just limiting which methods are exposed. class ReadOnlyCompute(direct.Limited): """Read-only Compute API.""" _allowed = ['get', 'get_all', 'get_console_output'] + # An example of an API that provides a backwards compatibility layer. +# In this case we're overwriting the implementation to ensure +# compatibility with an older version. In reality we would want the +# "description=None" to be part of the actual API so that code +# like this isn't even necessary, but this example shows what one can +# do if that isn't the situation. class VolumeVersionOne(direct.Limited): _allowed = ['create', 'delete', 'update', 'get'] @@ -76,8 +83,12 @@ if __name__ == '__main__': direct.register_service('volume', volume.API()) direct.register_service('network', network.API()) direct.register_service('reflect', direct.Reflection()) - direct.register_service('compute-readonly', ReadOnlyCompute(compute.API())) - direct.register_service('volume-v1', VolumeVersionOne(volume.API())) + + # Here is how we could expose the code in the examples above. + #direct.register_service('compute-readonly', + # ReadOnlyCompute(compute.API())) + #direct.register_service('volume-v1', VolumeVersionOne(volume.API())) + router = direct.Router() with_json = direct.JsonParamsMiddleware(router) with_req = direct.PostParamsMiddleware(with_json) -- cgit From 4a6db815b01c71076bae96c155396e5adbe8af90 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 12:42:47 -0700 Subject: better error handling and serialization --- nova/api/direct.py | 9 ++++++--- nova/tests/test_direct.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index 2e158e89e..bb2ace1c9 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -38,6 +38,7 @@ import routes import webob from nova import context +from nova import exception from nova import flags from nova import utils from nova import wsgi @@ -205,10 +206,12 @@ class ServiceWrapper(wsgi.Controller): # NOTE(vish): make sure we have no unicode keys for py2.6. params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) - if type(result) is dict or type(result) is list: - return self._serialize(result, req.best_match_content_type()) - else: + if result is None or type(result) is str or type(result) is unicode: return result + try: + return self._serialize(result, req.best_match_content_type()) + except: + raise exception.Error("returned non-serializable type: %s" % result) class Limited(object): diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 001246fc4..383840234 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -33,6 +33,9 @@ from nova.api import direct from nova.tests import test_cloud +class ArbitraryObject(object): + pass + class FakeService(object): def echo(self, context, data): return {'data': data} @@ -41,6 +44,9 @@ class FakeService(object): return {'user': context.user_id, 'project': context.project_id} + def invalid_return(self, context): + return ArbitraryObject() + class DirectTestCase(test.TestCase): def setUp(self): @@ -86,6 +92,12 @@ class DirectTestCase(test.TestCase): resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') + def test_invalid(self): + req = webob.Request.blank('/fake/invalid_return') + req.environ['openstack.context'] = self.context + req.method = 'POST' + self.assertRaises(exception.Error, req.get_response, self.router) + def test_proxy(self): proxy = direct.Proxy(self.router) rv = proxy.fake.echo(self.context, data='baz') -- cgit From c3b98443263de944aa54ae4948330b6cfb9a02a6 Mon Sep 17 00:00:00 2001 From: Ilya Alekseyev Date: Thu, 24 Mar 2011 23:00:16 +0300 Subject: style and spacing fixed --- nova/utils.py | 2 +- nova/virt/libvirt_conn.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index d3375abc3..29e33ad9e 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -316,7 +316,7 @@ def to_global_ipv6(prefix, mac): mac64_addr = netaddr.IPAddress(int_addr) maskIP = netaddr.IPNetwork(prefix).ip return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).\ - format() + format() except TypeError: raise TypeError(_("Bad mac for to_global_ipv6: %s") % mac) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0211cb4d8..f41f4df5e 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -154,7 +154,7 @@ def _get_ip_version(cidr): def _get_network_info(instance): - #TODO(ilyaalekseyev) If we will keep this function + # TODO(adiantum) If we will keep this function # we should cache network_info admin_context = context.get_admin_context() @@ -837,7 +837,7 @@ class LibvirtConnection(driver.ComputeDriver): # TODO(termie): cache? LOG.debug(_('instance %s: starting toXML method'), instance['name']) - #TODO(ilyaalekseyev) remove network_info creation code + # TODO(adiantum) remove network_info creation code # when multinics will be completed if not network_info: network_info = _get_network_info(instance) -- cgit From c5cbec20d2785d3060d57b55a264fbf936709500 Mon Sep 17 00:00:00 2001 From: termie Date: Thu, 24 Mar 2011 13:20:15 -0700 Subject: pep8 cleanups --- bin/nova-direct-api | 1 - nova/api/direct.py | 3 ++- nova/api/ec2/cloud.py | 4 +++- nova/tests/test_direct.py | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/nova-direct-api b/bin/nova-direct-api index bb3aa8ae7..83ec72722 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -52,7 +52,6 @@ flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) - # An example of an API that only exposes read-only methods. # In this case we're just limiting which methods are exposed. class ReadOnlyCompute(direct.Limited): diff --git a/nova/api/direct.py b/nova/api/direct.py index bb2ace1c9..e5f33cee4 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -211,7 +211,8 @@ class ServiceWrapper(wsgi.Controller): try: return self._serialize(result, req.best_match_content_type()) except: - raise exception.Error("returned non-serializable type: %s" % result) + raise exception.Error("returned non-serializable type: %s" + % result) class Limited(object): diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 5d31d71d3..0da642318 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -608,7 +608,9 @@ class CloudController(object): if field in kwargs: changes[field] = kwargs[field] if changes: - self.volume_api.update(context, volume_id=volume_id, fields=changes) + self.volume_api.update(context, + volume_id=volume_id, + fields=changes) return True def attach_volume(self, context, volume_id, instance_id, device, **kwargs): diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 383840234..588a24b35 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -36,6 +36,7 @@ from nova.tests import test_cloud class ArbitraryObject(object): pass + class FakeService(object): def echo(self, context, data): return {'data': data} -- cgit From 3426a9296c6f7d249a9c57ba9e614045ffe2f3c7 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 24 Mar 2011 16:11:48 -0500 Subject: Review feedback --- nova/virt/xenapi/vmops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 83d5e8310..419b9ad90 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -684,7 +684,7 @@ class VMOps(object): """ rescue_vm_ref = VMHelper.lookup(self._session, - str(instance.name) + "-rescue") + "%s-rescue" % instance.name) if rescue_vm_ref: raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) @@ -712,7 +712,7 @@ class VMOps(object): """ rescue_vm_ref = VMHelper.lookup(self._session, - str(instance.name) + "-rescue") + "%s-rescue" % instance.name) if not rescue_vm_ref: raise exception.NotFound(_( -- cgit