From 60c82177da9c4ebbb89e5534959d0d5a52bfa49a Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 3 Nov 2010 12:38:15 -0700 Subject: Fix for bug#613264, allowing hosts to be specified for nova-api and objectstore listeners. --- bin/nova-api | 6 ++++-- nova/objectstore/handler.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a9002ae2d..a9c53dbcd 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -38,15 +38,17 @@ from nova import server FLAGS = flags.FLAGS flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') +flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') +flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') def main(_args): from nova import api from nova import wsgi server = wsgi.Server() - server.start(api.API('os'), FLAGS.osapi_port) - server.start(api.API('ec2'), FLAGS.ec2api_port) + server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) + server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) server.wait() diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index b26906001..aaf207db4 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -438,6 +438,7 @@ def get_application(): # Disabled because of lack of proper introspection in Twisted # or possibly different versions of twisted? # pylint: disable-msg=E1101 - objectStoreService = internet.TCPServer(FLAGS.s3_port, factory) + objectStoreService = internet.TCPServer(FLAGS.s3_port, factory, + interface=FLAGS.s3_host) objectStoreService.setServiceParent(application) return application -- cgit From d65c35bcadc6cc4e4d1fc61502d43fd001ce2f0e Mon Sep 17 00:00:00 2001 From: Eric Day Date: Wed, 3 Nov 2010 13:13:59 -0700 Subject: Added an extra argument to the objectstore listen to separate out the listening host from the connecting host. --- nova/objectstore/handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index aaf207db4..c8920b00c 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -61,6 +61,7 @@ from nova.objectstore import image FLAGS = flags.FLAGS +flags.DEFINE_string('s3_listen_host', '', 'Host to listen on.') def render_xml(request, value): @@ -439,6 +440,6 @@ def get_application(): # or possibly different versions of twisted? # pylint: disable-msg=E1101 objectStoreService = internet.TCPServer(FLAGS.s3_port, factory, - interface=FLAGS.s3_host) + interface=FLAGS.s3_listen_host) objectStoreService.setServiceParent(application) return application -- cgit From 6c142b844ac3269633414d235a8a7bd83c2ecca5 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Fri, 12 Nov 2010 18:50:10 +0000 Subject: getting started --- doc/source/devref/rabbit.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/source/devref/rabbit.rst diff --git a/doc/source/devref/rabbit.rst b/doc/source/devref/rabbit.rst new file mode 100644 index 000000000..48cdce852 --- /dev/null +++ b/doc/source/devref/rabbit.rst @@ -0,0 +1 @@ +placeholder -- cgit From a1cdb02360cc18acb1b7836ce6a07dffd2481635 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Tue, 16 Nov 2010 18:37:19 +0000 Subject: First dump of content related to Nova RPC and RabbitMQ --- doc/source/devref/rabbit.rst | 114 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/doc/source/devref/rabbit.rst b/doc/source/devref/rabbit.rst index 48cdce852..a58f838a4 100644 --- a/doc/source/devref/rabbit.rst +++ b/doc/source/devref/rabbit.rst @@ -1 +1,113 @@ -placeholder +.. + Copyright 2010 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + Figures and content Copyright 2010 Citrix + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +RabbitMQ and Nova +================= + +RabbitMQ is the messaging component chosen by the OpenStack cloud. RabbitMQ sits between any two Nova components and allows them to communicate in a loosely coupled fashion. More precisely, Nova components (the compute fabric of OpenStack) use Remote Procedure Calls to communicate to one another; however such a paradigm is built atop the publish/subscribe paradigm so that the following benefits can be achieved: + + * Decoupling between client and servant (i.e. the client does not need to know where the servant's reference is); + * Full a-synchronism between client and servant (i.e. the client does not need the servant to run at the same time of the remote call); + * Random balancing of remote calls (i.e. if more servants are up and running, one-way calls are transparently dispatched to the first available servant); + +Nova (Austin release) uses both direct and topic-based exchanges. The architecture looks like the one depicted in the Figure 1: + +.. todo:: Figure 1 + +Nova implements RPC calls (both request/response - named call - and one-way - named cast -) over AMQP by providing an adapter class which take cares of marshalling and unmarshalling of messages into function calls. Each Nova service (e.g. Compute, Volume, etc.) create two queues at the initialization time, one which accepts messages with routing keys 'NODE-TYPE.NODE-ID' (e.g. compute.hostname) and another, which accepts messages with routing keys as generic 'NODE-TYPE' (e.g. compute). The former is used specifically when Nova-API needs to redirect commands to a specific node like 'euca-terminate instance'. In this case, only the compute node whose host's hypervisor is running the virtual machine can kill the instance. The API acts as a consumer when RPC calls are request/response, otherwise is acts as publisher only. + +Nova RPC Mappings +================= + +Figure 2 shows the internals of a RabbitMQ node when a single instance is deployed and shared in an OpenStack cloud. Every Nova component connects to the RabbitMQ instance and, depending on its personality (e.g. a compute node or a network node), may use the queue either as an Invoker (such as API or Scheduler) or a Worker (such as Compute, Volume or Network). Invokers and Workers do not actually exist in the Nova object model, but we are going to use them as an abstraction for sake of clarity. An Invoker is a component that sends messages in the queuing system via two operations: i) rpc.call (request/response) and ii) rpc.cast (one-way); a Worker is a component that receives messages from the queuing system and reply accordingly to rcp.call operations. Figure 2 shows the following internal elements: + + * Topic Publisher: a Topic Publisher comes to life when an rpc.call or an rpc.cast operation is executed; this object is instantiated and used to push a message to the queuing system. Every publisher connects always to the same topic-based exchange; its life-cycle is limited to the message delivery; + * Direct Consumer: a Direct Consumer comes to life if (an only if) a rpc.call operation is executed; this object is instantiated and used to receive a response message from the queuing system; Every consumer connects to a unique direct-based exchange via a unique exclusive queue; its life-cycle is limited to the message delivery; the exchange and queue identifiers are determined by a UUID generator, and are marshalled in the message sent by the Topic Publisher (only rpc.call operations); + * Topic Consumer: a Topic Consumer comes to life as soon as a Worker is instantiated and exists throughout its life-cycle; this object is used to receive messages from the queue and it invokes the appropriate action as defined by the Worker role. A Topic Consumer connects to the same topic-based exchange either via a shared queue or via a unique exclusive queue. Every Worker has two topic consumers, one that is addressed only during rpc.cast operations (and it connects to a shared queue whose exchange key is 'topic') and the other that is addressed only during rpc.call operations (and it connects to a unique queue whose exchange key is 'topic.host'). + * Direct Publisher: a Direct Publisher comes to life only during rpc.call operations and it is instantiated to return the message required by the request/response operation. The object connects to a direct-based exchange whose identity is dictated by the incoming message. + * Topic Exchange: The Exchange is a routing table that exists in the context of a virtual host (the multi-tenancy mechanism provided by RabbitMQ); its type (i.e. topic vs. direct) determines the routing policy; a RabbitMQ node will have only one topic-based exchange for every topic in Nova; + * Direct Exchange: this is a routing table that is created during rpc.call operations; there are many instances of this kind of exchange throughout the life-cycle of a RabbitMQ node, one for each rpc.call invoked; + * Queue Element: A Queue is a message bucket. Messages are kept in the queue until a Consumer (either Topic or Direct Consumer) connects to the queue and fetch it. Queues can be shared or can be exclusive. Queues whose routing key is 'topic' are shared amongst Workers of the same personality; + +.. todo:: Figure 2 + +RPC Calls +========= + +Figure 3 shows the message flow during an rp.call operation: i) a Topic Publisher is instantiated to send the message request to the queuing system; immediately before the publishing operation, a Direct Consumer is instantiated to wait for the response message; ii) once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (i.e. 'topic.host') and passed to the Worker in charge of the task; iii) once the task is completed, a Direct Publisher is allocated to send the response message to the queuing system; iv) once the message is dispatched by the exchange, it is fetched by the Direct Consumer dictated by the routing key (i.e. 'msg_id') and passed to the Invoker; + +.. todo:: Figure 3 + +RPC Casts +========= + +Figure 4 shows the message flow during an rp.cast operation: i) a Topic Publisher is instantiated to send the message request to the queuing system; ii) once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (i.e. 'topic) and passed to the Worker in charge of the task; + +.. todo:: Figure 4 + +RabbitMQ Load +============= + +At any given time the load of a RabbitMQ node is function of the following parameters: + + * Throughput of API calls: the number of API calls (more precisely rpc.call ops) being served by the OpenStack cloud dictates the number of direct-based exchanges, related queues and direct consumers connected to them; + * Number of Workers: there is one queue shared amongst workers with the same personality; however there are as many exclusive queues as the number of workers; the number of workers dictates also the number of routing keys within the +topic-based exchange, which is shared amongst all workers; + +Figure 5 shows the status of the RabbitMQ node after Nova components' bootstrap in a test environment. Queues being created by Nova components are: i) nova-exchange; ii) volume; iii) compute; iv) cloud; v) network; vi) network.phonic; +vii) volume.phonic; viii) compute.phonic; ix) compute.scheduler. Figure 6 shows the status of the RabbitMQ node after an rpc.call operation. In the latter case there is a direct exchange and a queue with the same name. + +.. todo:: Figure 5 + +RabbitMQ Gotchas +================ + +Nova uses Carrot to connect to the RabbitMQ environment. Carrot is a Python library that in turn uses AMQPLib, a library that implements the standard AMQP 0.8 at the time of writing. When using Carrot, Invokers and Workers need the following parameters in order to instantiate a Connection object that connects to the RabbitMQ server: + + * Hostname: The hostname to the AMQP server; + * Userid: A valid username used to authenticate to the server; + * Password: The password used to authenticate to the server; + * Virtual_host: The name of the virtual host to work with. This virtual host must exist on the server, and the user must have access to it. Default is "/"; + * Port: The port of the AMQP server. Default is 5672 (amqp); + +The following parameters are default: + + * Insist: insist on connecting to a server. In a configuration with multiple load-sharing servers, the Insist option tells the server that the client is insisting on a connection to the specified server. Default is False; + * Connect_timeout: the timeout in seconds before the client gives up connecting to the server. The default is no timeout; + * SSL: use SSL to connect to the server. The default is False; + More precisely Consumers need the following parameters: + * Connection: the above mentioned Connection object; + * Queue: name of the queue; + * Exchange: name of the exchange the queue binds to; + * Routing_key: the interpretation of the routing key depends on the value of the exchange_type attribute: + * Direct exchange: if the routing key property of the message and the routing_key attribute of the queue are identical, then the message is forwarded to the queue; + * Fanout exchange: messages are forwarded to the queues bound the exchange, even if the binding does not have a key; + * Topic exchange: if the routing key property of the message matches the routing key of the key according to a primitive pattern matching scheme, then the message is forwarded to the queue. The message routing key then consists of words separated by dots (".", like domain names), and two special characters are available; star ("") and hash ("#"). The star matches any word, and the hash matches zero or more words. For example ".stock.#" matches the routing keys "usd.stock" and "eur.stock.db" but not "stock.nasdaq"; + * Durable: this flag determines the durability of both exchanges and queues; durable exchanges and queues remain active when a RabbitMQ server restarts. Non-durable exchanges/queues (transient exchanges/queues) are purged when a server restarts. It is worth noting that AMQP specifies that durable queues cannot bind to transient exchanges. Default is True; + * Auto_delete: if set, the exchange is deleted when all queues have finished using it. Default is False. + * Exclusive: exclusive queues (i.e. non-shared) may only be consumed from by the current connection. When exclusive is on, this also implies auto_delete. Default is False; + * Exchange_type: AMQP defines several default exchange types (routing algorithms) that covers most of the common messaging use cases; + * Auto_ack: acknowledgement is handled automatically once messages are received. By default auto_ack is set to False, and the receiver is required to manually handle acknowledgment; + * No_ack: it disable acknowledgement on the server-side. This is different from auto_ack in that acknowledgement is turned off altogether. This functionality increases performance but at the cost of reliability. Messages can get lost if a client dies before it can deliver them to the application; + * Auto_declare: if this is True and the exchange name is set, the exchange will be automatically declared at instantiation. Auto declare is on by default; + Publishers specify most the parameters of Consumers (i.e. they do not specify a queue name), but they can also specify the following: + * Delivery_mode: the default delivery mode used for messages. The value is an integer. The following delivery modes are supported by RabbitMQ: + o 1 or "transient": the message is transient. Which means it is stored in memory only, and is lost if the server dies or restarts. + o 2 or "persistent": the message is persistent. Which means the message is stored both in-memory, and on disk, and therefore preserved if the server dies or restarts. + +The default value is 2 (persistent). During a send operation, Publishers can override the delivery mode of messages so that, for example, transient messages can be sent over a durable queue. -- cgit From 56c2df202d89f0954ab5a10284b3ee4d6111bc8b Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Wed, 17 Nov 2010 13:20:55 +0000 Subject: Further editing and added images --- doc/source/devref/index.rst | 7 ++ doc/source/devref/rabbit.rst | 148 +++++++++++++++++++++++-------------- doc/source/images/rabbit/arch.png | Bin 0 -> 26690 bytes doc/source/images/rabbit/flow1.png | Bin 0 -> 40982 bytes doc/source/images/rabbit/flow2.png | Bin 0 -> 30650 bytes doc/source/images/rabbit/rabt.png | Bin 0 -> 44964 bytes doc/source/images/rabbit/state.png | Bin 0 -> 38543 bytes 7 files changed, 99 insertions(+), 56 deletions(-) create mode 100644 doc/source/images/rabbit/arch.png create mode 100644 doc/source/images/rabbit/flow1.png create mode 100644 doc/source/images/rabbit/flow2.png create mode 100644 doc/source/images/rabbit/rabt.png create mode 100644 doc/source/images/rabbit/state.png diff --git a/doc/source/devref/index.rst b/doc/source/devref/index.rst index 6a93e3e18..e9c28305c 100644 --- a/doc/source/devref/index.rst +++ b/doc/source/devref/index.rst @@ -26,6 +26,13 @@ Programming HowTos and Tutorials .. todo:: Add some programming howtos and tuts +Programming Concepts +-------------------- +.. toctree:: + :maxdepth: 3 + + rabbit + API Reference ------------- .. toctree:: diff --git a/doc/source/devref/rabbit.rst b/doc/source/devref/rabbit.rst index a58f838a4..d285e3198 100644 --- a/doc/source/devref/rabbit.rst +++ b/doc/source/devref/rabbit.rst @@ -1,9 +1,6 @@ .. - Copyright 2010 United States Government as represented by the - Administrator of the National Aeronautics and Space Administration. - Figures and content Copyright 2010 Citrix - All Rights Reserved. - + Copyright (c) 2010 Citrix Systems, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -19,95 +16,134 @@ RabbitMQ and Nova ================= -RabbitMQ is the messaging component chosen by the OpenStack cloud. RabbitMQ sits between any two Nova components and allows them to communicate in a loosely coupled fashion. More precisely, Nova components (the compute fabric of OpenStack) use Remote Procedure Calls to communicate to one another; however such a paradigm is built atop the publish/subscribe paradigm so that the following benefits can be achieved: +RabbitMQ is the messaging component chosen by the OpenStack cloud. RabbitMQ sits between any two Nova components and allows them to communicate in a loosely coupled fashion. More precisely, Nova components (the compute fabric of OpenStack) use Remote Procedure Calls (RPC hereinafter) to communicate to one another; however such a paradigm is built atop the publish/subscribe paradigm so that the following benefits can be achieved: + + * Decoupling between client and servant (such as the client does not need to know where the servant's reference is). + * Full a-synchronism between client and servant (such as the client does not need the servant to run at the same time of the remote call). + * Random balancing of remote calls (such as if more servants are up and running, one-way calls are transparently dispatched to the first available servant). - * Decoupling between client and servant (i.e. the client does not need to know where the servant's reference is); - * Full a-synchronism between client and servant (i.e. the client does not need the servant to run at the same time of the remote call); - * Random balancing of remote calls (i.e. if more servants are up and running, one-way calls are transparently dispatched to the first available servant); +Nova (Austin release) uses both direct and topic-based exchanges. The architecture looks like the one depicted in the figure below: -Nova (Austin release) uses both direct and topic-based exchanges. The architecture looks like the one depicted in the Figure 1: +.. image:: /images/rabbit/arch.png + :width: 100% -.. todo:: Figure 1 +.. -Nova implements RPC calls (both request/response - named call - and one-way - named cast -) over AMQP by providing an adapter class which take cares of marshalling and unmarshalling of messages into function calls. Each Nova service (e.g. Compute, Volume, etc.) create two queues at the initialization time, one which accepts messages with routing keys 'NODE-TYPE.NODE-ID' (e.g. compute.hostname) and another, which accepts messages with routing keys as generic 'NODE-TYPE' (e.g. compute). The former is used specifically when Nova-API needs to redirect commands to a specific node like 'euca-terminate instance'. In this case, only the compute node whose host's hypervisor is running the virtual machine can kill the instance. The API acts as a consumer when RPC calls are request/response, otherwise is acts as publisher only. +Nova implements RPC (both request+response, and one-way, respectively nicknamed 'rpc.call' and 'rpc.cast') over AMQP by providing an adapter class which take cares of marshalling and unmarshalling of messages into function calls. Each Nova service (for example Compute, Volume, etc.) create two queues at the initialization time, one which accepts messages with routing keys 'NODE-TYPE.NODE-ID' (for example compute.hostname) and another, which accepts messages with routing keys as generic 'NODE-TYPE' (for example compute). The former is used specifically when Nova-API needs to redirect commands to a specific node like 'euca-terminate instance'. In this case, only the compute node whose host's hypervisor is running the virtual machine can kill the instance. The API acts as a consumer when RPC calls are request/response, otherwise is acts as publisher only. Nova RPC Mappings ================= -Figure 2 shows the internals of a RabbitMQ node when a single instance is deployed and shared in an OpenStack cloud. Every Nova component connects to the RabbitMQ instance and, depending on its personality (e.g. a compute node or a network node), may use the queue either as an Invoker (such as API or Scheduler) or a Worker (such as Compute, Volume or Network). Invokers and Workers do not actually exist in the Nova object model, but we are going to use them as an abstraction for sake of clarity. An Invoker is a component that sends messages in the queuing system via two operations: i) rpc.call (request/response) and ii) rpc.cast (one-way); a Worker is a component that receives messages from the queuing system and reply accordingly to rcp.call operations. Figure 2 shows the following internal elements: +The figure below shows the internals of a RabbitMQ node when a single instance is deployed and shared in an OpenStack cloud. Every Nova component connects to the RabbitMQ instance and, depending on its personality (for example a compute node or a network node), may use the queue either as an Invoker (such as API or Scheduler) or a Worker (such as Compute, Volume or Network). Invokers and Workers do not actually exist in the Nova object model, but we are going to use them as an abstraction for sake of clarity. An Invoker is a component that sends messages in the queuing system via two operations: 1) rpc.call and ii) rpc.cast; a Worker is a component that receives messages from the queuing system and reply accordingly to rcp.call operations. + +Figure 2 shows the following internal elements: - * Topic Publisher: a Topic Publisher comes to life when an rpc.call or an rpc.cast operation is executed; this object is instantiated and used to push a message to the queuing system. Every publisher connects always to the same topic-based exchange; its life-cycle is limited to the message delivery; - * Direct Consumer: a Direct Consumer comes to life if (an only if) a rpc.call operation is executed; this object is instantiated and used to receive a response message from the queuing system; Every consumer connects to a unique direct-based exchange via a unique exclusive queue; its life-cycle is limited to the message delivery; the exchange and queue identifiers are determined by a UUID generator, and are marshalled in the message sent by the Topic Publisher (only rpc.call operations); + * Topic Publisher: a Topic Publisher comes to life when an rpc.call or an rpc.cast operation is executed; this object is instantiated and used to push a message to the queuing system. Every publisher connects always to the same topic-based exchange; its life-cycle is limited to the message delivery. + * Direct Consumer: a Direct Consumer comes to life if (an only if) a rpc.call operation is executed; this object is instantiated and used to receive a response message from the queuing system; Every consumer connects to a unique direct-based exchange via a unique exclusive queue; its life-cycle is limited to the message delivery; the exchange and queue identifiers are determined by a UUID generator, and are marshalled in the message sent by the Topic Publisher (only rpc.call operations). * Topic Consumer: a Topic Consumer comes to life as soon as a Worker is instantiated and exists throughout its life-cycle; this object is used to receive messages from the queue and it invokes the appropriate action as defined by the Worker role. A Topic Consumer connects to the same topic-based exchange either via a shared queue or via a unique exclusive queue. Every Worker has two topic consumers, one that is addressed only during rpc.cast operations (and it connects to a shared queue whose exchange key is 'topic') and the other that is addressed only during rpc.call operations (and it connects to a unique queue whose exchange key is 'topic.host'). * Direct Publisher: a Direct Publisher comes to life only during rpc.call operations and it is instantiated to return the message required by the request/response operation. The object connects to a direct-based exchange whose identity is dictated by the incoming message. - * Topic Exchange: The Exchange is a routing table that exists in the context of a virtual host (the multi-tenancy mechanism provided by RabbitMQ); its type (i.e. topic vs. direct) determines the routing policy; a RabbitMQ node will have only one topic-based exchange for every topic in Nova; - * Direct Exchange: this is a routing table that is created during rpc.call operations; there are many instances of this kind of exchange throughout the life-cycle of a RabbitMQ node, one for each rpc.call invoked; - * Queue Element: A Queue is a message bucket. Messages are kept in the queue until a Consumer (either Topic or Direct Consumer) connects to the queue and fetch it. Queues can be shared or can be exclusive. Queues whose routing key is 'topic' are shared amongst Workers of the same personality; + * Topic Exchange: The Exchange is a routing table that exists in the context of a virtual host (the multi-tenancy mechanism provided by RabbitMQ); its type (such as topic vs. direct) determines the routing policy; a RabbitMQ node will have only one topic-based exchange for every topic in Nova. + * Direct Exchange: this is a routing table that is created during rpc.call operations; there are many instances of this kind of exchange throughout the life-cycle of a RabbitMQ node, one for each rpc.call invoked. + * Queue Element: A Queue is a message bucket. Messages are kept in the queue until a Consumer (either Topic or Direct Consumer) connects to the queue and fetch it. Queues can be shared or can be exclusive. Queues whose routing key is 'topic' are shared amongst Workers of the same personality. -.. todo:: Figure 2 +.. image:: /images/rabbit/rabt.png + :width: 100% + +.. RPC Calls ========= -Figure 3 shows the message flow during an rp.call operation: i) a Topic Publisher is instantiated to send the message request to the queuing system; immediately before the publishing operation, a Direct Consumer is instantiated to wait for the response message; ii) once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (i.e. 'topic.host') and passed to the Worker in charge of the task; iii) once the task is completed, a Direct Publisher is allocated to send the response message to the queuing system; iv) once the message is dispatched by the exchange, it is fetched by the Direct Consumer dictated by the routing key (i.e. 'msg_id') and passed to the Invoker; +The diagram below shows the message flow during an rp.call operation: + + 1. a Topic Publisher is instantiated to send the message request to the queuing system; immediately before the publishing operation, a Direct Consumer is instantiated to wait for the response message. + 2. once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (such as 'topic.host') and passed to the Worker in charge of the task. + 3. once the task is completed, a Direct Publisher is allocated to send the response message to the queuing system. + 4. once the message is dispatched by the exchange, it is fetched by the Direct Consumer dictated by the routing key (such as 'msg_id') and passed to the Invoker. -.. todo:: Figure 3 +.. image:: /images/rabbit/flow1.png + :width: 100% + +.. RPC Casts ========= -Figure 4 shows the message flow during an rp.cast operation: i) a Topic Publisher is instantiated to send the message request to the queuing system; ii) once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (i.e. 'topic) and passed to the Worker in charge of the task; +The diagram below the message flow during an rp.cast operation: -.. todo:: Figure 4 + 1. a Topic Publisher is instantiated to send the message request to the queuing system. + 2. once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (such as 'topic') and passed to the Worker in charge of the task. + +.. image:: /images/rabbit/flow2.png + :width: 100% + +.. RabbitMQ Load ============= At any given time the load of a RabbitMQ node is function of the following parameters: - * Throughput of API calls: the number of API calls (more precisely rpc.call ops) being served by the OpenStack cloud dictates the number of direct-based exchanges, related queues and direct consumers connected to them; - * Number of Workers: there is one queue shared amongst workers with the same personality; however there are as many exclusive queues as the number of workers; the number of workers dictates also the number of routing keys within the -topic-based exchange, which is shared amongst all workers; - -Figure 5 shows the status of the RabbitMQ node after Nova components' bootstrap in a test environment. Queues being created by Nova components are: i) nova-exchange; ii) volume; iii) compute; iv) cloud; v) network; vi) network.phonic; -vii) volume.phonic; viii) compute.phonic; ix) compute.scheduler. Figure 6 shows the status of the RabbitMQ node after an rpc.call operation. In the latter case there is a direct exchange and a queue with the same name. + * Throughput of API calls: the number of API calls (more precisely rpc.call ops) being served by the OpenStack cloud dictates the number of direct-based exchanges, related queues and direct consumers connected to them. + * Number of Workers: there is one queue shared amongst workers with the same personality; however there are as many exclusive queues as the number of workers; the number of workers dictates also the number of routing keys within the topic-based exchange, which is shared amongst all workers. + +The figure below shows the status of the RabbitMQ node after Nova components' bootstrap in a test environment. Queues being created by Nova components are: + + * Exchanges + 1. nova (topic exchange) + * Queues + 1. compute.phantom (phantom is hostname) + 2. compute + 3. network.phantom (phantom is hostname) + 4. network + 5. volume.phantom (phantom is hostname) + 6. volume + 7. scheduler.phantom (phantom is hostname) + 8. scheduler + +.. image:: /images/rabbit/state.png + :width: 100% -.. todo:: Figure 5 +.. RabbitMQ Gotchas ================ -Nova uses Carrot to connect to the RabbitMQ environment. Carrot is a Python library that in turn uses AMQPLib, a library that implements the standard AMQP 0.8 at the time of writing. When using Carrot, Invokers and Workers need the following parameters in order to instantiate a Connection object that connects to the RabbitMQ server: +Nova uses Carrot to connect to the RabbitMQ environment. Carrot is a Python library that in turn uses AMQPLib, a library that implements the standard AMQP 0.8 at the time of writing. When using Carrot, Invokers and Workers need the following parameters in order to instantiate a Connection object that connects to the RabbitMQ server (please note that most of the following material can be also found in the Carrot documentation; it has been summarized and revised here for sake of clarity): - * Hostname: The hostname to the AMQP server; - * Userid: A valid username used to authenticate to the server; - * Password: The password used to authenticate to the server; - * Virtual_host: The name of the virtual host to work with. This virtual host must exist on the server, and the user must have access to it. Default is "/"; - * Port: The port of the AMQP server. Default is 5672 (amqp); + * Hostname: The hostname to the AMQP server. + * Userid: A valid username used to authenticate to the server. + * Password: The password used to authenticate to the server. + * Virtual_host: The name of the virtual host to work with. This virtual host must exist on the server, and the user must have access to it. Default is "/". + * Port: The port of the AMQP server. Default is 5672 (amqp). The following parameters are default: - * Insist: insist on connecting to a server. In a configuration with multiple load-sharing servers, the Insist option tells the server that the client is insisting on a connection to the specified server. Default is False; - * Connect_timeout: the timeout in seconds before the client gives up connecting to the server. The default is no timeout; - * SSL: use SSL to connect to the server. The default is False; - More precisely Consumers need the following parameters: - * Connection: the above mentioned Connection object; - * Queue: name of the queue; - * Exchange: name of the exchange the queue binds to; - * Routing_key: the interpretation of the routing key depends on the value of the exchange_type attribute: - * Direct exchange: if the routing key property of the message and the routing_key attribute of the queue are identical, then the message is forwarded to the queue; - * Fanout exchange: messages are forwarded to the queues bound the exchange, even if the binding does not have a key; - * Topic exchange: if the routing key property of the message matches the routing key of the key according to a primitive pattern matching scheme, then the message is forwarded to the queue. The message routing key then consists of words separated by dots (".", like domain names), and two special characters are available; star ("") and hash ("#"). The star matches any word, and the hash matches zero or more words. For example ".stock.#" matches the routing keys "usd.stock" and "eur.stock.db" but not "stock.nasdaq"; - * Durable: this flag determines the durability of both exchanges and queues; durable exchanges and queues remain active when a RabbitMQ server restarts. Non-durable exchanges/queues (transient exchanges/queues) are purged when a server restarts. It is worth noting that AMQP specifies that durable queues cannot bind to transient exchanges. Default is True; + * Insist: insist on connecting to a server. In a configuration with multiple load-sharing servers, the Insist option tells the server that the client is insisting on a connection to the specified server. Default is False. + * Connect_timeout: the timeout in seconds before the client gives up connecting to the server. The default is no timeout. + * SSL: use SSL to connect to the server. The default is False. + +More precisely Consumers need the following parameters: + + * Connection: the above mentioned Connection object. + * Queue: name of the queue. + * Exchange: name of the exchange the queue binds to. + * Routing_key: the interpretation of the routing key depends on the value of the exchange_type attribute. + + ** Direct exchange: if the routing key property of the message and the routing_key attribute of the queue are identical, then the message is forwarded to the queue. + ** Fanout exchange: messages are forwarded to the queues bound the exchange, even if the binding does not have a key. + ** Topic exchange: if the routing key property of the message matches the routing key of the key according to a primitive pattern matching scheme, then the message is forwarded to the queue. The message routing key then consists of words separated by dots (".", like domain names), and two special characters are available; star ("") and hash ("#"). The star matches any word, and the hash matches zero or more words. For example ".stock.#" matches the routing keys "usd.stock" and "eur.stock.db" but not "stock.nasdaq". + + * Durable: this flag determines the durability of both exchanges and queues; durable exchanges and queues remain active when a RabbitMQ server restarts. Non-durable exchanges/queues (transient exchanges/queues) are purged when a server restarts. It is worth noting that AMQP specifies that durable queues cannot bind to transient exchanges. Default is True. * Auto_delete: if set, the exchange is deleted when all queues have finished using it. Default is False. - * Exclusive: exclusive queues (i.e. non-shared) may only be consumed from by the current connection. When exclusive is on, this also implies auto_delete. Default is False; - * Exchange_type: AMQP defines several default exchange types (routing algorithms) that covers most of the common messaging use cases; - * Auto_ack: acknowledgement is handled automatically once messages are received. By default auto_ack is set to False, and the receiver is required to manually handle acknowledgment; - * No_ack: it disable acknowledgement on the server-side. This is different from auto_ack in that acknowledgement is turned off altogether. This functionality increases performance but at the cost of reliability. Messages can get lost if a client dies before it can deliver them to the application; - * Auto_declare: if this is True and the exchange name is set, the exchange will be automatically declared at instantiation. Auto declare is on by default; - Publishers specify most the parameters of Consumers (i.e. they do not specify a queue name), but they can also specify the following: + * Exclusive: exclusive queues (such as non-shared) may only be consumed from by the current connection. When exclusive is on, this also implies auto_delete. Default is False. + * Exchange_type: AMQP defines several default exchange types (routing algorithms) that covers most of the common messaging use cases. + * Auto_ack: acknowledgement is handled automatically once messages are received. By default auto_ack is set to False, and the receiver is required to manually handle acknowledgment. + * No_ack: it disable acknowledgement on the server-side. This is different from auto_ack in that acknowledgement is turned off altogether. This functionality increases performance but at the cost of reliability. Messages can get lost if a client dies before it can deliver them to the application. + * Auto_declare: if this is True and the exchange name is set, the exchange will be automatically declared at instantiation. Auto declare is on by default. + Publishers specify most the parameters of Consumers (such as they do not specify a queue name), but they can also specify the following: * Delivery_mode: the default delivery mode used for messages. The value is an integer. The following delivery modes are supported by RabbitMQ: - o 1 or "transient": the message is transient. Which means it is stored in memory only, and is lost if the server dies or restarts. - o 2 or "persistent": the message is persistent. Which means the message is stored both in-memory, and on disk, and therefore preserved if the server dies or restarts. + ** 1 or "transient": the message is transient. Which means it is stored in memory only, and is lost if the server dies or restarts. + ** 2 or "persistent": the message is persistent. Which means the message is stored both in-memory, and on disk, and therefore preserved if the server dies or restarts. The default value is 2 (persistent). During a send operation, Publishers can override the delivery mode of messages so that, for example, transient messages can be sent over a durable queue. diff --git a/doc/source/images/rabbit/arch.png b/doc/source/images/rabbit/arch.png new file mode 100644 index 000000000..8f7d535b6 Binary files /dev/null and b/doc/source/images/rabbit/arch.png differ diff --git a/doc/source/images/rabbit/flow1.png b/doc/source/images/rabbit/flow1.png new file mode 100644 index 000000000..ea325ad08 Binary files /dev/null and b/doc/source/images/rabbit/flow1.png differ diff --git a/doc/source/images/rabbit/flow2.png b/doc/source/images/rabbit/flow2.png new file mode 100644 index 000000000..19de2aafd Binary files /dev/null and b/doc/source/images/rabbit/flow2.png differ diff --git a/doc/source/images/rabbit/rabt.png b/doc/source/images/rabbit/rabt.png new file mode 100644 index 000000000..e3923b9a7 Binary files /dev/null and b/doc/source/images/rabbit/rabt.png differ diff --git a/doc/source/images/rabbit/state.png b/doc/source/images/rabbit/state.png new file mode 100644 index 000000000..76ce675a9 Binary files /dev/null and b/doc/source/images/rabbit/state.png differ -- cgit From 0c19386f7c4ca063edbf8c10ffb86b399884e457 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Wed, 17 Nov 2010 16:20:50 +0000 Subject: small edit --- doc/source/devref/rabbit.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/devref/rabbit.rst b/doc/source/devref/rabbit.rst index d285e3198..9720b8bc2 100644 --- a/doc/source/devref/rabbit.rst +++ b/doc/source/devref/rabbit.rst @@ -87,7 +87,7 @@ At any given time the load of a RabbitMQ node is function of the following param * Throughput of API calls: the number of API calls (more precisely rpc.call ops) being served by the OpenStack cloud dictates the number of direct-based exchanges, related queues and direct consumers connected to them. * Number of Workers: there is one queue shared amongst workers with the same personality; however there are as many exclusive queues as the number of workers; the number of workers dictates also the number of routing keys within the topic-based exchange, which is shared amongst all workers. -The figure below shows the status of the RabbitMQ node after Nova components' bootstrap in a test environment. Queues being created by Nova components are: +The figure below shows the status of the RabbitMQ node after Nova components' bootstrap in a test environment. Exchanges and queues being created by Nova components are: * Exchanges 1. nova (topic exchange) -- cgit From 37fcda35e7e409b746e0d99ca4392dcb4fc8ed01 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Wed, 17 Nov 2010 19:17:51 +0000 Subject: adjusting images size and bulleted list --- doc/source/devref/rabbit.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/source/devref/rabbit.rst b/doc/source/devref/rabbit.rst index 9720b8bc2..4468e575b 100644 --- a/doc/source/devref/rabbit.rst +++ b/doc/source/devref/rabbit.rst @@ -25,7 +25,7 @@ RabbitMQ is the messaging component chosen by the OpenStack cloud. RabbitMQ sit Nova (Austin release) uses both direct and topic-based exchanges. The architecture looks like the one depicted in the figure below: .. image:: /images/rabbit/arch.png - :width: 100% + :width: 60% .. @@ -47,7 +47,7 @@ Figure 2 shows the following internal elements: * Queue Element: A Queue is a message bucket. Messages are kept in the queue until a Consumer (either Topic or Direct Consumer) connects to the queue and fetch it. Queues can be shared or can be exclusive. Queues whose routing key is 'topic' are shared amongst Workers of the same personality. .. image:: /images/rabbit/rabt.png - :width: 100% + :width: 60% .. @@ -62,7 +62,7 @@ The diagram below shows the message flow during an rp.call operation: 4. once the message is dispatched by the exchange, it is fetched by the Direct Consumer dictated by the routing key (such as 'msg_id') and passed to the Invoker. .. image:: /images/rabbit/flow1.png - :width: 100% + :width: 60% .. @@ -75,7 +75,7 @@ The diagram below the message flow during an rp.cast operation: 2. once the message is dispatched by the exchange, it is fetched by the Topic Consumer dictated by the routing key (such as 'topic') and passed to the Worker in charge of the task. .. image:: /images/rabbit/flow2.png - :width: 100% + :width: 60% .. @@ -102,7 +102,7 @@ The figure below shows the status of the RabbitMQ node after Nova components' bo 8. scheduler .. image:: /images/rabbit/state.png - :width: 100% + :width: 60% .. @@ -130,9 +130,9 @@ More precisely Consumers need the following parameters: * Exchange: name of the exchange the queue binds to. * Routing_key: the interpretation of the routing key depends on the value of the exchange_type attribute. - ** Direct exchange: if the routing key property of the message and the routing_key attribute of the queue are identical, then the message is forwarded to the queue. - ** Fanout exchange: messages are forwarded to the queues bound the exchange, even if the binding does not have a key. - ** Topic exchange: if the routing key property of the message matches the routing key of the key according to a primitive pattern matching scheme, then the message is forwarded to the queue. The message routing key then consists of words separated by dots (".", like domain names), and two special characters are available; star ("") and hash ("#"). The star matches any word, and the hash matches zero or more words. For example ".stock.#" matches the routing keys "usd.stock" and "eur.stock.db" but not "stock.nasdaq". + * Direct exchange: if the routing key property of the message and the routing_key attribute of the queue are identical, then the message is forwarded to the queue. + * Fanout exchange: messages are forwarded to the queues bound the exchange, even if the binding does not have a key. + * Topic exchange: if the routing key property of the message matches the routing key of the key according to a primitive pattern matching scheme, then the message is forwarded to the queue. The message routing key then consists of words separated by dots (".", like domain names), and two special characters are available; star ("") and hash ("#"). The star matches any word, and the hash matches zero or more words. For example ".stock.#" matches the routing keys "usd.stock" and "eur.stock.db" but not "stock.nasdaq". * Durable: this flag determines the durability of both exchanges and queues; durable exchanges and queues remain active when a RabbitMQ server restarts. Non-durable exchanges/queues (transient exchanges/queues) are purged when a server restarts. It is worth noting that AMQP specifies that durable queues cannot bind to transient exchanges. Default is True. * Auto_delete: if set, the exchange is deleted when all queues have finished using it. Default is False. @@ -143,7 +143,8 @@ More precisely Consumers need the following parameters: * Auto_declare: if this is True and the exchange name is set, the exchange will be automatically declared at instantiation. Auto declare is on by default. Publishers specify most the parameters of Consumers (such as they do not specify a queue name), but they can also specify the following: * Delivery_mode: the default delivery mode used for messages. The value is an integer. The following delivery modes are supported by RabbitMQ: - ** 1 or "transient": the message is transient. Which means it is stored in memory only, and is lost if the server dies or restarts. - ** 2 or "persistent": the message is persistent. Which means the message is stored both in-memory, and on disk, and therefore preserved if the server dies or restarts. + + * 1 or "transient": the message is transient. Which means it is stored in memory only, and is lost if the server dies or restarts. + * 2 or "persistent": the message is persistent. Which means the message is stored both in-memory, and on disk, and therefore preserved if the server dies or restarts. The default value is 2 (persistent). During a send operation, Publishers can override the delivery mode of messages so that, for example, transient messages can be sent over a durable queue. -- cgit From aa433547ff797678cd2aad17d70c1c0b569d1e37 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Fri, 19 Nov 2010 17:32:38 +0000 Subject: first cut of fixes for bug #676128 --- nova/virt/xenapi.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 0f563aa41..2f115c14b 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -286,11 +286,21 @@ class XenAPIConnection(object): # Don't complain, just return. This lets us clean up instances # that have already disappeared from the underlying platform. defer.returnValue(None) + # Get the VDIs related to the VM + vdis = yield self._lookup_vm_vdis(vm) try: task = yield self._call_xenapi('Async.VM.hard_shutdown', vm) yield self._wait_for_task(task) except Exception, exc: logging.warn(exc) + # Disk clean-up + if vdis: + for vdi in vdis: + try: + task = yield self._call_xenapi('Async.VDI.destroy', vdi) + yield self._wait_for_task(task) + except Exception, exc: + logging.warn(exc) try: task = yield self._call_xenapi('Async.VM.destroy', vm) yield self._wait_for_task(task) @@ -325,6 +335,24 @@ class XenAPIConnection(object): else: return vms[0] + @utils.deferredToThread + def _lookup_vm_vdis(self, vm): + return self._lookup_vm_vdis_blocking(vm) + + def _lookup_vm_vdis_blocking(self, vm): + # Firstly we get the VBDs, then the VDIs. + # TODO: do we leave the read-only devices? + vbds = self._conn.xenapi.VM.get_VBDs(vm) + vdis = [] + if vbds: + for vbd in vbds: + vdis.append(self._conn.xenapi.VBD.get_VDI(vbd)) + if len(vdis) > 0: + return vdis + else: + return None + + def _wait_for_task(self, task): """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" -- cgit From 89cb1e32da33815d5f8e6eb34380ca3401bfad28 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 20 Nov 2010 22:26:15 +0100 Subject: Make sure all templates are included (at least rescue tempaltes didn't used to be included). --- MANIFEST.in | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4fe5f0b34..982b727aa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,9 +13,7 @@ include nova/cloudpipe/client.ovpn.template include nova/compute/fakevirtinstance.xml include nova/compute/interfaces.template include nova/virt/interfaces.template -include nova/virt/libvirt.qemu.xml.template -include nova/virt/libvirt.uml.xml.template -include nova/virt/libvirt.xen.xml.template +include nova/virt/libvirt.*.xml.template include nova/tests/CA/ include nova/tests/CA/cacert.pem include nova/tests/CA/private/ -- cgit From 958591ab2996443ffb6d2f92f928eaad277aa2db Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Mon, 22 Nov 2010 13:11:00 +0000 Subject: pep8 violations fix --- nova/virt/xenapi.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index c3a0c2c7a..3169562a5 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -338,28 +338,27 @@ class XenAPIConnection(object): @utils.deferredToThread def _lookup_vm_vdis(self, vm): return self._lookup_vm_vdis_blocking(vm) - + def _lookup_vm_vdis_blocking(self, vm): # Firstly we get the VBDs, then the VDIs. # TODO: do we leave the read-only devices? vbds = self._conn.xenapi.VM.get_VBDs(vm) vdis = [] if vbds: - for vbd in vbds: - try: - vdi = self._conn.xenapi.VBD.get_VDI(vbd) - # Test valid VDI - record = self._conn.xenapi.VDI.get_record(vdi) - except Exception, exc: - logging.warn(exc) - else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis - else: - return None - - + for vbd in vbds: + try: + vdi = self._conn.xenapi.VBD.get_VDI(vbd) + # Test valid VDI + record = self._conn.xenapi.VDI.get_record(vdi) + except Exception, exc: + logging.warn(exc) + else: + vdis.append(vdi) + if len(vdis) > 0: + return vdis + else: + return None + def _wait_for_task(self, task): """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" -- cgit From 51be7159574d3e0cba8a81b8ea3e9706ce74ac3a Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 14:45:05 -0600 Subject: Set and use AMQP retry interval and max retry constants --- nova/rpc.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index 9938b0838..c2d18cb61 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -38,8 +38,11 @@ from nova import fakerabbit from nova import flags from nova import context + FLAGS = flags.FLAGS +AMQP_RETRY_INT = 10 +AMQP_MAX_RETRIES = 12 LOG = logging.getLogger('amqplib') LOG.setLevel(logging.DEBUG) @@ -85,17 +88,23 @@ class Consumer(messaging.Consumer): def __init__(self, *args, **kwargs): self.failed_connection = False - while True: + for i in range(AMQP_MAX_RETRIES): try: super(Consumer, self).__init__(*args, **kwargs) break except: # Catching all because carrot sucks - logging.exception("AMQP server on %s:%d is unreachable. " \ - "Trying again in 30 seconds." % ( - FLAGS.rabbit_host, - FLAGS.rabbit_port)) - time.sleep(30) - continue + if i + 1 == AMQP_MAX_RETRIES: + logging.exception("Unable to connect to AMQP server" \ + " after %d tries. Shutting down." % AMQP_MAX_RETRIES) + sys.exit(1) + else: + logging.exception("AMQP server on %s:%d is unreachable." \ + " Trying again in %d seconds." % ( + FLAGS.rabbit_host, + FLAGS.rabbit_port, + AMQP_RETRY_INT)) + time.sleep(AMQP_RETRY_INT) + continue def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" @@ -103,7 +112,7 @@ class Consumer(messaging.Consumer): # refactored into some sort of connection manager object try: if self.failed_connection: - # NOTE(vish): conn is defined in the parent class, we can + # NOTE(vish): connection is defined in the parent class, we can # recreate it as long as we create the backend too # pylint: disable-msg=W0201 self.connection = Connection.recreate() -- cgit From 14e4ba7f0e10fc3c2f532b445c1f656f53c8aa95 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 15:22:02 -0600 Subject: Refactor AMQP retry loop --- nova/rpc.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/nova/rpc.py b/nova/rpc.py index c2d18cb61..961b56de6 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -86,25 +86,25 @@ class Consumer(messaging.Consumer): Contains methods for connecting the fetch method to async loops """ def __init__(self, *args, **kwargs): - self.failed_connection = False - - for i in range(AMQP_MAX_RETRIES): + for i in xrange(AMQP_MAX_RETRIES): + if i > 0: + time.sleep(AMQP_RETRY_INT) try: super(Consumer, self).__init__(*args, **kwargs) + self.failed_connection = False break except: # Catching all because carrot sucks - if i + 1 == AMQP_MAX_RETRIES: - logging.exception("Unable to connect to AMQP server" \ - " after %d tries. Shutting down." % AMQP_MAX_RETRIES) - sys.exit(1) - else: - logging.exception("AMQP server on %s:%d is unreachable." \ - " Trying again in %d seconds." % ( - FLAGS.rabbit_host, - FLAGS.rabbit_port, - AMQP_RETRY_INT)) - time.sleep(AMQP_RETRY_INT) - continue + logging.exception("AMQP server on %s:%d is unreachable." \ + " Trying again in %d seconds." % ( + FLAGS.rabbit_host, + FLAGS.rabbit_port, + AMQP_RETRY_INT)) + self.failed_connection = True + continue + if self.failed_connection: + logging.exception("Unable to connect to AMQP server" \ + " after %d tries. Shutting down." % AMQP_MAX_RETRIES) + sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): """Wraps the parent fetch with some logic for failed connections""" -- cgit From a8497abaf24436a92a85129d9771a12f046f2f42 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 16:04:21 -0600 Subject: Removed unnecessary continue --- nova/rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/rpc.py b/nova/rpc.py index 961b56de6..57e522ad3 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -100,7 +100,6 @@ class Consumer(messaging.Consumer): FLAGS.rabbit_port, AMQP_RETRY_INT)) self.failed_connection = True - continue if self.failed_connection: logging.exception("Unable to connect to AMQP server" \ " after %d tries. Shutting down." % AMQP_MAX_RETRIES) -- cgit From deac609ceb1cd6e081445bfc4d8f8c3222b97774 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 16:28:28 -0600 Subject: Make time.sleep() non-blocking --- nova/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index b04b487ea..c7ee9ed14 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -28,7 +28,7 @@ from xml.dom import minidom import eventlet import eventlet.wsgi -eventlet.patcher.monkey_patch(all=False, socket=True) +eventlet.patcher.monkey_patch(all=False, socket=True, time=True) import routes import routes.middleware import webob -- cgit From f0f990495428c028401ba9a4740e6b7a0441213c Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 22 Nov 2010 16:48:44 -0600 Subject: Use FLAGS instead of constants --- nova/flags.py | 2 ++ nova/rpc.py | 11 ++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 4ae86d9b2..a39f22273 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -196,6 +196,8 @@ DEFINE_integer('rabbit_port', 5672, 'rabbit port') DEFINE_string('rabbit_userid', 'guest', 'rabbit userid') DEFINE_string('rabbit_password', 'guest', 'rabbit password') DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') +DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') +DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') DEFINE_string('cc_host', '127.0.0.1', 'ip of api server') DEFINE_integer('cc_port', 8773, 'cloud controller port') diff --git a/nova/rpc.py b/nova/rpc.py index 57e522ad3..86a29574f 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -41,9 +41,6 @@ from nova import context FLAGS = flags.FLAGS -AMQP_RETRY_INT = 10 -AMQP_MAX_RETRIES = 12 - LOG = logging.getLogger('amqplib') LOG.setLevel(logging.DEBUG) @@ -86,9 +83,9 @@ class Consumer(messaging.Consumer): Contains methods for connecting the fetch method to async loops """ def __init__(self, *args, **kwargs): - for i in xrange(AMQP_MAX_RETRIES): + for i in xrange(FLAGS.rabbit_max_retries): if i > 0: - time.sleep(AMQP_RETRY_INT) + time.sleep(FLAGS.rabbit_retry_interval) try: super(Consumer, self).__init__(*args, **kwargs) self.failed_connection = False @@ -98,11 +95,11 @@ class Consumer(messaging.Consumer): " Trying again in %d seconds." % ( FLAGS.rabbit_host, FLAGS.rabbit_port, - AMQP_RETRY_INT)) + FLAGS.rabbit_retry_interval)) self.failed_connection = True if self.failed_connection: logging.exception("Unable to connect to AMQP server" \ - " after %d tries. Shutting down." % AMQP_MAX_RETRIES) + " after %d tries. Shutting down." % FLAGS.rabbit_max_retries) sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): -- cgit From 01c5ce00381e2b08d907385d752bc42b496bf4af Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Tue, 23 Nov 2010 12:11:15 +0000 Subject: added svg files (state.svg is missing because its source is a screen snapshot) --- doc/source/images/rabbit/arch.svg | 292 ++++++++++++++++++ doc/source/images/rabbit/flow1.svg | 617 +++++++++++++++++++++++++++++++++++++ doc/source/images/rabbit/flow2.svg | 423 +++++++++++++++++++++++++ doc/source/images/rabbit/rabt.svg | 581 ++++++++++++++++++++++++++++++++++ 4 files changed, 1913 insertions(+) create mode 100644 doc/source/images/rabbit/arch.svg create mode 100644 doc/source/images/rabbit/flow1.svg create mode 100644 doc/source/images/rabbit/flow2.svg create mode 100644 doc/source/images/rabbit/rabt.svg diff --git a/doc/source/images/rabbit/arch.svg b/doc/source/images/rabbit/arch.svg new file mode 100644 index 000000000..efed6e981 --- /dev/null +++ b/doc/source/images/rabbit/arch.svg @@ -0,0 +1,292 @@ + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Box.8 + Compute + + + + + + + Compute + + Box.2 + Volume Storage + + + + + + + VolumeStorage + + Box + Auth Manager + + + + + + + Auth Manager + + Box.4 + Cloud Controller + + + + + + + CloudController + + Box.3 + API Server + + + + + + + API Server + + Box.6 + Object Store + + + + + + + ObjectStore + + Box.7 + Node Controller + + + + + + + NodeController + + Dynamic connector + + + + Dynamic connector.11 + + + + Dynamic connector.12 + http + + + + + http + + Circle + Nova-Manage + + + + + + + Nova-Manage + + Circle.15 + Euca2ools + + + + + + + Euca2ools + + Dynamic connector.16 + + + + Dynamic connector.17 + + + + Sheet.15 + Project User Role Network VPN + + + + ProjectUserRoleNetworkVPN + + Sheet.16 + VM instance Security group Volume Snapshot VM image IP addres... + + + + VM instanceSecurity groupVolumeSnapshotVM imageIP addressSSH keyAvailability zone + + Box.20 + Network Controller + + + + + + + Network Controller + + Box.5 + Storage Controller + + + + + + + Storage Controller + + Dot & arrow + + + + + + + + + + + + + Dot & arrow.14 + + + + + + + + + + + + + Dynamic connector.13 + + + + Sheet.22 + AMQP + + + + AMQP + + Sheet.23 + AMQP + + + + AMQP + + Sheet.24 + AMQP + + + + AMQP + + Sheet.25 + REST + + + + REST + + Sheet.26 + local method + + + + local method + + Sheet.27 + local method + + + + local method + + Sheet.28 + local method + + + + local method + + diff --git a/doc/source/images/rabbit/flow1.svg b/doc/source/images/rabbit/flow1.svg new file mode 100644 index 000000000..6d8f7e280 --- /dev/null +++ b/doc/source/images/rabbit/flow1.svg @@ -0,0 +1,617 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Rounded rectangle + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ATM switch + name: control_exchange (type: topic) + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.8 + + + + + + name: control_exchange(type: topic) + + + Sheet.9 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.17 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.25 + + + + Sheet.26 + key: topic + + + + key: topic + + Sheet.27 + key: topic.host + + + + key: topic.host + + Sheet.28 + + + + Rectangle + Topic Consumer + + + + + + + Topic Consumer + + Rectangle.30 + Topic Consumer + + + + + + + Topic Consumer + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + + + + Rectangle.34 + + + + + + + Rectangle.35 + Direct Publisher + + + + + + + DirectPublisher + + Sheet.36 + Worker (e.g. compute) + + + + Worker(e.g. compute) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ATM switch.37 + name: msg_id (type: direct) + + Sheet.38 + + + + Sheet.39 + + + + Sheet.40 + + + + Sheet.41 + + + + Sheet.42 + + + + Sheet.43 + + + + + + name: msg_id(type: direct) + + + Sheet.44 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.52 + key: msg_id + + + + key: msg_id + + Sheet.53 + + + + Sheet.54 + + + + Rectangle.57 + + + + + + + Rectangle.56 + Direct Consumer + + + + + + + DirectConsumer + + Sheet.57 + Invoker (e.g. api) + + + + Invoker(e.g. api) + + Rectangle.55 + Topic Publisher + + + + + + + Topic Publisher + + Sheet.59 + + + + Sheet.60 + + + + Sheet.61 + RabbitMQ Node + + + + RabbitMQ Node + + Sheet.62 + + + + Sheet.64 + rpc.call (topic.host) + + + + rpc.call(topic.host) + + Sheet.63 + + + + Sheet.66 + + + + Sheet.67 + + + + Sheet.68 + + + + diff --git a/doc/source/images/rabbit/flow2.svg b/doc/source/images/rabbit/flow2.svg new file mode 100644 index 000000000..fe4cdf341 --- /dev/null +++ b/doc/source/images/rabbit/flow2.svg @@ -0,0 +1,423 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Rounded rectangle + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ATM switch + name: control_exchange (type: topic) + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.8 + + + + + + name: control_exchange(type: topic) + + + Sheet.9 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.17 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.25 + + + + Sheet.26 + key: topic + + + + key: topic + + Sheet.27 + key: topic.host + + + + key: topic.host + + Sheet.28 + + + + Rectangle + Topic Consumer + + + + + + + Topic Consumer + + Rectangle.30 + Topic Consumer + + + + + + + Topic Consumer + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + + + + Rectangle.34 + + + + + + + Sheet.36 + Worker (e.g. compute) + + + + Worker(e.g. compute) + + Rectangle.57 + + + + + + + Sheet.57 + Invoker (e.g. api) + + + + Invoker(e.g. api) + + Rectangle.55 + Topic Publisher + + + + + + + Topic Publisher + + Sheet.59 + + + + Sheet.61 + RabbitMQ Node + + + + RabbitMQ Node + + Sheet.62 + + + + Sheet.63 + rpc.cast(topic) + + + + rpc.cast(topic) + + Sheet.64 + + + + Sheet.65 + + + + diff --git a/doc/source/images/rabbit/rabt.svg b/doc/source/images/rabbit/rabt.svg new file mode 100644 index 000000000..142a33ce0 --- /dev/null +++ b/doc/source/images/rabbit/rabt.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + Rounded rectangle + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ATM switch + name: control_exchange (type: topic) + + Sheet.3 + + + + Sheet.4 + + + + Sheet.5 + + + + Sheet.6 + + + + Sheet.7 + + + + Sheet.8 + + + + + + name: control_exchange(type: topic) + + + Sheet.17 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.9 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.25 + + + + Sheet.27 + key: topic + + + + key: topic + + Sheet.28 + key: topic.host + + + + key: topic.host + + Sheet.26 + + + + Rectangle + Topic Consumer + + + + + + + Topic Consumer + + Rectangle.30 + Topic Consumer + + + + + + + Topic Consumer + + Sheet.31 + + + + Sheet.32 + + + + Sheet.33 + + + + Rectangle.34 + + + + + + + Rectangle.35 + Direct Publisher + + + + + + + DirectPublisher + + Sheet.36 + Worker (e.g. compute) + + + + Worker(e.g. compute) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ATM switch.37 + name: msg_id (type: direct) + + Sheet.38 + + + + Sheet.39 + + + + Sheet.40 + + + + Sheet.41 + + + + Sheet.42 + + + + Sheet.43 + + + + + + name: msg_id(type: direct) + + + Sheet.44 + + Rectangle + + + + + + + Rectangle.10 + + + + + + + Rectangle.11 + + + + + + + Rectangle.12 + + + + + + + Rectangle.13 + + + + + + + Rectangle.14 + + + + + + + Rectangle.15 + + + + + + + + Sheet.52 + key: msg_id + + + + key: msg_id + + Sheet.53 + + + + Sheet.54 + + + + Rectangle.57 + + + + + + + Rectangle.58 + Direct Consumer + + + + + + + DirectConsumer + + Sheet.59 + Invoker (e.g. api) + + + + Invoker(e.g. api) + + Rectangle.55 + Topic Publisher + + + + + + + Topic Publisher + + Sheet.56 + + + + Sheet.60 + + + + Sheet.62 + RabbitMQ Node (single virtual host context) + + + + RabbitMQ Node(single virtual host context) + + -- cgit