diff options
| -rw-r--r-- | MAINTAINERS | 6 | ||||
| -rw-r--r-- | TESTING.rst | 88 | ||||
| -rw-r--r-- | openstack/common/db/sqlalchemy/session.py | 17 | ||||
| -rw-r--r-- | openstack/common/network_utils.py | 5 | ||||
| -rw-r--r-- | openstack/common/plugin/__init__.py | 14 | ||||
| -rw-r--r-- | openstack/common/plugin/callbackplugin.py | 93 | ||||
| -rw-r--r-- | openstack/common/plugin/plugin.py | 86 | ||||
| -rw-r--r-- | openstack/common/plugin/pluginmanager.py | 78 | ||||
| -rw-r--r-- | openstack/common/rpc/amqp.py | 6 | ||||
| -rw-r--r-- | openstack/common/rpc/impl_kombu.py | 51 | ||||
| -rw-r--r-- | openstack/common/rpc/impl_qpid.py | 3 | ||||
| -rw-r--r-- | openstack/common/strutils.py | 18 | ||||
| -rw-r--r-- | requirements.txt | 6 | ||||
| -rw-r--r-- | tests/unit/db/sqlalchemy/test_sqlalchemy.py | 61 | ||||
| -rw-r--r-- | tests/unit/db/test_api.py | 12 | ||||
| -rw-r--r-- | tests/unit/plugin/__init__.py | 14 | ||||
| -rw-r--r-- | tests/unit/plugin/test_callback_plugin.py | 92 | ||||
| -rw-r--r-- | tests/unit/rpc/test_kombu.py | 122 | ||||
| -rw-r--r-- | tests/unit/rpc/test_zmq.py | 1 | ||||
| -rw-r--r-- | tests/unit/test_plugin.py | 118 | ||||
| -rw-r--r-- | tests/utils.py | 11 |
21 files changed, 337 insertions, 565 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index 128cb22..0500c53 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -199,12 +199,6 @@ M: Michael Still <mikal@stillhq.com> S: Maintained F: periodic_task.py -== plugins == - -M: -S: Orphan -F: plugin/ - == policy == M: diff --git a/TESTING.rst b/TESTING.rst new file mode 100644 index 0000000..4191b1b --- /dev/null +++ b/TESTING.rst @@ -0,0 +1,88 @@ +=========================== +Testing Your OpenStack Code +=========================== +------------ +A Quickstart +------------ + +This is designed to be enough information for you to run your first tests. +Detailed information on testing can be found here: https://wiki.openstack.org/wiki/Testing + +*Install pip*:: + + [apt-get | yum] install python-pip +More information on pip here: http://www.pip-installer.org/en/latest/ + +*Use pip to install tox*:: + + pip install tox + +Run The Tests +------------- + +*Navigate to the project's root directory and execute*:: + + tox +Note: completing this command may take a long time (depends on system resources) +also, you might not see any output until tox is complete. + +Information about tox can be found here: http://testrun.org/tox/latest/ + + +Run The Tests in One Environment +-------------------------------- + +Tox will run your entire test suite in the environments specified in the project tox.ini:: + + [tox] + + envlist = <list of available environments> + +To run the test suite in just one of the environments in envlist execute:: + + tox -e <env> +so for example, *run the test suite in py26*:: + + tox -e py26 + +Run One Test +------------ + +To run individual tests with tox: + +if testr is in tox.ini, for example:: + + [testenv] + + includes "python setup.py testr --slowest --testr-args='{posargs}'" + +run individual tests with the following syntax:: + + tox -e <env> -- path.to.module:Class.test +so for example, *run the cpu_limited test in Nova*:: + + tox -e py27 -- nova.tests.test_claims:ClaimTestCase.test_cpu_unlimited + +if nose is in tox.ini, for example:: + + [testenv] + + includes "nosetests {posargs}" + +run individual tests with the following syntax:: + + tox -e <env> -- --tests path.to.module:Class.test +so for example, *run the list test in Glance*:: + + tox -e py27 -- --tests glance.tests.unit.test_auth.py:TestImageRepoProxy.test_list + +Need More Info? +--------------- + +More information about testr: https://wiki.openstack.org/wiki/Testr + +More information about nose: https://nose.readthedocs.org/en/latest/ + + +More information about testing OpenStack code can be found here: +https://wiki.openstack.org/wiki/Testing diff --git a/openstack/common/db/sqlalchemy/session.py b/openstack/common/db/sqlalchemy/session.py index b5e10f1..0951ee6 100644 --- a/openstack/common/db/sqlalchemy/session.py +++ b/openstack/common/db/sqlalchemy/session.py @@ -280,6 +280,8 @@ database_opts = [ 'database', deprecated_name='sql_connection', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('sql_connection', + group='DATABASE')], secret=True), cfg.StrOpt('slave_connection', default='', @@ -290,34 +292,46 @@ database_opts = [ default=3600, deprecated_name='sql_idle_timeout', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('sql_idle_timeout', + group='DATABASE')], help='timeout before idle sql connections are reaped'), cfg.IntOpt('min_pool_size', default=1, deprecated_name='sql_min_pool_size', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('sql_min_pool_size', + group='DATABASE')], help='Minimum number of SQL connections to keep open in a ' 'pool'), cfg.IntOpt('max_pool_size', default=None, deprecated_name='sql_max_pool_size', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('sql_max_pool_size', + group='DATABASE')], help='Maximum number of SQL connections to keep open in a ' 'pool'), cfg.IntOpt('max_retries', default=10, deprecated_name='sql_max_retries', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('sql_max_retries', + group='DATABASE')], help='maximum db connection retries during startup. ' '(setting -1 implies an infinite retry count)'), cfg.IntOpt('retry_interval', default=10, deprecated_name='sql_retry_interval', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('reconnect_interval', + group='DATABASE')], help='interval between retries of opening a sql connection'), cfg.IntOpt('max_overflow', default=None, deprecated_name='sql_max_overflow', deprecated_group=DEFAULT, + deprecated_opts=[cfg.DeprecatedOpt('sqlalchemy_max_overflow', + group='DATABASE')], help='If set, use this value for max_overflow with sqlalchemy'), cfg.IntOpt('connection_debug', default=0, @@ -332,12 +346,15 @@ database_opts = [ help='Add python stack traces to SQL as comment strings'), cfg.IntOpt('pool_timeout', default=None, + deprecated_name='sqlalchemy_pool_timeout', + deprecated_group='DATABASE', help='If set, use this value for pool_timeout with sqlalchemy'), ] CONF = cfg.CONF CONF.register_opts(sqlite_db_opts) CONF.register_opts(database_opts, 'database') + LOG = logging.getLogger(__name__) _ENGINE = None diff --git a/openstack/common/network_utils.py b/openstack/common/network_utils.py index 0527ab9..0fbf171 100644 --- a/openstack/common/network_utils.py +++ b/openstack/common/network_utils.py @@ -19,11 +19,6 @@ Network-related utilities and helper functions. """ -from openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - def parse_host_port(address, default_port=None): """Interpret a string as a host:port pair. diff --git a/openstack/common/plugin/__init__.py b/openstack/common/plugin/__init__.py deleted file mode 100644 index b706747..0000000 --- a/openstack/common/plugin/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/openstack/common/plugin/callbackplugin.py b/openstack/common/plugin/callbackplugin.py deleted file mode 100644 index 2de7fb0..0000000 --- a/openstack/common/plugin/callbackplugin.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from openstack.common import log as logging -from openstack.common.plugin import plugin - - -LOG = logging.getLogger(__name__) - - -class _CallbackNotifier(object): - """Manages plugin-defined notification callbacks. - - For each Plugin, a CallbackNotifier will be added to the - notification driver list. Calls to notify() with appropriate - messages will be hooked and prompt callbacks. - - A callback should look like this: - def callback(context, message, user_data) - """ - - def __init__(self): - self._callback_dict = {} - - def _add_callback(self, event_type, callback, user_data): - callback_list = self._callback_dict.get(event_type, []) - callback_list.append({'function': callback, - 'user_data': user_data}) - self._callback_dict[event_type] = callback_list - - def _remove_callback(self, callback): - for callback_list in self._callback_dict.values(): - for entry in callback_list: - if entry['function'] == callback: - callback_list.remove(entry) - - def notify(self, context, message): - if message.get('event_type') not in self._callback_dict: - return - - for entry in self._callback_dict[message.get('event_type')]: - entry['function'](context, message, entry.get('user_data')) - - def callbacks(self): - return self._callback_dict - - -class CallbackPlugin(plugin.Plugin): - """Plugin with a simple callback interface. - - This class is provided as a convenience for producing a simple - plugin that only watches a couple of events. For example, here's - a subclass which prints a line the first time an instance is created. - - class HookInstanceCreation(CallbackPlugin): - - def __init__(self, _service_name): - super(HookInstanceCreation, self).__init__() - self._add_callback(self.magic, 'compute.instance.create.start') - - def magic(self): - print "An instance was created!" - self._remove_callback(self, self.magic) - """ - - def __init__(self, service_name): - super(CallbackPlugin, self).__init__(service_name) - self._callback_notifier = _CallbackNotifier() - self._add_notifier(self._callback_notifier) - - def _add_callback(self, callback, event_type, user_data=None): - """Add callback for a given event notification. - - Subclasses can call this as an alternative to implementing - a fullblown notify notifier. - """ - self._callback_notifier._add_callback(event_type, callback, user_data) - - def _remove_callback(self, callback): - """Remove all notification callbacks to specified function.""" - self._callback_notifier._remove_callback(callback) diff --git a/openstack/common/plugin/plugin.py b/openstack/common/plugin/plugin.py deleted file mode 100644 index d2be0b3..0000000 --- a/openstack/common/plugin/plugin.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - - -class Plugin(object): - """Defines an interface for adding functionality to an OpenStack service. - - A plugin interacts with a service via the following pathways: - - - An optional set of notifiers, managed by calling add_notifier() - or by overriding _notifiers() - - - A set of api extensions, managed via add_api_extension_descriptor() - - - Direct calls to service functions. - - - Whatever else the plugin wants to do on its own. - - This is the reference implementation. - """ - - # The following functions are provided as convenience methods - # for subclasses. Subclasses should call them but probably not - # override them. - def _add_api_extension_descriptor(self, descriptor): - """Subclass convenience method which adds an extension descriptor. - - Subclass constructors should call this method when - extending a project's REST interface. - - Note that once the api service has loaded, the - API extension set is more-or-less fixed, so - this should mainly be called by subclass constructors. - """ - self._api_extension_descriptors.append(descriptor) - - def _add_notifier(self, notifier): - """Subclass convenience method which adds a notifier. - - Notifier objects should implement the function notify(message). - Each notifier receives a notify() call whenever an openstack - service broadcasts a notification. - - Best to call this during construction. Notifiers are enumerated - and registered by the pluginmanager at plugin load time. - """ - self._notifiers.append(notifier) - - # The following methods are called by OpenStack services to query - # plugin features. Subclasses should probably not override these. - def _notifiers(self): - """Returns list of notifiers for this plugin.""" - return self._notifiers - - notifiers = property(_notifiers) - - def _api_extension_descriptors(self): - """Return a list of API extension descriptors. - - Called by a project API during its load sequence. - """ - return self._api_extension_descriptors - - api_extension_descriptors = property(_api_extension_descriptors) - - # Most plugins will override this: - def __init__(self, service_name): - self._notifiers = [] - self._api_extension_descriptors = [] diff --git a/openstack/common/plugin/pluginmanager.py b/openstack/common/plugin/pluginmanager.py deleted file mode 100644 index 3962447..0000000 --- a/openstack/common/plugin/pluginmanager.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pkg_resources - -from oslo.config import cfg - -from openstack.common.gettextutils import _ -from openstack.common import log as logging -from openstack.common.notifier import api as notifier_api - - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -class PluginManager(object): - """Manages plugin entrypoints and loading. - - For a service to implement this plugin interface for callback purposes: - - - Make use of the openstack-common notifier system - - Instantiate this manager in each process (passing in - project and service name) - - For an API service to extend itself using this plugin interface, - it needs to query the plugin_extension_factory provided by - the already-instantiated PluginManager. - """ - - def __init__(self, project_name, service_name): - """Construct Plugin Manager; load and initialize plugins. - - project_name (e.g. 'nova' or 'glance') is used - to construct the entry point that identifies plugins. - - The service_name (e.g. 'compute') is passed on to - each plugin as a raw string for it to do what it will. - """ - self._project_name = project_name - self._service_name = service_name - self.plugins = [] - - def load_plugins(self): - self.plugins = [] - - for entrypoint in pkg_resources.iter_entry_points('%s.plugin' % - self._project_name): - try: - pluginclass = entrypoint.load() - plugin = pluginclass(self._service_name) - self.plugins.append(plugin) - except Exception as exc: - LOG.error(_("Failed to load plugin %(plug)s: %(exc)s") % - {'plug': entrypoint, 'exc': exc}) - - # Register individual notifiers. - for plugin in self.plugins: - for notifier in plugin.notifiers: - notifier_api.add_driver(notifier) - - def plugin_extension_factory(self, ext_mgr): - for plugin in self.plugins: - descriptors = plugin.api_extension_descriptors - for descriptor in descriptors: - ext_mgr.load_extension(descriptor) diff --git a/openstack/common/rpc/amqp.py b/openstack/common/rpc/amqp.py index 22e01d7..1ab7ae4 100644 --- a/openstack/common/rpc/amqp.py +++ b/openstack/common/rpc/amqp.py @@ -151,11 +151,13 @@ class ConnectionContext(rpc_common.Connection): def create_worker(self, topic, proxy, pool_name): self.connection.create_worker(topic, proxy, pool_name) - def join_consumer_pool(self, callback, pool_name, topic, exchange_name): + def join_consumer_pool(self, callback, pool_name, topic, exchange_name, + ack_on_error=True): self.connection.join_consumer_pool(callback, pool_name, topic, - exchange_name) + exchange_name, + ack_on_error) def consume_in_thread(self): self.connection.consume_in_thread() diff --git a/openstack/common/rpc/impl_kombu.py b/openstack/common/rpc/impl_kombu.py index c062d9a..716b120 100644 --- a/openstack/common/rpc/impl_kombu.py +++ b/openstack/common/rpc/impl_kombu.py @@ -129,6 +129,7 @@ class ConsumerBase(object): self.tag = str(tag) self.kwargs = kwargs self.queue = None + self.ack_on_error = kwargs.get('ack_on_error', True) self.reconnect(channel) def reconnect(self, channel): @@ -138,6 +139,36 @@ class ConsumerBase(object): self.queue = kombu.entity.Queue(**self.kwargs) self.queue.declare() + def _callback_handler(self, message, callback): + """Call callback with deserialized message. + + Messages that are processed without exception are ack'ed. + + If the message processing generates an exception, it will be + ack'ed if ack_on_error=True. Otherwise it will be .reject()'ed. + Rejection is better than waiting for the message to timeout. + Rejected messages are immediately requeued. + """ + + ack_msg = False + try: + msg = rpc_common.deserialize_msg(message.payload) + callback(msg) + ack_msg = True + except Exception: + if self.ack_on_error: + ack_msg = True + LOG.exception(_("Failed to process message" + " ... skipping it.")) + else: + LOG.exception(_("Failed to process message" + " ... will requeue.")) + finally: + if ack_msg: + message.ack() + else: + message.reject() + def consume(self, *args, **kwargs): """Actually declare the consumer on the amqp channel. This will start the flow of messages from the queue. Using the @@ -150,8 +181,6 @@ class ConsumerBase(object): If kwargs['nowait'] is True, then this call will block until a message is read. - Messages will automatically be acked if the callback doesn't - raise an exception """ options = {'consumer_tag': self.tag} @@ -162,13 +191,7 @@ class ConsumerBase(object): def _callback(raw_message): message = self.channel.message_to_python(raw_message) - try: - msg = rpc_common.deserialize_msg(message.payload) - callback(msg) - except Exception: - LOG.exception(_("Failed to process message... skipping it.")) - finally: - message.ack() + self._callback_handler(message, callback) self.queue.consume(*args, callback=_callback, **options) @@ -635,8 +658,8 @@ class Connection(object): def _consume(): if info['do_consume']: - queues_head = self.consumers[:-1] - queues_tail = self.consumers[-1] + queues_head = self.consumers[:-1] # not fanout. + queues_tail = self.consumers[-1] # fanout for queue in queues_head: queue.consume(nowait=True) queues_tail.consume(nowait=False) @@ -685,11 +708,12 @@ class Connection(object): self.declare_consumer(DirectConsumer, topic, callback) def declare_topic_consumer(self, topic, callback=None, queue_name=None, - exchange_name=None): + exchange_name=None, ack_on_error=True): """Create a 'topic' consumer.""" self.declare_consumer(functools.partial(TopicConsumer, name=queue_name, exchange_name=exchange_name, + ack_on_error=ack_on_error, ), topic, callback) @@ -754,7 +778,7 @@ class Connection(object): self.declare_topic_consumer(topic, proxy_cb, pool_name) def join_consumer_pool(self, callback, pool_name, topic, - exchange_name=None): + exchange_name=None, ack_on_error=True): """Register as a member of a group of consumers for a given topic from the specified exchange. @@ -775,6 +799,7 @@ class Connection(object): topic=topic, exchange_name=exchange_name, callback=callback_wrapper, + ack_on_error=ack_on_error, ) diff --git a/openstack/common/rpc/impl_qpid.py b/openstack/common/rpc/impl_qpid.py index 7352517..46dcb6a 100644 --- a/openstack/common/rpc/impl_qpid.py +++ b/openstack/common/rpc/impl_qpid.py @@ -152,6 +152,7 @@ class ConsumerBase(object): except Exception: LOG.exception(_("Failed to process message... skipping it.")) finally: + # TODO(sandy): Need support for optional ack_on_error. self.session.acknowledge(message) def get_receiver(self): @@ -615,7 +616,7 @@ class Connection(object): return consumer def join_consumer_pool(self, callback, pool_name, topic, - exchange_name=None): + exchange_name=None, ack_on_error=True): """Register as a member of a group of consumers for a given topic from the specified exchange. diff --git a/openstack/common/strutils.py b/openstack/common/strutils.py index a3fb53a..0ba9b44 100644 --- a/openstack/common/strutils.py +++ b/openstack/common/strutils.py @@ -23,6 +23,8 @@ import re import sys import unicodedata +import six + from openstack.common.gettextutils import _ @@ -71,7 +73,7 @@ def bool_from_string(subject, strict=False): ValueError which is useful when parsing values passed in from an API call. Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. """ - if not isinstance(subject, basestring): + if not isinstance(subject, six.string_types): subject = str(subject) lowered = subject.strip().lower() @@ -99,12 +101,12 @@ def safe_decode(text, incoming=None, errors='strict'): values http://docs.python.org/2/library/codecs.html :returns: text or a unicode `incoming` encoded representation of it. - :raises TypeError: If text is not an isntance of basestring + :raises TypeError: If text is not an isntance of str """ - if not isinstance(text, basestring): + if not isinstance(text, six.string_types): raise TypeError("%s can't be decoded" % type(text)) - if isinstance(text, unicode): + if isinstance(text, six.text_type): return text if not incoming: @@ -142,16 +144,16 @@ def safe_encode(text, incoming=None, values http://docs.python.org/2/library/codecs.html :returns: text or a bytestring `encoding` encoded representation of it. - :raises TypeError: If text is not an isntance of basestring + :raises TypeError: If text is not an isntance of str """ - if not isinstance(text, basestring): + if not isinstance(text, six.string_types): raise TypeError("%s can't be encoded" % type(text)) if not incoming: incoming = (sys.stdin.encoding or sys.getdefaultencoding()) - if isinstance(text, unicode): + if isinstance(text, six.text_type): return text.encode(encoding, errors) elif text and encoding != incoming: # Decode text before encoding it with `encoding` @@ -204,7 +206,7 @@ def to_slug(value, incoming=None, errors="strict"): :param errors: Errors handling policy. See here for valid values http://docs.python.org/2/library/codecs.html :returns: slugified unicode representation of `value` - :raises TypeError: If text is not an instance of basestring + :raises TypeError: If text is not an instance of str """ value = safe_decode(value, incoming, errors) # NOTE(aababilov): no need to use safe_(encode|decode) here: diff --git a/requirements.txt b/requirements.txt index 067af58..ec6dbdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,11 +7,11 @@ greenlet>=0.3.2 lxml routes==1.12.3 iso8601>=0.1.4 -anyjson==0.2.4 -kombu==1.0.4 +anyjson>=0.3.3 +kombu>2.4.7 argparse stevedore SQLAlchemy>=0.7.8,<=0.7.9 -oslo.config>=1.1.0 +http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a2.tar.gz#egg=oslo.config-1.2.0a2 qpid-python six diff --git a/tests/unit/db/sqlalchemy/test_sqlalchemy.py b/tests/unit/db/sqlalchemy/test_sqlalchemy.py index ac178b8..48d6cf7 100644 --- a/tests/unit/db/sqlalchemy/test_sqlalchemy.py +++ b/tests/unit/db/sqlalchemy/test_sqlalchemy.py @@ -52,15 +52,15 @@ sql_max_overflow=50 sql_connection_debug=60 sql_connection_trace=True """)]) - test_utils.CONF(['--config-file', paths[0]]) - self.assertEquals(test_utils.CONF.database.connection, 'x://y.z') - self.assertEquals(test_utils.CONF.database.min_pool_size, 10) - self.assertEquals(test_utils.CONF.database.max_pool_size, 20) - self.assertEquals(test_utils.CONF.database.max_retries, 30) - self.assertEquals(test_utils.CONF.database.retry_interval, 40) - self.assertEquals(test_utils.CONF.database.max_overflow, 50) - self.assertEquals(test_utils.CONF.database.connection_debug, 60) - self.assertEquals(test_utils.CONF.database.connection_trace, True) + self.conf(['--config-file', paths[0]]) + self.assertEquals(self.conf.database.connection, 'x://y.z') + self.assertEquals(self.conf.database.min_pool_size, 10) + self.assertEquals(self.conf.database.max_pool_size, 20) + self.assertEquals(self.conf.database.max_retries, 30) + self.assertEquals(self.conf.database.retry_interval, 40) + self.assertEquals(self.conf.database.max_overflow, 50) + self.assertEquals(self.conf.database.connection_debug, 60) + self.assertEquals(self.conf.database.connection_trace, True) def test_session_parameters(self): paths = self.create_tempfiles([('test', """[database] @@ -74,16 +74,39 @@ connection_debug=60 connection_trace=True pool_timeout=7 """)]) - test_utils.CONF(['--config-file', paths[0]]) - self.assertEquals(test_utils.CONF.database.connection, 'x://y.z') - self.assertEquals(test_utils.CONF.database.min_pool_size, 10) - self.assertEquals(test_utils.CONF.database.max_pool_size, 20) - self.assertEquals(test_utils.CONF.database.max_retries, 30) - self.assertEquals(test_utils.CONF.database.retry_interval, 40) - self.assertEquals(test_utils.CONF.database.max_overflow, 50) - self.assertEquals(test_utils.CONF.database.connection_debug, 60) - self.assertEquals(test_utils.CONF.database.connection_trace, True) - self.assertEquals(test_utils.CONF.database.pool_timeout, 7) + self.conf(['--config-file', paths[0]]) + self.assertEquals(self.conf.database.connection, 'x://y.z') + self.assertEquals(self.conf.database.min_pool_size, 10) + self.assertEquals(self.conf.database.max_pool_size, 20) + self.assertEquals(self.conf.database.max_retries, 30) + self.assertEquals(self.conf.database.retry_interval, 40) + self.assertEquals(self.conf.database.max_overflow, 50) + self.assertEquals(self.conf.database.connection_debug, 60) + self.assertEquals(self.conf.database.connection_trace, True) + self.assertEquals(self.conf.database.pool_timeout, 7) + + def test_dbapi_database_deprecated_parameters(self): + paths = self.create_tempfiles([('test', + '[DATABASE]\n' + 'sql_connection=fake_connection\n' + 'sql_idle_timeout=100\n' + 'sql_min_pool_size=99\n' + 'sql_max_pool_size=199\n' + 'sql_max_retries=22\n' + 'reconnect_interval=17\n' + 'sqlalchemy_max_overflow=101\n' + 'sqlalchemy_pool_timeout=5\n' + )]) + self.conf(['--config-file', paths[0]]) + self.assertEquals(self.conf.database.connection, + 'fake_connection') + self.assertEquals(self.conf.database.idle_timeout, 100) + self.assertEquals(self.conf.database.min_pool_size, 99) + self.assertEquals(self.conf.database.max_pool_size, 199) + self.assertEquals(self.conf.database.max_retries, 22) + self.assertEquals(self.conf.database.retry_interval, 17) + self.assertEquals(self.conf.database.max_overflow, 101) + self.assertEquals(self.conf.database.pool_timeout, 5) class SessionErrorWrapperTestCase(test_base.DbTestCase): diff --git a/tests/unit/db/test_api.py b/tests/unit/db/test_api.py index f6e0d4c..2a8db3b 100644 --- a/tests/unit/db/test_api.py +++ b/tests/unit/db/test_api.py @@ -40,9 +40,9 @@ class DBAPITestCase(test_utils.BaseTestCase): 'dbapi_use_tpool=True\n' )]) - test_utils.CONF(['--config-file', paths[0]]) - self.assertEquals(test_utils.CONF.database.backend, 'test_123') - self.assertEquals(test_utils.CONF.database.use_tpool, True) + self.conf(['--config-file', paths[0]]) + self.assertEquals(self.conf.database.backend, 'test_123') + self.assertEquals(self.conf.database.use_tpool, True) def test_dbapi_parameters(self): paths = self.create_tempfiles([('test', @@ -51,9 +51,9 @@ class DBAPITestCase(test_utils.BaseTestCase): 'use_tpool=True\n' )]) - test_utils.CONF(['--config-file', paths[0]]) - self.assertEquals(test_utils.CONF.database.backend, 'test_123') - self.assertEquals(test_utils.CONF.database.use_tpool, True) + self.conf(['--config-file', paths[0]]) + self.assertEquals(self.conf.database.backend, 'test_123') + self.assertEquals(self.conf.database.use_tpool, True) def test_dbapi_api_class_method_and_tpool_false(self): backend_mapping = {'test_known': 'tests.unit.db.test_api'} diff --git a/tests/unit/plugin/__init__.py b/tests/unit/plugin/__init__.py deleted file mode 100644 index b706747..0000000 --- a/tests/unit/plugin/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/tests/unit/plugin/test_callback_plugin.py b/tests/unit/plugin/test_callback_plugin.py deleted file mode 100644 index 3f3fd63..0000000 --- a/tests/unit/plugin/test_callback_plugin.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pkg_resources - -from openstack.common.notifier import api as notifier_api -from openstack.common.plugin import callbackplugin -from openstack.common.plugin import pluginmanager -from tests import utils as test_utils - -userdatastring = "magic user data string" - - -class TestCBP(callbackplugin.CallbackPlugin): - - def callback1(self, context, message, userdata): - self.callback1count += 1 - - def callback2(self, context, message, userdata): - self.callback2count += 1 - - def callback3(self, context, message, userdata): - self.callback3count += 1 - self.userdata = userdata - - def __init__(self, service_name): - super(TestCBP, self).__init__(service_name) - self.callback1count = 0 - self.callback2count = 0 - self.callback3count = 0 - - self._add_callback(self.callback1, 'type1', None) - self._add_callback(self.callback2, 'type1', None) - self._add_callback(self.callback3, 'type2', 'magic user data string') - - -class CallbackTestCase(test_utils.BaseTestCase): - """Tests for the callback plugin convenience class.""" - - def test_callback_plugin_subclass(self): - - class MockEntrypoint(pkg_resources.EntryPoint): - def load(self): - return TestCBP - - def mock_iter_entry_points(_t): - return [MockEntrypoint("fake", "fake", ["fake"])] - - self.stubs.Set(pkg_resources, 'iter_entry_points', - mock_iter_entry_points) - - plugmgr = pluginmanager.PluginManager("testproject", "testservice") - plugmgr.load_plugins() - self.assertEqual(len(plugmgr.plugins), 1) - plugin = plugmgr.plugins[0] - self.assertEqual(len(plugin.notifiers), 1) - - notifier_api.notify('contextarg', 'publisher_id', 'type1', - notifier_api.WARN, dict(a=3)) - - self.assertEqual(plugin.callback1count, 1) - self.assertEqual(plugin.callback2count, 1) - self.assertEqual(plugin.callback3count, 0) - - notifier_api.notify('contextarg', 'publisher_id', 'type2', - notifier_api.WARN, dict(a=3)) - - self.assertEqual(plugin.callback1count, 1) - self.assertEqual(plugin.callback2count, 1) - self.assertEqual(plugin.callback3count, 1) - self.assertEqual(plugin.userdata, userdatastring) - - plugin._remove_callback(plugin.callback1) - - notifier_api.notify('contextarg', 'publisher_id', 'type1', - notifier_api.WARN, dict(a=3)) - - self.assertEqual(plugin.callback1count, 1) - self.assertEqual(plugin.callback2count, 2) - self.assertEqual(plugin.callback3count, 1) diff --git a/tests/unit/rpc/test_kombu.py b/tests/unit/rpc/test_kombu.py index 159fefb..3debe52 100644 --- a/tests/unit/rpc/test_kombu.py +++ b/tests/unit/rpc/test_kombu.py @@ -23,6 +23,7 @@ import eventlet eventlet.monkey_patch() import contextlib +import functools import logging import mock @@ -69,6 +70,7 @@ class KombuStubs: @staticmethod def setUp(self): if kombu: + self.conf = FLAGS self.config(fake_rabbit=True) self.config(rpc_response_timeout=5) self.rpc = impl_kombu @@ -77,6 +79,20 @@ class KombuStubs: self.rpc = None +class FakeMessage(object): + acked = False + rejected = False + + def __init__(self, payload): + self.payload = payload + + def ack(self): + self.acked = True + + def reject(self): + self.rejected = True + + class RpcKombuTestCase(amqp.BaseRpcAMQPTestCase): def setUp(self): KombuStubs.setUp(self) @@ -112,6 +128,74 @@ class RpcKombuTestCase(amqp.BaseRpcAMQPTestCase): self.assertEqual(self.received_message, message) + def test_callback_handler_ack_on_error(self): + """The default case will ack on error. Same as before. + """ + def _callback(msg): + pass + + conn = self.rpc.create_connection(FLAGS) + consumer = conn.declare_consumer(functools.partial( + impl_kombu.TopicConsumer, + name=None, + exchange_name=None), + "a_topic", _callback) + message = FakeMessage("some message") + consumer._callback_handler(message, _callback) + self.assertTrue(message.acked) + self.assertFalse(message.rejected) + + def test_callback_handler_ack_on_error_exception(self): + + def _callback(msg): + raise MyException() + + conn = self.rpc.create_connection(FLAGS) + consumer = conn.declare_consumer(functools.partial( + impl_kombu.TopicConsumer, + name=None, + exchange_name=None, + ack_on_error=True), + "a_topic", _callback) + message = FakeMessage("some message") + consumer._callback_handler(message, _callback) + self.assertTrue(message.acked) + self.assertFalse(message.rejected) + + def test_callback_handler_no_ack_on_error_exception(self): + + def _callback(msg): + raise MyException() + + conn = self.rpc.create_connection(FLAGS) + consumer = conn.declare_consumer(functools.partial( + impl_kombu.TopicConsumer, + name=None, + exchange_name=None, + ack_on_error=False), + "a_topic", _callback) + message = FakeMessage("some message") + consumer._callback_handler(message, _callback) + self.assertFalse(message.acked) + self.assertTrue(message.rejected) + + def test_callback_handler_no_ack_on_error(self): + + def _callback(msg): + pass + + conn = self.rpc.create_connection(FLAGS) + consumer = conn.declare_consumer(functools.partial( + impl_kombu.TopicConsumer, + name=None, + exchange_name=None, + ack_on_error=False), + "a_topic", _callback) + message = FakeMessage("some message") + consumer._callback_handler(message, _callback) + self.assertTrue(message.acked) + self.assertFalse(message.rejected) + def test_message_ttl_on_timeout(self): """Test message ttl being set by request timeout. The message should die on the vine and never arrive. @@ -308,6 +392,22 @@ class RpcKombuTestCase(amqp.BaseRpcAMQPTestCase): impl_kombu.cast_to_server(FLAGS, ctxt, server_params, 'fake_topic', {'msg': 'fake'}) + def test_fanout_success(self): + # Overriding the method in the base class because the test + # seems to fail for the same reason as + # test_fanout_send_receive(). + self.skipTest("kombu memory transport seems buggy with " + "fanout queues as this test passes when " + "you use rabbit (fake_rabbit=False)") + + def test_cast_success_despite_missing_args(self): + # Overriding the method in the base class because the test + # seems to fail for the same reason as + # test_fanout_send_receive(). + self.skipTest("kombu memory transport seems buggy with " + "fanout queues as this test passes when " + "you use rabbit (fake_rabbit=False)") + def test_fanout_send_receive(self): """Test sending to a fanout exchange and consuming from 2 queues.""" @@ -514,7 +614,25 @@ class RpcKombuTestCase(amqp.BaseRpcAMQPTestCase): 'pool.name', ) - def test_join_consumer_pool(self): + def test_join_consumer_pool_default(self): + meth = 'declare_topic_consumer' + with mock.patch.object(self.rpc.Connection, meth) as p: + conn = self.rpc.create_connection(FLAGS) + conn.join_consumer_pool( + callback=lambda *a, **k: (a, k), + pool_name='pool.name', + topic='topic.name', + exchange_name='exchange.name', + ) + p.assert_called_with( + callback=mock.ANY, # the callback wrapper + queue_name='pool.name', + exchange_name='exchange.name', + topic='topic.name', + ack_on_error=True, + ) + + def test_join_consumer_pool_no_ack(self): meth = 'declare_topic_consumer' with mock.patch.object(self.rpc.Connection, meth) as p: conn = self.rpc.create_connection(FLAGS) @@ -523,12 +641,14 @@ class RpcKombuTestCase(amqp.BaseRpcAMQPTestCase): pool_name='pool.name', topic='topic.name', exchange_name='exchange.name', + ack_on_error=False, ) p.assert_called_with( callback=mock.ANY, # the callback wrapper queue_name='pool.name', exchange_name='exchange.name', topic='topic.name', + ack_on_error=False, ) diff --git a/tests/unit/rpc/test_zmq.py b/tests/unit/rpc/test_zmq.py index b0f0262..c87a040 100644 --- a/tests/unit/rpc/test_zmq.py +++ b/tests/unit/rpc/test_zmq.py @@ -60,6 +60,7 @@ class _RpcZmqBaseTestCase(common.BaseRpcTestCase): self.reactor = None self.rpc = impl_zmq + self.conf = FLAGS self.config(rpc_zmq_bind_address='127.0.0.1') self.config(rpc_zmq_host='127.0.0.1') self.config(rpc_response_timeout=5) diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py deleted file mode 100644 index fd653d7..0000000 --- a/tests/unit/test_plugin.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2012 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pkg_resources - -from openstack.common.notifier import api as notifier_api -from openstack.common.plugin import plugin -from openstack.common.plugin import pluginmanager -from tests import utils - - -class SimpleNotifier(object): - def __init__(self): - self.message_list = [] - - def notify(self, context, message): - self.context = context - self.message_list.append(message) - - -class ManagerTestCase(utils.BaseTestCase): - def test_constructs(self): - manager1 = pluginmanager.PluginManager("testproject", "testservice") - self.assertNotEqual(manager1, False) - - -class NotifyTestCase(utils.BaseTestCase): - """Test case for the plugin notification interface.""" - - def test_add_notifier(self): - notifier1 = SimpleNotifier() - notifier2 = SimpleNotifier() - notifier3 = SimpleNotifier() - - testplugin = plugin.Plugin('service') - testplugin._add_notifier(notifier1) - testplugin._add_notifier(notifier2) - self.assertEqual(len(testplugin.notifiers), 2) - - testplugin._add_notifier(notifier3) - self.assertEqual(len(testplugin.notifiers), 3) - - def test_notifier_action(self): - def mock_iter_entry_points(_t): - return [MockEntrypoint("fake", "fake", ["fake"])] - - self.stubs.Set(pkg_resources, 'iter_entry_points', - mock_iter_entry_points) - - plugmgr = pluginmanager.PluginManager("testproject", "testservice") - plugmgr.load_plugins() - self.assertEqual(len(plugmgr.plugins), 1) - self.assertEqual(len(plugmgr.plugins[0].notifiers), 1) - notifier = plugmgr.plugins[0].notifiers[0] - - notifier_api.notify('contextarg', 'publisher_id', 'event_type', - notifier_api.WARN, dict(a=3)) - - self.assertEqual(len(notifier.message_list), 1) - - -class StubControllerExtension(object): - name = 'stubextension' - alias = 'stubby' - - -class TestPluginClass(plugin.Plugin): - - def __init__(self, service_name): - super(TestPluginClass, self).__init__(service_name) - self._add_api_extension_descriptor(StubControllerExtension) - notifier1 = SimpleNotifier() - self._add_notifier(notifier1) - - -class MockEntrypoint(pkg_resources.EntryPoint): - def load(self): - return TestPluginClass - - -class MockExtManager(): - def __init__(self): - self.descriptors = [] - - def load_extension(self, descriptor): - self.descriptors.append(descriptor) - - -class APITestCase(utils.BaseTestCase): - """Test case for the plugin api extension interface.""" - def test_add_extension(self): - def mock_load(_s): - return TestPluginClass() - - def mock_iter_entry_points(_t): - return [MockEntrypoint("fake", "fake", ["fake"])] - - self.stubs.Set(pkg_resources, 'iter_entry_points', - mock_iter_entry_points) - - mgr = MockExtManager() - plugmgr = pluginmanager.PluginManager("testproject", "testservice") - plugmgr.load_plugins() - plugmgr.plugin_extension_factory(mgr) - - self.assertTrue(StubControllerExtension in mgr.descriptors) diff --git a/tests/utils.py b/tests/utils.py index 794a3d2..e93c278 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,17 +27,16 @@ import testtools from openstack.common import exception from openstack.common.fixture import moxstubout -CONF = cfg.CONF - class BaseTestCase(testtools.TestCase): - def setUp(self): + def setUp(self, conf=cfg.CONF): super(BaseTestCase, self).setUp() moxfixture = self.useFixture(moxstubout.MoxStubout()) self.mox = moxfixture.mox self.stubs = moxfixture.stubs - self.addCleanup(CONF.reset) + self.conf = conf + self.addCleanup(self.conf.reset) self.useFixture(fixtures.FakeLogger('openstack.common')) self.useFixture(fixtures.Timeout(30, True)) self.stubs.Set(exception, '_FATAL_EXCEPTION_FORMAT_ERRORS', True) @@ -46,7 +45,7 @@ class BaseTestCase(testtools.TestCase): def tearDown(self): super(BaseTestCase, self).tearDown() - CONF.reset() + self.conf.reset() self.stubs.UnsetAll() self.stubs.SmartUnsetAll() @@ -79,4 +78,4 @@ class BaseTestCase(testtools.TestCase): """ group = kw.pop('group', None) for k, v in kw.iteritems(): - CONF.set_override(k, v, group) + self.conf.set_override(k, v, group) |