diff options
25 files changed, 838 insertions, 27 deletions
@@ -2,3 +2,4 @@ # <preferred e-mail> <other e-mail 1> # <preferred e-mail> <other e-mail 2> Zhongyue Luo <zhongyue.nah@intel.com> <lzyeval@gmail.com> +<yufang521247@gmail.com> <zhangyufang@360.cn> diff --git a/openstack/common/gettextutils.py b/openstack/common/gettextutils.py index bbf8fe9..321fdd0 100644 --- a/openstack/common/gettextutils.py +++ b/openstack/common/gettextutils.py @@ -38,10 +38,26 @@ _localedir = os.environ.get('oslo'.upper() + '_LOCALEDIR') _t = gettext.translation('oslo', localedir=_localedir, fallback=True) _AVAILABLE_LANGUAGES = [] +USE_LAZY = False + + +def enable_lazy(): + """Convenience function for configuring _() to use lazy gettext + + Call this at the start of execution to enable the gettextutils._ + function to use lazy gettext functionality. This is useful if + your project is importing _ directly instead of using the + gettextutils.install() way of importing the _ function. + """ + global USE_LAZY + USE_LAZY = True def _(msg): - return _t.ugettext(msg) + if USE_LAZY: + return Message(msg, 'oslo') + else: + return _t.ugettext(msg) def install(domain, lazy=False): diff --git a/openstack/common/middleware/base.py b/openstack/common/middleware/base.py index 7236731..2099549 100644 --- a/openstack/common/middleware/base.py +++ b/openstack/common/middleware/base.py @@ -28,11 +28,7 @@ class Middleware(object): @classmethod def factory(cls, global_conf, **local_conf): """Factory method for paste.deploy.""" - - def filter(app): - return cls(app) - - return filter + return cls def __init__(self, application): self.application = application diff --git a/openstack/common/middleware/sizelimit.py b/openstack/common/middleware/sizelimit.py index ecbdde1..23ba9b6 100644 --- a/openstack/common/middleware/sizelimit.py +++ b/openstack/common/middleware/sizelimit.py @@ -71,9 +71,6 @@ class LimitingReader(object): class RequestBodySizeLimiter(base.Middleware): """Limit the size of incoming requests.""" - def __init__(self, *args, **kwargs): - super(RequestBodySizeLimiter, self).__init__(*args, **kwargs) - @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): if req.content_length > CONF.max_request_body_size: diff --git a/openstack/common/notifier/log_notifier.py b/openstack/common/notifier/log_notifier.py index d3ef0ae..96072ed 100644 --- a/openstack/common/notifier/log_notifier.py +++ b/openstack/common/notifier/log_notifier.py @@ -25,7 +25,7 @@ CONF = cfg.CONF def notify(_context, message): """Notifies the recipient of the desired event given the model. - Log notifications using openstack's default logging system. + Log notifications using OpenStack's default logging system. """ priority = message.get('priority', diff --git a/openstack/common/notifier/rpc_notifier.py b/openstack/common/notifier/rpc_notifier.py index 6bfc333..db47a8a 100644 --- a/openstack/common/notifier/rpc_notifier.py +++ b/openstack/common/notifier/rpc_notifier.py @@ -24,7 +24,7 @@ LOG = logging.getLogger(__name__) notification_topic_opt = cfg.ListOpt( 'notification_topics', default=['notifications', ], - help='AMQP topic used for openstack notifications') + help='AMQP topic used for OpenStack notifications') CONF = cfg.CONF CONF.register_opt(notification_topic_opt) diff --git a/openstack/common/notifier/rpc_notifier2.py b/openstack/common/notifier/rpc_notifier2.py index 55dd780..505a94e 100644 --- a/openstack/common/notifier/rpc_notifier2.py +++ b/openstack/common/notifier/rpc_notifier2.py @@ -26,7 +26,7 @@ LOG = logging.getLogger(__name__) notification_topic_opt = cfg.ListOpt( 'topics', default=['notifications', ], - help='AMQP topic(s) used for openstack notifications') + help='AMQP topic(s) used for OpenStack notifications') opt_group = cfg.OptGroup(name='rpc_notifier2', title='Options for rpc_notifier2') diff --git a/openstack/common/policy.py b/openstack/common/policy.py index 3a3c63f..ffb8668 100644 --- a/openstack/common/policy.py +++ b/openstack/common/policy.py @@ -221,7 +221,7 @@ class Enforcer(object): if policy_file: return policy_file - raise cfg.ConfigFilesNotFoundError(path=CONF.policy_file) + raise cfg.ConfigFilesNotFoundError((CONF.policy_file,)) def enforce(self, rule, target, creds, do_raise=False, exc=None, *args, **kwargs): diff --git a/openstack/common/processutils.py b/openstack/common/processutils.py index 13f6222..06fe411 100644 --- a/openstack/common/processutils.py +++ b/openstack/common/processutils.py @@ -19,6 +19,7 @@ System-level utilities and helper functions. """ +import logging as stdlib_logging import os import random import shlex @@ -102,6 +103,9 @@ def execute(*cmd, **kwargs): :param shell: whether or not there should be a shell used to execute this command. Defaults to false. :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be stdlib_logging.DEBUG or + stdlib_logging.INFO) :returns: (stdout, stderr) from process execution :raises: :class:`UnknownArgumentError` on receiving unknown arguments @@ -116,6 +120,7 @@ def execute(*cmd, **kwargs): run_as_root = kwargs.pop('run_as_root', False) root_helper = kwargs.pop('root_helper', '') shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) if isinstance(check_exit_code, bool): ignore_exit_code = not check_exit_code @@ -139,7 +144,7 @@ def execute(*cmd, **kwargs): while attempts > 0: attempts -= 1 try: - LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd)) + LOG.log(loglevel, _('Running cmd (subprocess): %s'), ' '.join(cmd)) _PIPE = subprocess.PIPE # pylint: disable=E1101 if os.name == 'nt': @@ -164,7 +169,7 @@ def execute(*cmd, **kwargs): obj.stdin.close() # pylint: disable=E1101 _returncode = obj.returncode # pylint: disable=E1101 if _returncode: - LOG.debug(_('Result was %s') % _returncode) + LOG.log(loglevel, _('Result was %s') % _returncode) if not ignore_exit_code and _returncode not in check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=_returncode, @@ -176,7 +181,7 @@ def execute(*cmd, **kwargs): if not attempts: raise else: - LOG.debug(_('%r failed. Retrying.'), cmd) + LOG.log(loglevel, _('%r failed. Retrying.'), cmd) if delay_on_retry: greenthread.sleep(random.randint(20, 200) / 100.0) finally: diff --git a/openstack/common/py3kcompat/urlutils.py b/openstack/common/py3kcompat/urlutils.py index 4471021..95ad274 100644 --- a/openstack/common/py3kcompat/urlutils.py +++ b/openstack/common/py3kcompat/urlutils.py @@ -17,7 +17,7 @@ # """ -Python2/Python3 compatibility layer for openstack +Python2/Python3 compatibility layer for OpenStack """ import six diff --git a/openstack/common/rootwrap/cmd.py b/openstack/common/rootwrap/cmd.py index 500f6c9..473dafb 100755..100644 --- a/openstack/common/rootwrap/cmd.py +++ b/openstack/common/rootwrap/cmd.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright (c) 2011 OpenStack Foundation. diff --git a/openstack/common/rpc/amqp.py b/openstack/common/rpc/amqp.py index 1afd2ab..38f2515 100644 --- a/openstack/common/rpc/amqp.py +++ b/openstack/common/rpc/amqp.py @@ -300,8 +300,13 @@ def pack_context(msg, context): for args at some point. """ - context_d = dict([('_context_%s' % key, value) - for (key, value) in context.to_dict().iteritems()]) + if isinstance(context, dict): + context_d = dict([('_context_%s' % key, value) + for (key, value) in context.iteritems()]) + else: + context_d = dict([('_context_%s' % key, value) + for (key, value) in context.to_dict().iteritems()]) + msg.update(context_d) diff --git a/openstack/common/rpc/impl_qpid.py b/openstack/common/rpc/impl_qpid.py index 99c4619..e54beb4 100644 --- a/openstack/common/rpc/impl_qpid.py +++ b/openstack/common/rpc/impl_qpid.py @@ -320,7 +320,7 @@ class DirectPublisher(Publisher): def __init__(self, conf, session, msg_id): """Init a 'direct' publisher.""" super(DirectPublisher, self).__init__(session, msg_id, - {"type": "Direct"}) + {"type": "direct"}) class TopicPublisher(Publisher): diff --git a/openstack/common/rpc/securemessage.py b/openstack/common/rpc/securemessage.py index 9dc2eca..c5530a6 100644 --- a/openstack/common/rpc/securemessage.py +++ b/openstack/common/rpc/securemessage.py @@ -47,10 +47,9 @@ secure_message_opts = [ cfg.StrOpt('kds_endpoint', help='KDS endpoint (ex: http://kds.example.com:35357/v3)'), ] -CONF = cfg.CONF -CONF.register_group(cfg.OptGroup('secure_messages', - title='Secure Messaging options')) -CONF.register_opts(secure_message_opts, group='secure_messages') +secure_message_group = cfg.OptGroup('secure_messages', + title='Secure Messaging options') + LOG = logging.getLogger(__name__) @@ -349,6 +348,9 @@ class SecureMessage(object): def __init__(self, topic, host, conf, key=None, key_store=None, encrypt=None, enctype='AES', hashtype='SHA256'): + conf.register_group(secure_message_group) + conf.register_opts(secure_message_opts, group='secure_messages') + self._name = '%s.%s' % (topic, host) self._key = key self._conf = conf.secure_messages diff --git a/openstack/common/rpc/zmq_receiver.py b/openstack/common/rpc/zmq_receiver.py index e74da22..000a7dd 100755..100644 --- a/openstack/common/rpc/zmq_receiver.py +++ b/openstack/common/rpc/zmq_receiver.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2011 OpenStack Foundation diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..bd850af --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Current scriipt is the simple wrapper on common tools/run_tests_common.sh +# scrip. It pass project specific variables to common script. +# +# Optins list (from tools/run_tests_common.sh). +# Use `./run_tests.sh -h` `./run_tests.sh --help` to get help message +# +# -V, --virtual-env Always use virtualenv. Install automatically if not present +# -N, --no-virtual-env Don't use virtualenv. Run tests in local environment +# -s, --no-site-packages Isolate the virtualenv from the global Python environment +# -r, --recreate-db Recreate the test database (deprecated, as this is now the default). +# -n, --no-recreate-db Don't recreate the test database. +# -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added. +# -u, --update Update the virtual environment with any newer package versions +# -p, --pep8 Just run PEP8 and HACKING compliance check +# -P, --no-pep8 Don't run static code checks +# -c, --coverage Generate coverage report +# -d, --debug Run tests with testtools instead of testr. This allows you to use the debugger. +# -h, --help Print this usage message +# --hide-elapsed Don't print the elapsed time for each test along with slow test list +# --virtual-env-path <path> Location of the virtualenv directory. Default: \$(pwd) +# --virtual-env-name <name> Name of the virtualenv directory. Default: .venv +# --tools-path <dir> Location of the tools directory. Default: \$(pwd) +# +# Note: with no options specified, the script will try to run the tests in a +# virtual environment, if no virtualenv is found, the script will ask if +# you would like to create one. If you prefer to run tests NOT in a +# virtual environment, simply pass the -N option. + + +# On Linux, testrepository will inspect /proc/cpuinfo to determine how many +# CPUs are present in the machine, and run one worker per CPU. +# Set workers_count=0 if you want to run one worker per CPU. +# Make our paths available to run_tests_common.sh using `export` statement +# export WORKERS_COUNT=0 + +# there are no possibility to run some oslo tests with concurrency > 1 +# or separately due to dependencies between tests (see bug 1192207) +export WORKERS_COUNT=1 +# option include {PROJECT_NAME}/* directory to coverage report if `-c` or +# `--coverage` uses +export PROJECT_NAME="openstack" +# option exclude "${PROJECT_NAME}/openstack/common/*" from coverage report +# if equals to 1 +export OMIT_OSLO_FROM_COVERAGE=0 +# path to directory with tests +export TESTS_DIR="tests/" +export EGG_INFO_FILE="openstack.common.egg-info/entry_points.txt" + +# run common test script +tools/run_tests_common.sh $* diff --git a/tests/unit/rpc/amqp.py b/tests/unit/rpc/amqp.py index 83713c7..76d6946 100644 --- a/tests/unit/rpc/amqp.py +++ b/tests/unit/rpc/amqp.py @@ -22,6 +22,7 @@ Unit Tests for AMQP-based remote procedure calls import logging from eventlet import greenthread +import mock from oslo.config import cfg from openstack.common import jsonutils @@ -177,3 +178,13 @@ class BaseRpcAMQPTestCase(common.BaseRpcTestCase): conn.close() self.assertTrue(self.exc_raised) + + def test_context_dict_type_check(self): + """Test that context is handled properly depending on the type.""" + fake_context = {'fake': 'context'} + mock_msg = mock.MagicMock() + rpc_amqp.pack_context(mock_msg, fake_context) + + # assert first arg in args was a dict type + args = mock_msg.update.call_args[0] + self.assertIsInstance(args[0], dict) diff --git a/tests/unit/test_gettext.py b/tests/unit/test_gettext.py index d9cb9b8..ef16ac3 100644 --- a/tests/unit/test_gettext.py +++ b/tests/unit/test_gettext.py @@ -33,6 +33,42 @@ LOG = logging.getLogger(__name__) class GettextTest(utils.BaseTestCase): + def setUp(self): + super(GettextTest, self).setUp() + # remember so we can reset to it later + self._USE_LAZY = gettextutils.USE_LAZY + + def tearDown(self): + # reset to value before test + gettextutils.USE_LAZY = self._USE_LAZY + super(GettextTest, self).tearDown() + + def test_enable_lazy(self): + gettextutils.USE_LAZY = False + + gettextutils.enable_lazy() + # assert now enabled + self.assertTrue(gettextutils.USE_LAZY) + + def test_underscore_non_lazy(self): + # set lazy off + gettextutils.USE_LAZY = False + + self.mox.StubOutWithMock(gettextutils._t, 'ugettext') + gettextutils._t.ugettext('blah').AndReturn('translated blah') + self.mox.ReplayAll() + + result = gettextutils._('blah') + self.assertEqual('translated blah', result) + + def test_underscore_lazy(self): + # set lazy off + gettextutils.USE_LAZY = False + + gettextutils.enable_lazy() + result = gettextutils._('blah') + self.assertIsInstance(result, gettextutils.Message) + def test_gettext_does_not_blow_up(self): LOG.info(gettextutils._('test')) diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index bd412bf..cc2afe7 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -208,6 +208,11 @@ class EnforcerTest(PolicyBaseTestCase): self.assertEquals(self.enforcer.rules, {'test': 'test', 'test1': 'test1'}) + def test_get_policy_path_raises_exc(self): + enforcer = policy.Enforcer(policy_file='raise_error.json') + self.assertRaises(cfg.ConfigFilesNotFoundError, + enforcer._get_policy_path) + class FakeCheck(policy.BaseCheck): def __init__(self, result=None): diff --git a/tests/utils.py b/tests/utils.py index e93c278..f49e0af 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -38,7 +38,24 @@ class BaseTestCase(testtools.TestCase): self.conf = conf self.addCleanup(self.conf.reset) self.useFixture(fixtures.FakeLogger('openstack.common')) - self.useFixture(fixtures.Timeout(30, True)) + + test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) + try: + test_timeout = int(test_timeout) + except ValueError: + # If timeout value is invalid do not set a timeout. + test_timeout = 0 + if test_timeout > 0: + self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) + if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or + os.environ.get('OS_STDOUT_CAPTURE') == '1'): + stdout = self.useFixture(fixtures.StringStream('stdout')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) + if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or + os.environ.get('OS_STDERR_CAPTURE') == '1'): + stderr = self.useFixture(fixtures.StringStream('stderr')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) + self.stubs.Set(exception, '_FATAL_EXCEPTION_FORMAT_ERRORS', True) self.useFixture(fixtures.NestedTempfile()) self.tempdirs = [] diff --git a/tools/colorizer.py b/tools/colorizer.py new file mode 100755 index 0000000..13364ba --- /dev/null +++ b/tools/colorizer.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2013, Nebula, Inc. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Display a subunit stream through a colorized unittest test runner.""" + +import heapq +import subunit +import sys +import unittest + +import testtools + + +class _AnsiColorizer(object): + """Colorizer allows callers to write text in a particular color. + + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """Check is the current platform supports coloring terminal output. + + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except Exception: + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """See _AnsiColorizer docstring.""" + def __init__(self, stream): + import win32console + red, green, blue, bold = (win32console.FOREGROUND_RED, + win32console.FOREGROUND_GREEN, + win32console.FOREGROUND_BLUE, + win32console.FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold, + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """See _AnsiColorizer docstring.""" + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + + +def get_elapsed_time_color(elapsed_time): + if elapsed_time > 1.0: + return 'red' + elif elapsed_time > 0.25: + return 'yellow' + else: + return 'green' + + +class OpenStackTestResult(testtools.TestResult): + def __init__(self, stream, descriptions, verbosity): + super(OpenStackTestResult, self).__init__() + self.stream = stream + self.showAll = verbosity > 1 + self.num_slow_tests = 10 + self.slow_tests = [] # this is a fixed-sized heap + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout + self.start_time = None + self.last_time = {} + self.results = {} + self.last_written = None + + def _writeElapsedTime(self, elapsed): + color = get_elapsed_time_color(elapsed) + self.colorizer.write(" %.2f" % elapsed, color) + + def _addResult(self, test, *args): + try: + name = test.id() + except AttributeError: + name = 'Unknown.unknown' + test_class, test_name = name.rsplit('.', 1) + + elapsed = (self._now() - self.start_time).total_seconds() + item = (elapsed, test_class, test_name) + if len(self.slow_tests) >= self.num_slow_tests: + heapq.heappushpop(self.slow_tests, item) + else: + heapq.heappush(self.slow_tests, item) + + self.results.setdefault(test_class, []) + self.results[test_class].append((test_name, elapsed) + args) + self.last_time[test_class] = self._now() + self.writeTests() + + def _writeResult(self, test_name, elapsed, long_result, color, + short_result, success): + if self.showAll: + self.stream.write(' %s' % str(test_name).ljust(66)) + self.colorizer.write(long_result, color) + if success: + self._writeElapsedTime(elapsed) + self.stream.writeln() + else: + self.colorizer.write(short_result, color) + + def addSuccess(self, test): + super(OpenStackTestResult, self).addSuccess(test) + self._addResult(test, 'OK', 'green', '.', True) + + def addFailure(self, test, err): + if test.id() == 'process-returncode': + return + super(OpenStackTestResult, self).addFailure(test, err) + self._addResult(test, 'FAIL', 'red', 'F', False) + + def addError(self, test, err): + super(OpenStackTestResult, self).addFailure(test, err) + self._addResult(test, 'ERROR', 'red', 'E', False) + + def addSkip(self, test, reason=None, details=None): + super(OpenStackTestResult, self).addSkip(test, reason, details) + self._addResult(test, 'SKIP', 'blue', 'S', True) + + def startTest(self, test): + self.start_time = self._now() + super(OpenStackTestResult, self).startTest(test) + + def writeTestCase(self, cls): + if not self.results.get(cls): + return + if cls != self.last_written: + self.colorizer.write(cls, 'white') + self.stream.writeln() + for result in self.results[cls]: + self._writeResult(*result) + del self.results[cls] + self.stream.flush() + self.last_written = cls + + def writeTests(self): + time = self.last_time.get(self.last_written, self._now()) + if not self.last_written or (self._now() - time).total_seconds() > 2.0: + diff = 3.0 + while diff > 2.0: + classes = self.results.keys() + oldest = min(classes, key=lambda x: self.last_time[x]) + diff = (self._now() - self.last_time[oldest]).total_seconds() + self.writeTestCase(oldest) + else: + self.writeTestCase(self.last_written) + + def done(self): + self.stopTestRun() + + def stopTestRun(self): + for cls in list(self.results.iterkeys()): + self.writeTestCase(cls) + self.stream.writeln() + self.writeSlowTests() + + def writeSlowTests(self): + # Pare out 'fast' tests + slow_tests = [item for item in self.slow_tests + if get_elapsed_time_color(item[0]) != 'green'] + if slow_tests: + slow_total_time = sum(item[0] for item in slow_tests) + slow = ("Slowest %i tests took %.2f secs:" + % (len(slow_tests), slow_total_time)) + self.colorizer.write(slow, 'yellow') + self.stream.writeln() + last_cls = None + # sort by name + for elapsed, cls, name in sorted(slow_tests, + key=lambda x: x[1] + x[2]): + if cls != last_cls: + self.colorizer.write(cls, 'white') + self.stream.writeln() + last_cls = cls + self.stream.write(' %s' % str(name).ljust(68)) + self._writeElapsedTime(elapsed) + self.stream.writeln() + + def printErrors(self): + if self.showAll: + self.stream.writeln() + self.printErrorList('ERROR', self.errors) + self.printErrorList('FAIL', self.failures) + + def printErrorList(self, flavor, errors): + for test, err in errors: + self.colorizer.write("=" * 70, 'red') + self.stream.writeln() + self.colorizer.write(flavor, 'red') + self.stream.writeln(": %s" % test.id()) + self.colorizer.write("-" * 70, 'red') + self.stream.writeln() + self.stream.writeln("%s" % err) + + +test = subunit.ProtocolTestCase(sys.stdin, passthrough=None) + +if sys.version_info[0:2] <= (2, 6): + runner = unittest.TextTestRunner(verbosity=2) +else: + runner = unittest.TextTestRunner(verbosity=2, + resultclass=OpenStackTestResult) + +if runner.run(test).wasSuccessful(): + exit_code = 0 +else: + exit_code = 1 +sys.exit(exit_code) diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh index 26f02dd..f5e5a67 100755 --- a/tools/config/generate_sample.sh +++ b/tools/config/generate_sample.sh @@ -64,6 +64,9 @@ FILES=$(find $BASEDIR/$PACKAGENAME -type f -name "*.py" ! -path "*/tests/*" \ export EVENTLET_NO_GREENDNS=yes +OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) +[ "$OS_VARS" ] && eval "unset \$OS_VARS" + MODULEPATH=openstack.common.config.generator OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample python -m $MODULEPATH $FILES > $OUTPUTFILE diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 0000000..1d4e7e0 --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import install_venv_common as install_venv # noqa + + +def print_help(venv, root): + help = """ + Openstack development environment setup is complete. + + Openstack development uses virtualenv to track and manage Python + dependencies while in development and testing. + + To activate the Openstack virtualenv for the extent of your current shell + session you can run: + + $ source %s/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ %s/tools/with_venv.sh <your command> + + Also, make test will automatically use the virtualenv. + """ + print(help % (venv, root)) + + +def main(argv): + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + if os.environ.get('tools_path'): + root = os.environ['tools_path'] + venv = os.path.join(root, '.venv') + if os.environ.get('venv'): + venv = os.environ['venv'] + + pip_requires = os.path.join(root, 'requirements.txt') + test_requires = os.path.join(root, 'test-requirements.txt') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + project = 'Openstack' + install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, + py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies() + install.post_process() + print_help(venv, root) + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/run_tests_common.sh b/tools/run_tests_common.sh new file mode 100755 index 0000000..1cc4e6f --- /dev/null +++ b/tools/run_tests_common.sh @@ -0,0 +1,253 @@ +#!/bin/bash + +set -eu + +function usage { + echo "Usage: $0 [OPTION]..." + echo "Run project's test suite(s)" + echo "" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -s, --no-site-packages Isolate the virtualenv from the global Python environment" + echo " -r, --recreate-db Recreate the test database (deprecated, as this is now the default)." + echo " -n, --no-recreate-db Don't recreate the test database." + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." + echo " -u, --update Update the virtual environment with any newer package versions" + echo " -p, --pep8 Just run PEP8 and HACKING compliance check" + echo " -P, --no-pep8 Don't run static code checks" + echo " -c, --coverage Generate coverage report" + echo " -d, --debug Run tests with testtools instead of testr. This allows you to use the debugger." + echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" + echo " --virtual-env-path <path> Location of the virtualenv directory" + echo " Default: \$(pwd)" + echo " --virtual-env-name <name> Name of the virtualenv directory" + echo " Default: .venv" + echo " --tools-path <dir> Location of the tools directory" + echo " Default: \$(pwd)" + echo "" + echo "Note: with no options specified, the script will try to run the tests in a virtual environment," + echo " If no virtualenv is found, the script will ask if you would like to create one. If you " + echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." + exit +} + +function process_options { + i=1 + while [ $i -le $# ]; do + case "${!i}" in + -h|--help) usage;; + -V|--virtual-env) ALWAYS_VENV=1; NEVER_VENV=0;; + -N|--no-virtual-env) ALWAYS_VENV=0; NEVER_VENV=1;; + -s|--no-site-packages) NO_SITE_PACKAGES=1;; + -r|--recreate-db) RECREATE_DB=1;; + -n|--no-recreate-db) RECREATE_DB=0;; + -f|--force) FORCE=1;; + -u|--update) UPDATE=1;; + -p|--pep8) JUST_PEP8=1;; + -P|--no-pep8) NO_PEP8=1;; + -c|--coverage) COVERAGE=1;; + -d|--debug) DEBUG=1;; + --virtual-env-path) + (( i++ )) + VENV_PATH=${!i} + ;; + --virtual-env-name) + (( i++ )) + VENV_DIR=${!i} + ;; + --tools-path) + (( i++ )) + TOOLS_PATH=${!i} + ;; + -*) TESTOPTS="$TESTOPTS ${!i}";; + *) TESTRARGS="$TESTRARGS ${!i}" + esac + (( i++ )) + done +} + + +TOOLS_PATH=${TOOLS_PATH:-${PWD}} +VENV_PATH=${VENV_PATH:-${PWD}} +VENV_DIR=${VENV_NAME:-.venv} +WITH_VENV=${TOOLS_PATH}/tools/with_venv.sh + +ALWAYS_VENV=0 +NEVER_VENV=0 +FORCE=0 +NO_SITE_PACKAGES=1 +INSTALLVENVOPTS= +TESTRARGS= +TESTOPTS= +WRAPPER="" +JUST_PEP8=0 +NO_PEP8=0 +COVERAGE=0 +DEBUG=0 +RECREATE_DB=1 +UPDATE=0 + +LANG=en_US.UTF-8 +LANGUAGE=en_US:en +LC_ALL=C + +process_options $@ +# Make our paths available to other scripts we call +export VENV_PATH +export TOOLS_PATH +export VENV_DIR +export VENV_NAME +export WITH_VENV +export VENV=${VENV_PATH}/${VENV_DIR} + +function init_testr { + if [ ! -d .testrepository ]; then + ${WRAPPER} testr init + fi +} + +function run_tests { + # Cleanup *pyc + ${WRAPPER} find . -type f -name "*.pyc" -delete + + if [ ${DEBUG} -eq 1 ]; then + if [ "${TESTOPTS}" = "" ] && [ "${TESTRARGS}" = "" ]; then + # Default to running all tests if specific test is not + # provided. + TESTRARGS="discover ./${TESTS_DIR}" + fi + ${WRAPPER} python -m testtools.run ${TESTOPTS} ${TESTRARGS} + + # Short circuit because all of the testr and coverage stuff + # below does not make sense when running testtools.run for + # debugging purposes. + return $? + fi + + if [ ${COVERAGE} -eq 1 ]; then + TESTRTESTS="${TESTRTESTS} --coverage" + else + TESTRTESTS="${TESTRTESTS}" + fi + + # Just run the test suites in current environment + set +e + TESTRARGS=`echo "${TESTRARGS}" | sed -e's/^\s*\(.*\)\s*$/\1/'` + + if [ ${WORKERS_COUNT} -ne 0 ]; then + TESTRTESTS="${TESTRTESTS} --testr-args='--concurrency=${WORKERS_COUNT} --subunit ${TESTOPTS} ${TESTRARGS}'" + else + TESTRTESTS="${TESTRTESTS} --testr-args='--subunit ${TESTOPTS} ${TESTRARGS}'" + fi + + if [ setup.cfg -nt ${EGG_INFO_FILE} ]; then + ${WRAPPER} python setup.py egg_info + fi + + echo "Running \`${WRAPPER} ${TESTRTESTS}\`" + if ${WRAPPER} which subunit-2to1 2>&1 > /dev/null; then + # subunit-2to1 is present, testr subunit stream should be in version 2 + # format. Convert to version one before colorizing. + bash -c "${WRAPPER} ${TESTRTESTS} | ${WRAPPER} subunit-2to1 | ${WRAPPER} ${TOOLS_PATH}/tools/colorizer.py" + else + bash -c "${WRAPPER} ${TESTRTESTS} | ${WRAPPER} ${TOOLS_PATH}/tools/colorizer.py" + fi + RESULT=$? + set -e + + copy_subunit_log + + if [ $COVERAGE -eq 1 ]; then + echo "Generating coverage report in covhtml/" + ${WRAPPER} coverage combine + # Don't compute coverage for common code, which is tested elsewhere + # if we are not in `oslo-incubator` project + if [ ${OMIT_OSLO_FROM_COVERAGE} -eq 0 ]; then + OMIT_OSLO="" + else + OMIT_OSLO="--omit='${PROJECT_NAME}/openstack/common/*'" + fi + ${WRAPPER} coverage html --include='${PROJECT_NAME}/*' ${OMIT_OSLO} -d covhtml -i + fi + + return ${RESULT} +} + +function copy_subunit_log { + LOGNAME=`cat .testrepository/next-stream` + LOGNAME=$((${LOGNAME} - 1)) + LOGNAME=".testrepository/${LOGNAME}" + cp ${LOGNAME} subunit.log +} + +function run_pep8 { + echo "Running flake8 ..." + bash -c "${WRAPPER} flake8" +} + + +TESTRTESTS="python setup.py testr" + +if [ ${NO_SITE_PACKAGES} -eq 1 ]; then + INSTALLVENVOPTS="--no-site-packages" +fi + +if [ ${NEVER_VENV} -eq 0 ]; then + # Remove the virtual environment if -f or --force used + if [ ${FORCE} -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${VENV} + fi + + # Update the virtual environment if -u or --update used + if [ ${UPDATE} -eq 1 ]; then + echo "Updating virtualenv..." + python ${TOOLS_PATH}/tools/install_venv.py ${INSTALLVENVOPTS} + fi + + if [ -e ${VENV} ]; then + WRAPPER="${WITH_VENV}" + else + if [ ${ALWAYS_VENV} -eq 1 ]; then + # Automatically install the virtualenv + python ${TOOLS_PATH}/tools/install_venv.py ${INSTALLVENVOPTS} + WRAPPER="${WITH_VENV}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read USE_VENV + if [ "x${USE_VENV}" = "xY" -o "x${USE_VENV}" = "x" -o "x${USE_VENV}" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python ${TOOLS_PATH}/tools/install_venv.py ${INSTALLVENVOPTS} + WRAPPER=${WITH_VENV} + fi + fi + fi +fi + +# Delete old coverage data from previous runs +if [ ${COVERAGE} -eq 1 ]; then + ${WRAPPER} coverage erase +fi + +if [ ${JUST_PEP8} -eq 1 ]; then + run_pep8 + exit +fi + +if [ ${RECREATE_DB} -eq 1 ]; then + rm -f tests.sqlite +fi + +init_testr +run_tests + +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (testropts), which begin with a '-', and +# arguments (testrargs). +if [ -z "${TESTRARGS}" ]; then + if [ ${NO_PEP8} -eq 0 ]; then + run_pep8 + fi +fi diff --git a/tools/with_venv.sh b/tools/with_venv.sh new file mode 100755 index 0000000..7303990 --- /dev/null +++ b/tools/with_venv.sh @@ -0,0 +1,7 @@ +#!/bin/bash +TOOLS_PATH=${TOOLS_PATH:-$(dirname $0)} +VENV_PATH=${VENV_PATH:-${TOOLS_PATH}} +VENV_DIR=${VENV_NAME:-/../.venv} +TOOLS=${TOOLS_PATH} +VENV=${VENV:-${VENV_PATH}/${VENV_DIR}} +source ${VENV}/bin/activate && "$@" |