diff options
27 files changed, 667 insertions, 112 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index 0500c53..b1b3b38 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -85,6 +85,12 @@ M: Zhongyue Luo <zhongyue.nah@intel.com> S: Maintained F: config/ +== crypto == + +M: Simo Sorce <simo@redhat.com> +S: Maintained +F: crypto/ + == db == M: diff --git a/openstack/common/config/generator.py b/openstack/common/config/generator.py index 8ebfba1..404a09a 100755 --- a/openstack/common/config/generator.py +++ b/openstack/common/config/generator.py @@ -18,8 +18,11 @@ # # @author: Zhongyue Luo, SINA Corporation. # + """Extracts OpenStack config option info from module(s).""" +from __future__ import print_function + import imp import os import re @@ -97,7 +100,7 @@ def generate(srcfiles): for group, opts in opts_by_group.items(): print_group_opts(group, opts) - print "# Total option count: %d" % OPTION_COUNT + print("# Total option count: %d" % OPTION_COUNT) def _import_module(mod_str): @@ -161,18 +164,18 @@ def _list_opts(obj): def print_group_opts(group, opts_by_module): - print "[%s]" % group - print + print("[%s]" % group) + print('') global OPTION_COUNT for mod, opts in opts_by_module: OPTION_COUNT += len(opts) - print '#' - print '# Options defined in %s' % mod - print '#' - print + print('#') + print('# Options defined in %s' % mod) + print('#') + print('') for opt in opts: _print_opt(opt) - print + print('') def _get_my_ip(): @@ -188,7 +191,12 @@ def _get_my_ip(): def _sanitize_default(s): """Set up a reasonably sensible default for pybasedir, my_ip and host.""" - if s.startswith(BASEDIR): + if s.startswith(sys.prefix): + # NOTE(jd) Don't use os.path.join, because it is likely to think the + # second part is an absolute pathname and therefore drop the first + # part. + s = os.path.normpath("/usr/" + s[len(sys.prefix):]) + elif s.startswith(BASEDIR): return s.replace(BASEDIR, '/usr/lib/python/site-packages') elif BASEDIR in s: return s.replace(BASEDIR, '') @@ -213,33 +221,33 @@ def _print_opt(opt): sys.stderr.write("%s\n" % str(err)) sys.exit(1) opt_help += ' (' + OPT_TYPES[opt_type] + ')' - print '#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)) + print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) try: if opt_default is None: - print '#%s=<None>' % opt_name + print('#%s=<None>' % opt_name) elif opt_type == STROPT: assert(isinstance(opt_default, basestring)) - print '#%s=%s' % (opt_name, _sanitize_default(opt_default)) + print('#%s=%s' % (opt_name, _sanitize_default(opt_default))) elif opt_type == BOOLOPT: assert(isinstance(opt_default, bool)) - print '#%s=%s' % (opt_name, str(opt_default).lower()) + print('#%s=%s' % (opt_name, str(opt_default).lower())) elif opt_type == INTOPT: assert(isinstance(opt_default, int) and not isinstance(opt_default, bool)) - print '#%s=%s' % (opt_name, opt_default) + print('#%s=%s' % (opt_name, opt_default)) elif opt_type == FLOATOPT: assert(isinstance(opt_default, float)) - print '#%s=%s' % (opt_name, opt_default) + print('#%s=%s' % (opt_name, opt_default)) elif opt_type == LISTOPT: assert(isinstance(opt_default, list)) - print '#%s=%s' % (opt_name, ','.join(opt_default)) + print('#%s=%s' % (opt_name, ','.join(opt_default))) elif opt_type == MULTISTROPT: assert(isinstance(opt_default, list)) if not opt_default: opt_default = [''] for default in opt_default: - print '#%s=%s' % (opt_name, default) - print + print('#%s=%s' % (opt_name, default)) + print('') except Exception: sys.stderr.write('Error in option "%s"\n' % opt_name) sys.exit(1) @@ -247,7 +255,7 @@ def _print_opt(opt): def main(): if len(sys.argv) < 2: - print "usage: %s [srcfile]...\n" % sys.argv[0] + print("usage: %s [srcfile]...\n" % sys.argv[0]) sys.exit(0) generate(sys.argv[1:]) diff --git a/openstack/common/context.py b/openstack/common/context.py index 3899c2c..81772bc 100644 --- a/openstack/common/context.py +++ b/openstack/common/context.py @@ -61,7 +61,7 @@ class RequestContext(object): 'request_id': self.request_id} -def get_admin_context(show_deleted="no"): +def get_admin_context(show_deleted=False): context = RequestContext(None, tenant=None, is_admin=True, diff --git a/openstack/common/crypto/__init__.py b/openstack/common/crypto/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/openstack/common/crypto/__init__.py diff --git a/openstack/common/crypto/utils.py b/openstack/common/crypto/utils.py new file mode 100644 index 0000000..61c1a50 --- /dev/null +++ b/openstack/common/crypto/utils.py @@ -0,0 +1,179 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 + +from Crypto.Hash import HMAC +from Crypto import Random + +from openstack.common.gettextutils import _ +from openstack.common.importutils import import_module + + +class CryptoutilsException(Exception): + """Generic Exception for Crypto utilities.""" + + message = _("An unknown error occurred in crypto utils.") + + +class CipherBlockLengthTooBig(CryptoutilsException): + """The block size is too big.""" + + def __init__(self, requested, permitted): + msg = _("Block size of %(given)d is too big, max = %(maximum)d") + message = msg % {'given': requested, 'maximum': permitted} + super(CryptoutilsException, self).__init__(message) + + +class HKDFOutputLengthTooLong(CryptoutilsException): + """The amount of Key Material asked is too much.""" + + def __init__(self, requested, permitted): + msg = _("Length of %(given)d is too long, max = %(maximum)d") + message = msg % {'given': requested, 'maximum': permitted} + super(CryptoutilsException, self).__init__(message) + + +class HKDF(object): + """An HMAC-based Key Derivation Function implementation (RFC5869) + + This class creates an object that allows to use HKDF to derive keys. + """ + + def __init__(self, hashtype='SHA256'): + self.hashfn = import_module('Crypto.Hash.' + hashtype) + self.max_okm_length = 255 * self.hashfn.digest_size + + def extract(self, ikm, salt=None): + """An extract function that can be used to derive a robust key given + weak Input Key Material (IKM) which could be a password. + Returns a pseudorandom key (of HashLen octets) + + :param ikm: input keying material (ex a password) + :param salt: optional salt value (a non-secret random value) + """ + if salt is None: + salt = '\x00' * self.hashfn.digest_size + + return HMAC.new(salt, ikm, self.hashfn).digest() + + def expand(self, prk, info, length): + """An expand function that will return arbitrary length output that can + be used as keys. + Returns a buffer usable as key material. + + :param prk: a pseudorandom key of at least HashLen octets + :param info: optional string (can be a zero-length string) + :param length: length of output keying material (<= 255 * HashLen) + """ + if length > self.max_okm_length: + raise HKDFOutputLengthTooLong(length, self.max_okm_length) + + N = (length + self.hashfn.digest_size - 1) / self.hashfn.digest_size + + okm = "" + tmp = "" + for block in range(1, N + 1): + tmp = HMAC.new(prk, tmp + info + chr(block), self.hashfn).digest() + okm += tmp + + return okm[:length] + + +MAX_CB_SIZE = 256 + + +class SymmetricCrypto(object): + """Symmetric Key Crypto object. + + This class creates a Symmetric Key Crypto object that can be used + to encrypt, decrypt, or sign arbitrary data. + + :param enctype: Encryption Cipher name (default: AES) + :param hashtype: Hash/HMAC type name (default: SHA256) + """ + + def __init__(self, enctype='AES', hashtype='SHA256'): + self.cipher = import_module('Crypto.Cipher.' + enctype) + self.hashfn = import_module('Crypto.Hash.' + hashtype) + + def new_key(self, size): + return Random.new().read(size) + + def encrypt(self, key, msg, b64encode=True): + """Encrypt the provided msg and returns the cyphertext optionally + base64 encoded. + + Uses AES-128-CBC with a Random IV by default. + + The plaintext is padded to reach blocksize length. + The last byte of the block is the length of the padding. + The length of the padding does not include the length byte itself. + + :param key: The Encryption key. + :param msg: the plain text. + + :returns encblock: a block of encrypted data. + """ + iv = Random.new().read(self.cipher.block_size) + cipher = self.cipher.new(key, self.cipher.MODE_CBC, iv) + + # CBC mode requires a fixed block size. Append padding and length of + # padding. + if self.cipher.block_size > MAX_CB_SIZE: + raise CipherBlockLengthTooBig(self.cipher.block_size, MAX_CB_SIZE) + r = len(msg) % self.cipher.block_size + padlen = self.cipher.block_size - r - 1 + msg += '\x00' * padlen + msg += chr(padlen) + + enc = iv + cipher.encrypt(msg) + if b64encode: + enc = base64.b64encode(enc) + return enc + + def decrypt(self, key, msg, b64decode=True): + """Decrypts the provided ciphertext, optionally base 64 encoded, and + returns the plaintext message, after padding is removed. + + Uses AES-128-CBC with an IV by default. + + :param key: The Encryption key. + :param msg: the ciphetext, the first block is the IV + """ + if b64decode: + msg = base64.b64decode(msg) + iv = msg[:self.cipher.block_size] + cipher = self.cipher.new(key, self.cipher.MODE_CBC, iv) + + padded = cipher.decrypt(msg[self.cipher.block_size:]) + l = ord(padded[-1]) + 1 + plain = padded[:-l] + return plain + + def sign(self, key, msg, b64encode=True): + """Signs a message string and returns a base64 encoded signature. + + Uses HMAC-SHA-256 by default. + + :param key: The Signing key. + :param msg: the message to sign. + """ + h = HMAC.new(key, msg, self.hashfn) + out = h.digest() + if b64encode: + out = base64.b64encode(out) + return out diff --git a/openstack/common/excutils.py b/openstack/common/excutils.py index d40d46c..336e147 100644 --- a/openstack/common/excutils.py +++ b/openstack/common/excutils.py @@ -19,7 +19,6 @@ Exception related utilities. """ -import contextlib import logging import sys import time @@ -28,8 +27,7 @@ import traceback from openstack.common.gettextutils import _ -@contextlib.contextmanager -def save_and_reraise_exception(): +class save_and_reraise_exception(object): """Save current exception, run some code and then re-raise. In some cases the exception context can be cleared, resulting in None @@ -41,15 +39,33 @@ def save_and_reraise_exception(): To work around this, we save the exception state, run handler code, and then re-raise the original exception. If another exception occurs, the saved exception is logged and the new exception is re-raised. - """ - type_, value, tb = sys.exc_info() - try: - yield + + In some cases the caller may not want to re-raise the exception, and + for those circumstances this context provides a reraise flag that + can be used to suppress the exception. For example: + except Exception: - logging.error(_('Original exception being dropped: %s'), - traceback.format_exception(type_, value, tb)) - raise - raise type_, value, tb + with save_and_reraise_exception() as ctxt: + decide_if_need_reraise() + if not should_be_reraised: + ctxt.reraise = False + """ + def __init__(self): + self.reraise = True + + def __enter__(self): + self.type_, self.value, self.tb, = sys.exc_info() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + logging.error(_('Original exception being dropped: %s'), + traceback.format_exception(self.type_, + self.value, + self.tb)) + return False + if self.reraise: + raise self.type_, self.value, self.tb def forever_retry_uncaught_exceptions(infunc): diff --git a/openstack/common/log.py b/openstack/common/log.py index 8097b23..0447a52 100644 --- a/openstack/common/log.py +++ b/openstack/common/log.py @@ -74,7 +74,8 @@ logging_cli_opts = [ cfg.StrOpt('log-format', default=None, metavar='FORMAT', - help='A logging.Formatter log message format string which may ' + help='DEPRECATED. ' + 'A logging.Formatter log message format string which may ' 'use any of the available logging.LogRecord attributes. ' 'This option is deprecated. Please use ' 'logging_context_format_string and ' diff --git a/openstack/common/notifier/api.py b/openstack/common/notifier/api.py index 7c4dbd1..dc4f578 100644 --- a/openstack/common/notifier/api.py +++ b/openstack/common/notifier/api.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import socket import uuid from oslo.config import cfg @@ -35,7 +36,7 @@ notifier_opts = [ default='INFO', help='Default notification level for outgoing notifications'), cfg.StrOpt('default_publisher_id', - default='$host', + default=None, help='Default publisher_id for outgoing notifications'), ] @@ -74,7 +75,7 @@ def notify_decorator(name, fn): ctxt = context.get_context_from_function_and_args(fn, args, kwarg) notify(ctxt, - CONF.default_publisher_id, + CONF.default_publisher_id or socket.gethostname(), name, CONF.default_notification_level, body) @@ -84,7 +85,10 @@ def notify_decorator(name, fn): def publisher_id(service, host=None): if not host: - host = CONF.host + try: + host = CONF.host + except AttributeError: + host = CONF.default_publisher_id or socket.gethostname() return "%s.%s" % (service, host) diff --git a/openstack/common/rootwrap/filters.py b/openstack/common/rootwrap/filters.py index dfec412..660434a 100644 --- a/openstack/common/rootwrap/filters.py +++ b/openstack/common/rootwrap/filters.py @@ -47,7 +47,7 @@ class CommandFilter(object): def match(self, userargs): """Only check that the first argument (command) matches exec_path.""" - return os.path.basename(self.exec_path) == userargs[0] + return userargs and os.path.basename(self.exec_path) == userargs[0] def get_command(self, userargs, exec_dirs=[]): """Returns command to execute (with sudo -u if run_as != root).""" @@ -67,7 +67,7 @@ class RegExpFilter(CommandFilter): def match(self, userargs): # Early skip if command or number of args don't match - if (len(self.args) != len(userargs)): + if (not userargs or len(self.args) != len(userargs)): # DENY: argument numbers don't match return False # Compare each arg (anchoring pattern explicitly at end of string) @@ -101,6 +101,9 @@ class PathFilter(CommandFilter): """ def match(self, userargs): + if not userargs or len(userargs) < 2: + return False + command, arguments = userargs[0], userargs[1:] equal_args_num = len(self.args) == len(arguments) @@ -178,7 +181,7 @@ class KillFilter(CommandFilter): super(KillFilter, self).__init__("/bin/kill", *args) def match(self, userargs): - if userargs[0] != "kill": + if not userargs or userargs[0] != "kill": return False args = list(userargs) if len(args) == 3: @@ -217,7 +220,8 @@ class KillFilter(CommandFilter): return (os.path.isabs(command) and kill_command == os.path.basename(command) and - os.path.dirname(command) in os.environ['PATH'].split(':')) + os.path.dirname(command) in os.environ.get('PATH', '' + ).split(':')) class ReadFileFilter(CommandFilter): @@ -228,13 +232,7 @@ class ReadFileFilter(CommandFilter): super(ReadFileFilter, self).__init__("/bin/cat", "root", *args) def match(self, userargs): - if userargs[0] != 'cat': - return False - if userargs[1] != self.file_path: - return False - if len(userargs) != 2: - return False - return True + return (userargs == ['cat', self.file_path]) class IpFilter(CommandFilter): diff --git a/openstack/common/rootwrap/wrapper.py b/openstack/common/rootwrap/wrapper.py index df1a9f4..6bd829e 100644 --- a/openstack/common/rootwrap/wrapper.py +++ b/openstack/common/rootwrap/wrapper.py @@ -46,8 +46,10 @@ class RootwrapConfig(object): if config.has_option("DEFAULT", "exec_dirs"): self.exec_dirs = config.get("DEFAULT", "exec_dirs").split(",") else: + self.exec_dirs = [] # Use system PATH if exec_dirs is not specified - self.exec_dirs = os.environ["PATH"].split(':') + if "PATH" in os.environ: + self.exec_dirs = os.environ['PATH'].split(':') # syslog_log_facility if config.has_option("DEFAULT", "syslog_log_facility"): diff --git a/openstack/common/rpc/impl_kombu.py b/openstack/common/rpc/impl_kombu.py index ce8f2d8..d1d04e0 100644 --- a/openstack/common/rpc/impl_kombu.py +++ b/openstack/common/rpc/impl_kombu.py @@ -18,7 +18,6 @@ import functools import itertools import socket import ssl -import sys import time import uuid @@ -565,13 +564,11 @@ class Connection(object): log_info.update(params) if self.max_retries and attempt == self.max_retries: - LOG.error(_('Unable to connect to AMQP server on ' - '%(hostname)s:%(port)d after %(max_retries)d ' - 'tries: %(err_str)s') % log_info) - # NOTE(comstud): Copied from original code. There's - # really no better recourse because if this was a queue we - # need to consume on, we have no way to consume anymore. - sys.exit(1) + msg = _('Unable to connect to AMQP server on ' + '%(hostname)s:%(port)d after %(max_retries)d ' + 'tries: %(err_str)s') % log_info + LOG.error(msg) + raise rpc_common.RPCException(msg) if attempt == 1: sleep_time = self.interval_start or 1 diff --git a/openstack/common/service.py b/openstack/common/service.py index 55e23ed..36cf300 100644 --- a/openstack/common/service.py +++ b/openstack/common/service.py @@ -27,6 +27,7 @@ import sys import time import eventlet +from eventlet import event import logging as std_logging from oslo.config import cfg @@ -51,20 +52,9 @@ class Launcher(object): :returns: None """ - self._services = threadgroup.ThreadGroup() + self.services = Services() self.backdoor_port = eventlet_backdoor.initialize_if_enabled() - @staticmethod - def run_service(service): - """Start and wait for a service to finish. - - :param service: service to run and wait for. - :returns: None - - """ - service.start() - service.wait() - def launch_service(self, service): """Load and start the given service. @@ -73,7 +63,7 @@ class Launcher(object): """ service.backdoor_port = self.backdoor_port - self._services.add_thread(self.run_service, service) + self.services.add(service) def stop(self): """Stop all services which are currently running. @@ -81,7 +71,7 @@ class Launcher(object): :returns: None """ - self._services.stop() + self.services.stop() def wait(self): """Waits until all services have been stopped, and then returns. @@ -89,7 +79,7 @@ class Launcher(object): :returns: None """ - self._services.wait() + self.services.wait() class SignalExit(SystemExit): @@ -124,9 +114,9 @@ class ServiceLauncher(Launcher): except SystemExit as exc: status = exc.code finally: + self.stop() if rpc: rpc.cleanup() - self.stop() return status @@ -189,7 +179,8 @@ class ProcessLauncher(object): random.seed() launcher = Launcher() - launcher.run_service(service) + launcher.launch_service(service) + launcher.wait() def _start_child(self, wrap): if len(wrap.forktimes) > wrap.workers: @@ -313,15 +304,60 @@ class Service(object): def __init__(self, threads=1000): self.tg = threadgroup.ThreadGroup(threads) + # signal that the service is done shutting itself down: + self._done = event.Event() + def start(self): pass def stop(self): self.tg.stop() + self.tg.wait() + self._done.send() + + def wait(self): + self._done.wait() + + +class Services(object): + + def __init__(self): + self.services = [] + self.tg = threadgroup.ThreadGroup() + self.done = event.Event() + + def add(self, service): + self.services.append(service) + self.tg.add_thread(self.run_service, service, self.done) + + def stop(self): + # wait for graceful shutdown of services: + for service in self.services: + service.stop() + service.wait() + + # each service has performed cleanup, now signal that the run_service + # wrapper threads can now die: + self.done.send() + + # reap threads: + self.tg.stop() def wait(self): self.tg.wait() + @staticmethod + def run_service(service, done): + """Service start wrapper. + + :param service: service to run + :param done: event to wait on until a shutdown is triggered + :returns: None + + """ + service.start() + done.wait() + def launch(service, workers=None): if workers: diff --git a/openstack/common/timeutils.py b/openstack/common/timeutils.py index ac2441b..bd60489 100644 --- a/openstack/common/timeutils.py +++ b/openstack/common/timeutils.py @@ -23,6 +23,7 @@ import calendar import datetime import iso8601 +import six # ISO 8601 extended time format with microseconds @@ -75,14 +76,14 @@ def normalize_time(timestamp): def is_older_than(before, seconds): """Return True if before is older than seconds.""" - if isinstance(before, basestring): + if isinstance(before, six.string_types): before = parse_strtime(before).replace(tzinfo=None) return utcnow() - before > datetime.timedelta(seconds=seconds) def is_newer_than(after, seconds): """Return True if after is newer than seconds.""" - if isinstance(after, basestring): + if isinstance(after, six.string_types): after = parse_strtime(after).replace(tzinfo=None) return after - utcnow() > datetime.timedelta(seconds=seconds) diff --git a/requirements.txt b/requirements.txt index 552f84b..0a2d1c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a2.tar.gz#egg=oslo.co qpid-python six netaddr +pycrypto>=2.6 diff --git a/test-requirements.txt b/test-requirements.txt index 7ffabfe..8fbc5ab 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ coverage discover fixtures>=0.3.12 flake8==2.0 -hacking>=0.5.3,<0.6 +hacking>=0.5.6,<0.6 mock mox==0.5.3 mysql-python diff --git a/tests/unit/crypto/__init__.py b/tests/unit/crypto/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/unit/crypto/__init__.py diff --git a/tests/unit/crypto/test_utils.py b/tests/unit/crypto/test_utils.py new file mode 100644 index 0000000..3a39100 --- /dev/null +++ b/tests/unit/crypto/test_utils.py @@ -0,0 +1,186 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Unit Tests for crypto utils. +""" + +from openstack.common.crypto import utils as cryptoutils +from tests import utils as test_utils + + +class CryptoUtilsTestCase(test_utils.BaseTestCase): + + # Uses Tests from RFC5869 + def _test_HKDF(self, ikm, prk, okm, length, + salt=None, info='', hashtype='SHA256'): + hkdf = cryptoutils.HKDF(hashtype=hashtype) + + tprk = hkdf.extract(ikm, salt=salt) + self.assertEqual(prk, tprk) + + tokm = hkdf.expand(prk, info, length) + self.assertEqual(okm, tokm) + + def test_HKDF_1(self): + ikm = '\x0b' * 22 + salt = ''.join(map(lambda x: chr(x), range(0x00, 0x0d))) + info = ''.join(map(lambda x: chr(x), range(0xf0, 0xfa))) + length = 42 + + prk = ('\x07\x77\x09\x36\x2c\x2e\x32\xdf\x0d\xdc\x3f\x0d\xc4\x7b' + '\xba\x63\x90\xb6\xc7\x3b\xb5\x0f\x9c\x31\x22\xec\x84\x4a' + '\xd7\xc2\xb3\xe5') + + okm = ('\x3c\xb2\x5f\x25\xfa\xac\xd5\x7a\x90\x43\x4f\x64\xd0\x36' + '\x2f\x2a\x2d\x2d\x0a\x90\xcf\x1a\x5a\x4c\x5d\xb0\x2d\x56' + '\xec\xc4\xc5\xbf\x34\x00\x72\x08\xd5\xb8\x87\x18\x58\x65') + + self._test_HKDF(ikm, prk, okm, length, salt, info) + + def test_HKDF_2(self): + ikm = ''.join(map(lambda x: chr(x), range(0x00, 0x50))) + salt = ''.join(map(lambda x: chr(x), range(0x60, 0xb0))) + info = ''.join(map(lambda x: chr(x), range(0xb0, 0x100))) + length = 82 + + prk = ('\x06\xa6\xb8\x8c\x58\x53\x36\x1a\x06\x10\x4c\x9c\xeb\x35' + '\xb4\x5c\xef\x76\x00\x14\x90\x46\x71\x01\x4a\x19\x3f\x40' + '\xc1\x5f\xc2\x44') + + okm = ('\xb1\x1e\x39\x8d\xc8\x03\x27\xa1\xc8\xe7\xf7\x8c\x59\x6a' + '\x49\x34\x4f\x01\x2e\xda\x2d\x4e\xfa\xd8\xa0\x50\xcc\x4c' + '\x19\xaf\xa9\x7c\x59\x04\x5a\x99\xca\xc7\x82\x72\x71\xcb' + '\x41\xc6\x5e\x59\x0e\x09\xda\x32\x75\x60\x0c\x2f\x09\xb8' + '\x36\x77\x93\xa9\xac\xa3\xdb\x71\xcc\x30\xc5\x81\x79\xec' + '\x3e\x87\xc1\x4c\x01\xd5\xc1\xf3\x43\x4f\x1d\x87') + + self._test_HKDF(ikm, prk, okm, length, salt, info) + + def test_HKDF_3(self): + ikm = '\x0b' * 22 + length = 42 + + prk = ('\x19\xef\x24\xa3\x2c\x71\x7b\x16\x7f\x33\xa9\x1d\x6f\x64' + '\x8b\xdf\x96\x59\x67\x76\xaf\xdb\x63\x77\xac\x43\x4c\x1c' + '\x29\x3c\xcb\x04') + + okm = ('\x8d\xa4\xe7\x75\xa5\x63\xc1\x8f\x71\x5f\x80\x2a\x06\x3c' + '\x5a\x31\xb8\xa1\x1f\x5c\x5e\xe1\x87\x9e\xc3\x45\x4e\x5f' + '\x3c\x73\x8d\x2d\x9d\x20\x13\x95\xfa\xa4\xb6\x1a\x96\xc8') + + self._test_HKDF(ikm, prk, okm, length) + + def test_HKDF_4(self): + ikm = '\x0b' * 11 + salt = ''.join(map(lambda x: chr(x), range(0x00, 0x0d))) + info = ''.join(map(lambda x: chr(x), range(0xf0, 0xfa))) + length = 42 + + prk = ('\x9b\x6c\x18\xc4\x32\xa7\xbf\x8f\x0e\x71\xc8\xeb\x88\xf4' + '\xb3\x0b\xaa\x2b\xa2\x43') + + okm = ('\x08\x5a\x01\xea\x1b\x10\xf3\x69\x33\x06\x8b\x56\xef\xa5' + '\xad\x81\xa4\xf1\x4b\x82\x2f\x5b\x09\x15\x68\xa9\xcd\xd4' + '\xf1\x55\xfd\xa2\xc2\x2e\x42\x24\x78\xd3\x05\xf3\xf8\x96') + + self._test_HKDF(ikm, prk, okm, length, salt, info, hashtype='SHA') + + def test_HKDF_5(self): + ikm = ''.join(map(lambda x: chr(x), range(0x00, 0x50))) + salt = ''.join(map(lambda x: chr(x), range(0x60, 0xb0))) + info = ''.join(map(lambda x: chr(x), range(0xb0, 0x100))) + length = 82 + + prk = ('\x8a\xda\xe0\x9a\x2a\x30\x70\x59\x47\x8d\x30\x9b\x26\xc4' + '\x11\x5a\x22\x4c\xfa\xf6') + + okm = ('\x0b\xd7\x70\xa7\x4d\x11\x60\xf7\xc9\xf1\x2c\xd5\x91\x2a' + '\x06\xeb\xff\x6a\xdc\xae\x89\x9d\x92\x19\x1f\xe4\x30\x56' + '\x73\xba\x2f\xfe\x8f\xa3\xf1\xa4\xe5\xad\x79\xf3\xf3\x34' + '\xb3\xb2\x02\xb2\x17\x3c\x48\x6e\xa3\x7c\xe3\xd3\x97\xed' + '\x03\x4c\x7f\x9d\xfe\xb1\x5c\x5e\x92\x73\x36\xd0\x44\x1f' + '\x4c\x43\x00\xe2\xcf\xf0\xd0\x90\x0b\x52\xd3\xb4') + + self._test_HKDF(ikm, prk, okm, length, salt, info, hashtype='SHA') + + def test_HKDF_6(self): + ikm = '\x0b' * 22 + length = 42 + + prk = ('\xda\x8c\x8a\x73\xc7\xfa\x77\x28\x8e\xc6\xf5\xe7\xc2\x97' + '\x78\x6a\xa0\xd3\x2d\x01') + + okm = ('\x0a\xc1\xaf\x70\x02\xb3\xd7\x61\xd1\xe5\x52\x98\xda\x9d' + '\x05\x06\xb9\xae\x52\x05\x72\x20\xa3\x06\xe0\x7b\x6b\x87' + '\xe8\xdf\x21\xd0\xea\x00\x03\x3d\xe0\x39\x84\xd3\x49\x18') + + self._test_HKDF(ikm, prk, okm, length, hashtype='SHA') + + def test_HKDF_7(self): + ikm = '\x0c' * 22 + length = 42 + + prk = ('\x2a\xdc\xca\xda\x18\x77\x9e\x7c\x20\x77\xad\x2e\xb1\x9d' + '\x3f\x3e\x73\x13\x85\xdd') + + okm = ('\x2c\x91\x11\x72\x04\xd7\x45\xf3\x50\x0d\x63\x6a\x62\xf6' + '\x4f\x0a\xb3\xba\xe5\x48\xaa\x53\xd4\x23\xb0\xd1\xf2\x7e' + '\xbb\xa6\xf5\xe5\x67\x3a\x08\x1d\x70\xcc\xe7\xac\xfc\x48') + + self._test_HKDF(ikm, prk, okm, length, hashtype='SHA') + + def test_HKDF_8(self): + ikm = '\x0b' * 22 + prk = ('\x19\xef\x24\xa3\x2c\x71\x7b\x16\x7f\x33\xa9\x1d\x6f\x64' + '\x8b\xdf\x96\x59\x67\x76\xaf\xdb\x63\x77\xac\x43\x4c\x1c' + '\x29\x3c\xcb\x04') + + # Just testing HKDFOutputLengthTooLong is returned + try: + self._test_HKDF(ikm, prk, None, 1000000) + except cryptoutils.HKDFOutputLengthTooLong: + pass + + def test_SymmetricCrypto_encrypt_string(self): + msg = 'Plain Text' + + skc = cryptoutils.SymmetricCrypto() + key = skc.new_key(16) + cipher = skc.encrypt(key, msg) + plain = skc.decrypt(key, cipher) + self.assertEqual(msg, plain) + + def test_SymmetricCrypto_encrypt_blocks(self): + cb = 16 + et = 'AES' + + skc = cryptoutils.SymmetricCrypto(enctype=et) + key = skc.new_key(16) + msg = skc.new_key(cb * 2) + + for i in range(0, cb * 2): + cipher = skc.encrypt(key, msg[0:i], b64encode=False) + plain = skc.decrypt(key, cipher, b64decode=False) + self.assertEqual(msg[0:i], plain) + + def test_SymmetricCrypto_signing(self): + msg = 'Authenticated Message' + signature = 'KWjl6i30RMjc5PjnaccRwTPKTRCWM6sPpmGS2bxm5fQ=' + skey = 'L\xdd0\xf3\xb4\xc6\xe2p\xef\xc7\xbd\xaa\xc9eNC' + + skc = cryptoutils.SymmetricCrypto() + validate = skc.sign(skey, msg) + self.assertEqual(signature, validate) diff --git a/tests/unit/rpc/test_kombu.py b/tests/unit/rpc/test_kombu.py index 470f98a..9838652 100644 --- a/tests/unit/rpc/test_kombu.py +++ b/tests/unit/rpc/test_kombu.py @@ -41,6 +41,8 @@ from tests import utils try: import kombu + import kombu.connection + import kombu.entity from openstack.common.rpc import impl_kombu except ImportError: kombu = None @@ -715,6 +717,32 @@ class RpcKombuTestCase(amqp.BaseRpcAMQPTestCase): "args": {"value": value}}) self.assertEqual(value, result) + def test_reconnect_max_retries(self): + self.config(rabbit_hosts=[ + 'host1:1234', 'host2:5678', '[::1]:2345', + '[2001:0db8:85a3:0042:0000:8a2e:0370:7334]'], + rabbit_max_retries=2, + rabbit_retry_interval=0.1, + rabbit_retry_backoff=0.1) + + info = {'attempt': 0} + + class MyConnection(kombu.connection.BrokerConnection): + def __init__(self, *args, **params): + super(MyConnection, self).__init__(*args, **params) + info['attempt'] += 1 + + def connect(self): + if info['attempt'] < 3: + # the word timeout is important (see impl_kombu.py:486) + raise Exception('connection timeout') + super(kombu.connection.BrokerConnection, self).connect() + + self.stubs.Set(kombu.connection, 'BrokerConnection', MyConnection) + + self.assertRaises(rpc_common.RPCException, self.rpc.Connection, FLAGS) + self.assertEqual(info['attempt'], 2) + class RpcKombuHATestCase(utils.BaseTestCase): def setUp(self): @@ -765,15 +793,13 @@ class RpcKombuHATestCase(utils.BaseTestCase): ] } - import kombu.connection - class MyConnection(kombu.connection.BrokerConnection): def __init__(myself, *args, **params): super(MyConnection, myself).__init__(*args, **params) self.assertEqual(params, info['params_list'][info['attempt'] % len(info['params_list'])]) - info['attempt'] = info['attempt'] + 1 + info['attempt'] += 1 def connect(myself): if info['attempt'] < 5: @@ -790,8 +816,6 @@ class RpcKombuHATestCase(utils.BaseTestCase): def test_queue_not_declared_ha_if_ha_off(self): self.config(rabbit_ha_queues=False) - import kombu.entity - def my_declare(myself): self.assertEqual(None, (myself.queue_arguments or {}).get('x-ha-policy')) @@ -804,8 +828,6 @@ class RpcKombuHATestCase(utils.BaseTestCase): def test_queue_declared_ha_if_ha_on(self): self.config(rabbit_ha_queues=True) - import kombu.entity - def my_declare(myself): self.assertEqual('all', (myself.queue_arguments or {}).get('x-ha-policy')) diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index 0db7aaa..2f9a3de 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -24,3 +24,7 @@ class ContextTest(utils.BaseTestCase): def test_context(self): ctx = context.RequestContext() self.assertTrue(ctx) + + def test_admin_context_show_deleted_flag_default(self): + ctx = context.get_admin_context() + self.assertFalse(ctx.show_deleted) diff --git a/tests/unit/test_excutils.py b/tests/unit/test_excutils.py index b8f9b96..1386eaa 100644 --- a/tests/unit/test_excutils.py +++ b/tests/unit/test_excutils.py @@ -52,6 +52,14 @@ class SaveAndReraiseTest(utils.BaseTestCase): self.assertEqual(str(e), msg) + def test_save_and_reraise_exception_no_reraise(self): + """Test that suppressing the reraise works.""" + try: + raise Exception('foo') + except Exception: + with excutils.save_and_reraise_exception() as ctxt: + ctxt.reraise = False + class ForeverRetryUncaughtExceptionsTest(utils.BaseTestCase): diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index a65801b..e58bd7d 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -1,3 +1,19 @@ +# Copyright (c) 2011 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import cStringIO import logging import os diff --git a/tests/unit/test_notifier.py b/tests/unit/test_notifier.py index 6c3b886..891b0ec 100644 --- a/tests/unit/test_notifier.py +++ b/tests/unit/test_notifier.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import socket + from oslo.config import cfg from openstack.common import context @@ -308,3 +310,9 @@ class MultiNotifierTestCase(test_utils.BaseTestCase): notifier_api.WARN, dict(a=3)) self.assertEqual(self.notify_count, 1) + + def test_publisher_id(self): + self.assertEqual(notifier_api.publisher_id('foobar'), + 'foobar.' + socket.gethostname()) + self.assertEqual(notifier_api.publisher_id('foobar', 'baz'), + 'foobar.baz') diff --git a/tests/unit/test_rootwrap.py b/tests/unit/test_rootwrap.py index 02789ec..a649660 100644 --- a/tests/unit/test_rootwrap.py +++ b/tests/unit/test_rootwrap.py @@ -40,6 +40,32 @@ class RootwrapTestCase(utils.BaseTestCase): filters.CommandFilter("/bin/cat", "root") # Keep this one last ] + def test_CommandFilter(self): + f = filters.CommandFilter("sleep", 'root', '10') + self.assertFalse(f.match(["sleep2"])) + + # verify that any arguments are accepted + self.assertTrue(f.match(["sleep"])) + self.assertTrue(f.match(["sleep", "anything"])) + self.assertTrue(f.match(["sleep", "10"])) + f = filters.CommandFilter("sleep", 'root') + self.assertTrue(f.match(["sleep", "10"])) + + def test_empty_commandfilter(self): + f = filters.CommandFilter("sleep", "root") + self.assertFalse(f.match([])) + self.assertFalse(f.match(None)) + + def test_empty_regexpfilter(self): + f = filters.RegExpFilter("sleep", "root", "sleep") + self.assertFalse(f.match([])) + self.assertFalse(f.match(None)) + + def test_empty_invalid_regexpfilter(self): + f = filters.RegExpFilter("sleep", "root") + self.assertFalse(f.match(["anything"])) + self.assertFalse(f.match([])) + def test_RegExpFilter_match(self): usercmd = ["ls", "/root"] filtermatch = wrapper.match_filter(self.filters, usercmd) @@ -178,8 +204,9 @@ class RootwrapTestCase(utils.BaseTestCase): # Filter shouldn't be able to find binary in $PATH, so fail with fixtures.EnvironmentVariable("PATH", "/foo:/bar"): self.assertFalse(f.match(usercmd)) - pass - + # ensure that unset $PATH is not causing an exception + with fixtures.EnvironmentVariable("PATH"): + self.assertFalse(f.match(usercmd)) finally: # Terminate the "cat" process and wait for it to finish p.terminate() @@ -194,6 +221,9 @@ class RootwrapTestCase(utils.BaseTestCase): # Providing something that is not a pid should be False usercmd = ['kill', 'notapid'] self.assertFalse(f.match(usercmd)) + # no arguments should also be fine + self.assertFalse(f.match([])) + self.assertFalse(f.match(None)) def test_KillFilter_deleted_exe(self): """Makes sure deleted exe's are killed correctly.""" @@ -288,6 +318,12 @@ class RootwrapTestCase(utils.BaseTestCase): self.assertRaises(wrapper.NoFilterMatched, wrapper.match_filter, filter_list, args) + def test_ReadFileFilter_empty_args(self): + goodfn = '/good/file.name' + f = filters.ReadFileFilter(goodfn) + self.assertFalse(f.match([])) + self.assertFalse(f.match(None)) + def test_exec_dirs_search(self): # This test supposes you have /bin/cat or /usr/bin/cat locally f = filters.CommandFilter("cat", "root") @@ -314,6 +350,11 @@ class RootwrapTestCase(utils.BaseTestCase): config = wrapper.RootwrapConfig(raw) self.assertEqual(config.filters_path, ['/a', '/b']) self.assertEqual(config.exec_dirs, os.environ["PATH"].split(':')) + + with fixtures.EnvironmentVariable("PATH"): + c = wrapper.RootwrapConfig(raw) + self.assertEqual(c.exec_dirs, []) + self.assertFalse(config.use_syslog) self.assertEqual(config.syslog_log_facility, logging.handlers.SysLogHandler.LOG_SYSLOG) @@ -381,6 +422,10 @@ class PathFilterTestCase(utils.BaseTestCase): self.SYMLINK_OUTSIDE_DIR = os.path.join(tmpdir.path, gen_name()) os.symlink(os.path.join('/tmp', 'some_file'), self.SYMLINK_OUTSIDE_DIR) + def test_empty_args(self): + self.assertFalse(self.f.match([])) + self.assertFalse(self.f.match(None)) + def test_argument_pass_constraint(self): f = filters.PathFilter('/bin/chown', 'root', 'pass', 'pass') diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 0f93830..20007de 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -31,6 +31,7 @@ import socket import time import traceback +from eventlet import event from oslo.config import cfg from openstack.common import eventlet_backdoor @@ -195,6 +196,20 @@ class ServiceLauncherTest(utils.BaseTestCase): self.assertEqual(os.WEXITSTATUS(status), 0) +class _Service(service.Service): + def __init__(self): + super(_Service, self).__init__() + self.init = event.Event() + self.cleaned_up = False + + def start(self): + self.init.send() + + def stop(self): + self.cleaned_up = True + super(_Service, self).stop() + + class LauncherTest(utils.BaseTestCase): def test_backdoor_port(self): @@ -252,3 +267,15 @@ class LauncherTest(utils.BaseTestCase): svc = service.Service() self.assertRaises(eventlet_backdoor.EventletBackdoorConfigValueError, service.launch, svc) + + def test_graceful_shutdown(self): + # test that services are given a chance to clean up: + svc = _Service() + + launcher = service.launch(svc) + # wait on 'init' so we know the service had time to start: + svc.init.wait() + + launcher.stop() + self.assertTrue(svc.cleaned_up) + self.assertTrue(svc._done.ready()) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py index 42a44e8..f428c1e 100644 --- a/tools/install_venv_common.py +++ b/tools/install_venv_common.py @@ -34,12 +34,13 @@ import sys class InstallVenv(object): - def __init__(self, root, venv, pip_requires, test_requires, py_version, + def __init__(self, root, venv, requirements, + test_requirements, py_version, project): self.root = root self.venv = venv - self.pip_requires = pip_requires - self.test_requires = test_requires + self.requirements = requirements + self.test_requirements = test_requirements self.py_version = py_version self.project = project @@ -75,11 +76,13 @@ class InstallVenv(object): def get_distro(self): if (os.path.exists('/etc/fedora-release') or os.path.exists('/etc/redhat-release')): - return Fedora(self.root, self.venv, self.pip_requires, - self.test_requires, self.py_version, self.project) + return Fedora( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) else: - return Distro(self.root, self.venv, self.pip_requires, - self.test_requires, self.py_version, self.project) + return Distro( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) def check_dependencies(self): self.get_distro().install_virtualenv() @@ -98,11 +101,6 @@ class InstallVenv(object): else: self.run_command(['virtualenv', '-q', self.venv]) print('done.') - print('Installing pip in venv...', end=' ') - if not self.run_command(['tools/with_venv.sh', 'easy_install', - 'pip>1.0']).strip(): - self.die("Failed to install pip.") - print('done.') else: print("venv already exists...") pass @@ -116,20 +114,12 @@ class InstallVenv(object): print('Installing dependencies with pip (this can take a while)...') # First things first, make sure our venv has the latest pip and - # distribute. - # NOTE: we keep pip at version 1.1 since the most recent version causes - # the .venv creation to fail. See: - # https://bugs.launchpad.net/nova/+bug/1047120 - self.pip_install('pip==1.1') - self.pip_install('distribute') - - # Install greenlet by hand - just listing it in the requires file does - # not - # get it installed in the right order - self.pip_install('greenlet') - - self.pip_install('-r', self.pip_requires) - self.pip_install('-r', self.test_requires) + # setuptools. + self.pip_install('pip>=1.3') + self.pip_install('setuptools') + + self.pip_install('-r', self.requirements) + self.pip_install('-r', self.test_requirements) def post_process(self): self.get_distro().post_process() diff --git a/tools/patch_tox_venv.py b/tools/patch_tox_venv.py index a3340f2..dc9ce83 100644 --- a/tools/patch_tox_venv.py +++ b/tools/patch_tox_venv.py @@ -17,7 +17,7 @@ import os import sys -import install_venv_common as install_venv +import install_venv_common as install_venv # noqa def first_file(file_list): @@ -13,7 +13,7 @@ commands = [flake8] show-source = True -ignore = H302,H304 +ignore = H302 exclude = .venv,.tox,dist,doc,*.egg,.update-venv [testenv:pep8] |