diff options
Diffstat (limited to 'openstack')
| -rwxr-xr-x | openstack/common/config/generator.py | 46 | ||||
| -rw-r--r-- | openstack/common/context.py | 2 | ||||
| -rw-r--r-- | openstack/common/crypto/__init__.py | 0 | ||||
| -rw-r--r-- | openstack/common/crypto/utils.py | 179 | ||||
| -rw-r--r-- | openstack/common/excutils.py | 38 | ||||
| -rw-r--r-- | openstack/common/log.py | 3 | ||||
| -rw-r--r-- | openstack/common/notifier/api.py | 10 | ||||
| -rw-r--r-- | openstack/common/rootwrap/filters.py | 20 | ||||
| -rw-r--r-- | openstack/common/rootwrap/wrapper.py | 4 | ||||
| -rw-r--r-- | openstack/common/rpc/impl_kombu.py | 13 | ||||
| -rw-r--r-- | openstack/common/service.py | 70 | ||||
| -rw-r--r-- | openstack/common/timeutils.py | 5 |
12 files changed, 316 insertions, 74 deletions
diff --git a/openstack/common/config/generator.py b/openstack/common/config/generator.py index 8ebfba1..404a09a 100755 --- a/openstack/common/config/generator.py +++ b/openstack/common/config/generator.py @@ -18,8 +18,11 @@ # # @author: Zhongyue Luo, SINA Corporation. # + """Extracts OpenStack config option info from module(s).""" +from __future__ import print_function + import imp import os import re @@ -97,7 +100,7 @@ def generate(srcfiles): for group, opts in opts_by_group.items(): print_group_opts(group, opts) - print "# Total option count: %d" % OPTION_COUNT + print("# Total option count: %d" % OPTION_COUNT) def _import_module(mod_str): @@ -161,18 +164,18 @@ def _list_opts(obj): def print_group_opts(group, opts_by_module): - print "[%s]" % group - print + print("[%s]" % group) + print('') global OPTION_COUNT for mod, opts in opts_by_module: OPTION_COUNT += len(opts) - print '#' - print '# Options defined in %s' % mod - print '#' - print + print('#') + print('# Options defined in %s' % mod) + print('#') + print('') for opt in opts: _print_opt(opt) - print + print('') def _get_my_ip(): @@ -188,7 +191,12 @@ def _get_my_ip(): def _sanitize_default(s): """Set up a reasonably sensible default for pybasedir, my_ip and host.""" - if s.startswith(BASEDIR): + if s.startswith(sys.prefix): + # NOTE(jd) Don't use os.path.join, because it is likely to think the + # second part is an absolute pathname and therefore drop the first + # part. + s = os.path.normpath("/usr/" + s[len(sys.prefix):]) + elif s.startswith(BASEDIR): return s.replace(BASEDIR, '/usr/lib/python/site-packages') elif BASEDIR in s: return s.replace(BASEDIR, '') @@ -213,33 +221,33 @@ def _print_opt(opt): sys.stderr.write("%s\n" % str(err)) sys.exit(1) opt_help += ' (' + OPT_TYPES[opt_type] + ')' - print '#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)) + print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) try: if opt_default is None: - print '#%s=<None>' % opt_name + print('#%s=<None>' % opt_name) elif opt_type == STROPT: assert(isinstance(opt_default, basestring)) - print '#%s=%s' % (opt_name, _sanitize_default(opt_default)) + print('#%s=%s' % (opt_name, _sanitize_default(opt_default))) elif opt_type == BOOLOPT: assert(isinstance(opt_default, bool)) - print '#%s=%s' % (opt_name, str(opt_default).lower()) + print('#%s=%s' % (opt_name, str(opt_default).lower())) elif opt_type == INTOPT: assert(isinstance(opt_default, int) and not isinstance(opt_default, bool)) - print '#%s=%s' % (opt_name, opt_default) + print('#%s=%s' % (opt_name, opt_default)) elif opt_type == FLOATOPT: assert(isinstance(opt_default, float)) - print '#%s=%s' % (opt_name, opt_default) + print('#%s=%s' % (opt_name, opt_default)) elif opt_type == LISTOPT: assert(isinstance(opt_default, list)) - print '#%s=%s' % (opt_name, ','.join(opt_default)) + print('#%s=%s' % (opt_name, ','.join(opt_default))) elif opt_type == MULTISTROPT: assert(isinstance(opt_default, list)) if not opt_default: opt_default = [''] for default in opt_default: - print '#%s=%s' % (opt_name, default) - print + print('#%s=%s' % (opt_name, default)) + print('') except Exception: sys.stderr.write('Error in option "%s"\n' % opt_name) sys.exit(1) @@ -247,7 +255,7 @@ def _print_opt(opt): def main(): if len(sys.argv) < 2: - print "usage: %s [srcfile]...\n" % sys.argv[0] + print("usage: %s [srcfile]...\n" % sys.argv[0]) sys.exit(0) generate(sys.argv[1:]) diff --git a/openstack/common/context.py b/openstack/common/context.py index 3899c2c..81772bc 100644 --- a/openstack/common/context.py +++ b/openstack/common/context.py @@ -61,7 +61,7 @@ class RequestContext(object): 'request_id': self.request_id} -def get_admin_context(show_deleted="no"): +def get_admin_context(show_deleted=False): context = RequestContext(None, tenant=None, is_admin=True, diff --git a/openstack/common/crypto/__init__.py b/openstack/common/crypto/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/openstack/common/crypto/__init__.py diff --git a/openstack/common/crypto/utils.py b/openstack/common/crypto/utils.py new file mode 100644 index 0000000..61c1a50 --- /dev/null +++ b/openstack/common/crypto/utils.py @@ -0,0 +1,179 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 + +from Crypto.Hash import HMAC +from Crypto import Random + +from openstack.common.gettextutils import _ +from openstack.common.importutils import import_module + + +class CryptoutilsException(Exception): + """Generic Exception for Crypto utilities.""" + + message = _("An unknown error occurred in crypto utils.") + + +class CipherBlockLengthTooBig(CryptoutilsException): + """The block size is too big.""" + + def __init__(self, requested, permitted): + msg = _("Block size of %(given)d is too big, max = %(maximum)d") + message = msg % {'given': requested, 'maximum': permitted} + super(CryptoutilsException, self).__init__(message) + + +class HKDFOutputLengthTooLong(CryptoutilsException): + """The amount of Key Material asked is too much.""" + + def __init__(self, requested, permitted): + msg = _("Length of %(given)d is too long, max = %(maximum)d") + message = msg % {'given': requested, 'maximum': permitted} + super(CryptoutilsException, self).__init__(message) + + +class HKDF(object): + """An HMAC-based Key Derivation Function implementation (RFC5869) + + This class creates an object that allows to use HKDF to derive keys. + """ + + def __init__(self, hashtype='SHA256'): + self.hashfn = import_module('Crypto.Hash.' + hashtype) + self.max_okm_length = 255 * self.hashfn.digest_size + + def extract(self, ikm, salt=None): + """An extract function that can be used to derive a robust key given + weak Input Key Material (IKM) which could be a password. + Returns a pseudorandom key (of HashLen octets) + + :param ikm: input keying material (ex a password) + :param salt: optional salt value (a non-secret random value) + """ + if salt is None: + salt = '\x00' * self.hashfn.digest_size + + return HMAC.new(salt, ikm, self.hashfn).digest() + + def expand(self, prk, info, length): + """An expand function that will return arbitrary length output that can + be used as keys. + Returns a buffer usable as key material. + + :param prk: a pseudorandom key of at least HashLen octets + :param info: optional string (can be a zero-length string) + :param length: length of output keying material (<= 255 * HashLen) + """ + if length > self.max_okm_length: + raise HKDFOutputLengthTooLong(length, self.max_okm_length) + + N = (length + self.hashfn.digest_size - 1) / self.hashfn.digest_size + + okm = "" + tmp = "" + for block in range(1, N + 1): + tmp = HMAC.new(prk, tmp + info + chr(block), self.hashfn).digest() + okm += tmp + + return okm[:length] + + +MAX_CB_SIZE = 256 + + +class SymmetricCrypto(object): + """Symmetric Key Crypto object. + + This class creates a Symmetric Key Crypto object that can be used + to encrypt, decrypt, or sign arbitrary data. + + :param enctype: Encryption Cipher name (default: AES) + :param hashtype: Hash/HMAC type name (default: SHA256) + """ + + def __init__(self, enctype='AES', hashtype='SHA256'): + self.cipher = import_module('Crypto.Cipher.' + enctype) + self.hashfn = import_module('Crypto.Hash.' + hashtype) + + def new_key(self, size): + return Random.new().read(size) + + def encrypt(self, key, msg, b64encode=True): + """Encrypt the provided msg and returns the cyphertext optionally + base64 encoded. + + Uses AES-128-CBC with a Random IV by default. + + The plaintext is padded to reach blocksize length. + The last byte of the block is the length of the padding. + The length of the padding does not include the length byte itself. + + :param key: The Encryption key. + :param msg: the plain text. + + :returns encblock: a block of encrypted data. + """ + iv = Random.new().read(self.cipher.block_size) + cipher = self.cipher.new(key, self.cipher.MODE_CBC, iv) + + # CBC mode requires a fixed block size. Append padding and length of + # padding. + if self.cipher.block_size > MAX_CB_SIZE: + raise CipherBlockLengthTooBig(self.cipher.block_size, MAX_CB_SIZE) + r = len(msg) % self.cipher.block_size + padlen = self.cipher.block_size - r - 1 + msg += '\x00' * padlen + msg += chr(padlen) + + enc = iv + cipher.encrypt(msg) + if b64encode: + enc = base64.b64encode(enc) + return enc + + def decrypt(self, key, msg, b64decode=True): + """Decrypts the provided ciphertext, optionally base 64 encoded, and + returns the plaintext message, after padding is removed. + + Uses AES-128-CBC with an IV by default. + + :param key: The Encryption key. + :param msg: the ciphetext, the first block is the IV + """ + if b64decode: + msg = base64.b64decode(msg) + iv = msg[:self.cipher.block_size] + cipher = self.cipher.new(key, self.cipher.MODE_CBC, iv) + + padded = cipher.decrypt(msg[self.cipher.block_size:]) + l = ord(padded[-1]) + 1 + plain = padded[:-l] + return plain + + def sign(self, key, msg, b64encode=True): + """Signs a message string and returns a base64 encoded signature. + + Uses HMAC-SHA-256 by default. + + :param key: The Signing key. + :param msg: the message to sign. + """ + h = HMAC.new(key, msg, self.hashfn) + out = h.digest() + if b64encode: + out = base64.b64encode(out) + return out diff --git a/openstack/common/excutils.py b/openstack/common/excutils.py index d40d46c..336e147 100644 --- a/openstack/common/excutils.py +++ b/openstack/common/excutils.py @@ -19,7 +19,6 @@ Exception related utilities. """ -import contextlib import logging import sys import time @@ -28,8 +27,7 @@ import traceback from openstack.common.gettextutils import _ -@contextlib.contextmanager -def save_and_reraise_exception(): +class save_and_reraise_exception(object): """Save current exception, run some code and then re-raise. In some cases the exception context can be cleared, resulting in None @@ -41,15 +39,33 @@ def save_and_reraise_exception(): To work around this, we save the exception state, run handler code, and then re-raise the original exception. If another exception occurs, the saved exception is logged and the new exception is re-raised. - """ - type_, value, tb = sys.exc_info() - try: - yield + + In some cases the caller may not want to re-raise the exception, and + for those circumstances this context provides a reraise flag that + can be used to suppress the exception. For example: + except Exception: - logging.error(_('Original exception being dropped: %s'), - traceback.format_exception(type_, value, tb)) - raise - raise type_, value, tb + with save_and_reraise_exception() as ctxt: + decide_if_need_reraise() + if not should_be_reraised: + ctxt.reraise = False + """ + def __init__(self): + self.reraise = True + + def __enter__(self): + self.type_, self.value, self.tb, = sys.exc_info() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + logging.error(_('Original exception being dropped: %s'), + traceback.format_exception(self.type_, + self.value, + self.tb)) + return False + if self.reraise: + raise self.type_, self.value, self.tb def forever_retry_uncaught_exceptions(infunc): diff --git a/openstack/common/log.py b/openstack/common/log.py index 8097b23..0447a52 100644 --- a/openstack/common/log.py +++ b/openstack/common/log.py @@ -74,7 +74,8 @@ logging_cli_opts = [ cfg.StrOpt('log-format', default=None, metavar='FORMAT', - help='A logging.Formatter log message format string which may ' + help='DEPRECATED. ' + 'A logging.Formatter log message format string which may ' 'use any of the available logging.LogRecord attributes. ' 'This option is deprecated. Please use ' 'logging_context_format_string and ' diff --git a/openstack/common/notifier/api.py b/openstack/common/notifier/api.py index 7c4dbd1..dc4f578 100644 --- a/openstack/common/notifier/api.py +++ b/openstack/common/notifier/api.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import socket import uuid from oslo.config import cfg @@ -35,7 +36,7 @@ notifier_opts = [ default='INFO', help='Default notification level for outgoing notifications'), cfg.StrOpt('default_publisher_id', - default='$host', + default=None, help='Default publisher_id for outgoing notifications'), ] @@ -74,7 +75,7 @@ def notify_decorator(name, fn): ctxt = context.get_context_from_function_and_args(fn, args, kwarg) notify(ctxt, - CONF.default_publisher_id, + CONF.default_publisher_id or socket.gethostname(), name, CONF.default_notification_level, body) @@ -84,7 +85,10 @@ def notify_decorator(name, fn): def publisher_id(service, host=None): if not host: - host = CONF.host + try: + host = CONF.host + except AttributeError: + host = CONF.default_publisher_id or socket.gethostname() return "%s.%s" % (service, host) diff --git a/openstack/common/rootwrap/filters.py b/openstack/common/rootwrap/filters.py index dfec412..660434a 100644 --- a/openstack/common/rootwrap/filters.py +++ b/openstack/common/rootwrap/filters.py @@ -47,7 +47,7 @@ class CommandFilter(object): def match(self, userargs): """Only check that the first argument (command) matches exec_path.""" - return os.path.basename(self.exec_path) == userargs[0] + return userargs and os.path.basename(self.exec_path) == userargs[0] def get_command(self, userargs, exec_dirs=[]): """Returns command to execute (with sudo -u if run_as != root).""" @@ -67,7 +67,7 @@ class RegExpFilter(CommandFilter): def match(self, userargs): # Early skip if command or number of args don't match - if (len(self.args) != len(userargs)): + if (not userargs or len(self.args) != len(userargs)): # DENY: argument numbers don't match return False # Compare each arg (anchoring pattern explicitly at end of string) @@ -101,6 +101,9 @@ class PathFilter(CommandFilter): """ def match(self, userargs): + if not userargs or len(userargs) < 2: + return False + command, arguments = userargs[0], userargs[1:] equal_args_num = len(self.args) == len(arguments) @@ -178,7 +181,7 @@ class KillFilter(CommandFilter): super(KillFilter, self).__init__("/bin/kill", *args) def match(self, userargs): - if userargs[0] != "kill": + if not userargs or userargs[0] != "kill": return False args = list(userargs) if len(args) == 3: @@ -217,7 +220,8 @@ class KillFilter(CommandFilter): return (os.path.isabs(command) and kill_command == os.path.basename(command) and - os.path.dirname(command) in os.environ['PATH'].split(':')) + os.path.dirname(command) in os.environ.get('PATH', '' + ).split(':')) class ReadFileFilter(CommandFilter): @@ -228,13 +232,7 @@ class ReadFileFilter(CommandFilter): super(ReadFileFilter, self).__init__("/bin/cat", "root", *args) def match(self, userargs): - if userargs[0] != 'cat': - return False - if userargs[1] != self.file_path: - return False - if len(userargs) != 2: - return False - return True + return (userargs == ['cat', self.file_path]) class IpFilter(CommandFilter): diff --git a/openstack/common/rootwrap/wrapper.py b/openstack/common/rootwrap/wrapper.py index df1a9f4..6bd829e 100644 --- a/openstack/common/rootwrap/wrapper.py +++ b/openstack/common/rootwrap/wrapper.py @@ -46,8 +46,10 @@ class RootwrapConfig(object): if config.has_option("DEFAULT", "exec_dirs"): self.exec_dirs = config.get("DEFAULT", "exec_dirs").split(",") else: + self.exec_dirs = [] # Use system PATH if exec_dirs is not specified - self.exec_dirs = os.environ["PATH"].split(':') + if "PATH" in os.environ: + self.exec_dirs = os.environ['PATH'].split(':') # syslog_log_facility if config.has_option("DEFAULT", "syslog_log_facility"): diff --git a/openstack/common/rpc/impl_kombu.py b/openstack/common/rpc/impl_kombu.py index ce8f2d8..d1d04e0 100644 --- a/openstack/common/rpc/impl_kombu.py +++ b/openstack/common/rpc/impl_kombu.py @@ -18,7 +18,6 @@ import functools import itertools import socket import ssl -import sys import time import uuid @@ -565,13 +564,11 @@ class Connection(object): log_info.update(params) if self.max_retries and attempt == self.max_retries: - LOG.error(_('Unable to connect to AMQP server on ' - '%(hostname)s:%(port)d after %(max_retries)d ' - 'tries: %(err_str)s') % log_info) - # NOTE(comstud): Copied from original code. There's - # really no better recourse because if this was a queue we - # need to consume on, we have no way to consume anymore. - sys.exit(1) + msg = _('Unable to connect to AMQP server on ' + '%(hostname)s:%(port)d after %(max_retries)d ' + 'tries: %(err_str)s') % log_info + LOG.error(msg) + raise rpc_common.RPCException(msg) if attempt == 1: sleep_time = self.interval_start or 1 diff --git a/openstack/common/service.py b/openstack/common/service.py index 55e23ed..36cf300 100644 --- a/openstack/common/service.py +++ b/openstack/common/service.py @@ -27,6 +27,7 @@ import sys import time import eventlet +from eventlet import event import logging as std_logging from oslo.config import cfg @@ -51,20 +52,9 @@ class Launcher(object): :returns: None """ - self._services = threadgroup.ThreadGroup() + self.services = Services() self.backdoor_port = eventlet_backdoor.initialize_if_enabled() - @staticmethod - def run_service(service): - """Start and wait for a service to finish. - - :param service: service to run and wait for. - :returns: None - - """ - service.start() - service.wait() - def launch_service(self, service): """Load and start the given service. @@ -73,7 +63,7 @@ class Launcher(object): """ service.backdoor_port = self.backdoor_port - self._services.add_thread(self.run_service, service) + self.services.add(service) def stop(self): """Stop all services which are currently running. @@ -81,7 +71,7 @@ class Launcher(object): :returns: None """ - self._services.stop() + self.services.stop() def wait(self): """Waits until all services have been stopped, and then returns. @@ -89,7 +79,7 @@ class Launcher(object): :returns: None """ - self._services.wait() + self.services.wait() class SignalExit(SystemExit): @@ -124,9 +114,9 @@ class ServiceLauncher(Launcher): except SystemExit as exc: status = exc.code finally: + self.stop() if rpc: rpc.cleanup() - self.stop() return status @@ -189,7 +179,8 @@ class ProcessLauncher(object): random.seed() launcher = Launcher() - launcher.run_service(service) + launcher.launch_service(service) + launcher.wait() def _start_child(self, wrap): if len(wrap.forktimes) > wrap.workers: @@ -313,15 +304,60 @@ class Service(object): def __init__(self, threads=1000): self.tg = threadgroup.ThreadGroup(threads) + # signal that the service is done shutting itself down: + self._done = event.Event() + def start(self): pass def stop(self): self.tg.stop() + self.tg.wait() + self._done.send() + + def wait(self): + self._done.wait() + + +class Services(object): + + def __init__(self): + self.services = [] + self.tg = threadgroup.ThreadGroup() + self.done = event.Event() + + def add(self, service): + self.services.append(service) + self.tg.add_thread(self.run_service, service, self.done) + + def stop(self): + # wait for graceful shutdown of services: + for service in self.services: + service.stop() + service.wait() + + # each service has performed cleanup, now signal that the run_service + # wrapper threads can now die: + self.done.send() + + # reap threads: + self.tg.stop() def wait(self): self.tg.wait() + @staticmethod + def run_service(service, done): + """Service start wrapper. + + :param service: service to run + :param done: event to wait on until a shutdown is triggered + :returns: None + + """ + service.start() + done.wait() + def launch(service, workers=None): if workers: diff --git a/openstack/common/timeutils.py b/openstack/common/timeutils.py index ac2441b..bd60489 100644 --- a/openstack/common/timeutils.py +++ b/openstack/common/timeutils.py @@ -23,6 +23,7 @@ import calendar import datetime import iso8601 +import six # ISO 8601 extended time format with microseconds @@ -75,14 +76,14 @@ def normalize_time(timestamp): def is_older_than(before, seconds): """Return True if before is older than seconds.""" - if isinstance(before, basestring): + if isinstance(before, six.string_types): before = parse_strtime(before).replace(tzinfo=None) return utcnow() - before > datetime.timedelta(seconds=seconds) def is_newer_than(after, seconds): """Return True if after is newer than seconds.""" - if isinstance(after, basestring): + if isinstance(after, six.string_types): after = parse_strtime(after).replace(tzinfo=None) return after - utcnow() > datetime.timedelta(seconds=seconds) |