diff options
Diffstat (limited to 'openstack')
| -rw-r--r-- | openstack/common/cfg.py | 19 | ||||
| -rw-r--r-- | openstack/common/eventlet_backdoor.py | 78 | ||||
| -rw-r--r-- | openstack/common/log.py | 21 | ||||
| -rw-r--r-- | openstack/common/notifier/api.py | 60 | ||||
| -rw-r--r-- | openstack/common/notifier/list_notifier.py | 118 | ||||
| -rw-r--r-- | openstack/common/plugin/plugin.py | 1 | ||||
| -rw-r--r-- | openstack/common/plugin/pluginmanager.py | 21 | ||||
| -rw-r--r-- | openstack/common/policy.py | 152 | ||||
| -rw-r--r-- | openstack/common/rpc/common.py | 6 | ||||
| -rw-r--r-- | openstack/common/setup.py | 5 | ||||
| -rw-r--r-- | openstack/common/timeutils.py | 18 |
11 files changed, 288 insertions, 211 deletions
diff --git a/openstack/common/cfg.py b/openstack/common/cfg.py index 728ecdc..b42c30e 100644 --- a/openstack/common/cfg.py +++ b/openstack/common/cfg.py @@ -1156,6 +1156,25 @@ class ConfigOpts(collections.Mapping): for opt in opts: self.unregister_opt(opt, group, clear_cache=False) + def import_opt(self, name, module_str, group=None): + """Import an option definition from a module. + + Import a module and check that a given option is registered. + + This is intended for use with global configuration objects + like cfg.CONF where modules commonly register options with + CONF at module load time. If one module requires an option + defined by another module it can use this method to explicitly + declare the dependency. + + :param name: the name/dest of the opt + :param module_str: the name of a module to import + :param group: an option OptGroup object or group name + :raises: NoSuchOptError, NoSuchGroupError + """ + __import__(module_str) + self._get_opt_info(name, group) + @__clear_cache def set_override(self, name, override, group=None): """Override an opt value. diff --git a/openstack/common/eventlet_backdoor.py b/openstack/common/eventlet_backdoor.py new file mode 100644 index 0000000..9f1404a --- /dev/null +++ b/openstack/common/eventlet_backdoor.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 Openstack, LLC. +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import gc +import pprint +import sys +import traceback + +import eventlet +import eventlet.backdoor +import greenlet + +from openstack.common import cfg + +eventlet_backdoor_opts = [ + cfg.IntOpt('backdoor_port', + default=None, + help='port for eventlet backdoor to listen') + ] + +CONF = cfg.CONF +CONF.register_opts(eventlet_backdoor_opts) + + +def _dont_use_this(): + print "Don't use this, just disconnect instead" + + +def _find_objects(t): + return filter(lambda o: isinstance(o, t), gc.get_objects()) + + +def _print_greenthreads(): + for i, gt in enumerate(find_objects(greenlet.greenlet)): + print i, gt + traceback.print_stack(gt.gr_frame) + print + + +def initialize_if_enabled(): + backdoor_locals = { + 'exit': _dont_use_this, # So we don't exit the entire process + 'quit': _dont_use_this, # So we don't exit the entire process + 'fo': _find_objects, + 'pgt': _print_greenthreads, + } + + if CONF.backdoor_port is None: + return + + # NOTE(johannes): The standard sys.displayhook will print the value of + # the last expression and set it to __builtin__._, which overwrites + # the __builtin__._ that gettext sets. Let's switch to using pprint + # since it won't interact poorly with gettext, and it's easier to + # read the output too. + def displayhook(val): + if val is not None: + pprint.pprint(val) + sys.displayhook = displayhook + + eventlet.spawn(eventlet.backdoor.backdoor_server, + eventlet.listen(('localhost', CONF.backdoor_port)), + locals=backdoor_locals) diff --git a/openstack/common/log.py b/openstack/common/log.py index 6d1d9f9..49e6107 100644 --- a/openstack/common/log.py +++ b/openstack/common/log.py @@ -247,26 +247,27 @@ class JSONFormatter(logging.Formatter): class PublishErrorsHandler(logging.Handler): def emit(self, record): - if 'list_notifier_drivers' in CONF: - if ('openstack.common.notifier.log_notifier' in - CONF.list_notifier_drivers): - return + if ('openstack.common.notifier.log_notifier' in + CONF.notification_driver): + return notifier.api.notify(None, 'error.publisher', 'error_notification', notifier.api.ERROR, dict(error=record.msg)) -def handle_exception(type, value, tb): - extra = {} - if CONF.verbose: - extra['exc_info'] = (type, value, tb) - getLogger().critical(str(value), **extra) +def _create_logging_excepthook(product_name): + def logging_excepthook(type, value, tb): + extra = {} + if CONF.verbose: + extra['exc_info'] = (type, value, tb) + getLogger(product_name).critical(str(value), **extra) + return logging_excepthook def setup(product_name): """Setup logging.""" - sys.excepthook = handle_exception + sys.excepthook = _create_logging_excepthook(product_name) if CONF.log_config: try: diff --git a/openstack/common/notifier/api.py b/openstack/common/notifier/api.py index e699620..f36469c 100644 --- a/openstack/common/notifier/api.py +++ b/openstack/common/notifier/api.py @@ -28,9 +28,10 @@ from openstack.common import timeutils LOG = logging.getLogger(__name__) notifier_opts = [ - cfg.StrOpt('notification_driver', - default='openstack.common.notifier.no_op_notifier', - help='Default driver for sending notifications'), + cfg.MultiStrOpt('notification_driver', + default=[], + deprecated_name='list_notifier_drivers', + help='Driver or drivers to handle sending notifications'), cfg.StrOpt('default_notification_level', default='INFO', help='Default notification level for outgoing notifications'), @@ -127,16 +128,55 @@ def notify(context, publisher_id, event_type, priority, payload): # Ensure everything is JSON serializable. payload = jsonutils.to_primitive(payload, convert_instances=True) - driver = importutils.import_module(CONF.notification_driver) msg = dict(message_id=str(uuid.uuid4()), publisher_id=publisher_id, event_type=event_type, priority=priority, payload=payload, timestamp=str(timeutils.utcnow())) - try: - driver.notify(context, msg) - except Exception, e: - LOG.exception(_("Problem '%(e)s' attempting to " - "send to notification system. Payload=%(payload)s") % - locals()) + + for driver in _get_drivers(): + try: + driver.notify(context, msg) + except Exception, e: + LOG.exception(_("Problem '%(e)s' attempting to " + "send to notification system. Payload=%(payload)s") % + locals()) + + +_drivers = None + + +def _get_drivers(): + """Instantiate, cache, and return drivers based on the CONF.""" + global _drivers + if _drivers is None: + _drivers = {} + for notification_driver in CONF.notification_driver: + add_driver(notification_driver) + + return _drivers.values() + + +def add_driver(notification_driver): + """Add a notification driver at runtime.""" + # Make sure the driver list is initialized. + _get_drivers() + if isinstance(notification_driver, basestring): + # Load and add + try: + driver = importutils.import_module(notification_driver) + _drivers[notification_driver] = driver + except ImportError as e: + LOG.exception(_("Failed to load notifier %s. " + "These notifications will not be sent.") % + notification_driver) + else: + # Driver is already loaded; just add the object. + _drivers[notification_driver] = notification_driver + + +def _reset_drivers(): + """Used by unit tests to reset the drivers.""" + global _drivers + _drivers = None diff --git a/openstack/common/notifier/list_notifier.py b/openstack/common/notifier/list_notifier.py deleted file mode 100644 index 15ae470..0000000 --- a/openstack/common/notifier/list_notifier.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from openstack.common import cfg -from openstack.common.gettextutils import _ -from openstack.common import importutils -from openstack.common import log as logging - - -list_notifier_drivers_opt = cfg.MultiStrOpt( - 'list_notifier_drivers', - default=['openstack.common.notifier.no_op_notifier'], - help='List of drivers to send notifications') - -CONF = cfg.CONF -CONF.register_opt(list_notifier_drivers_opt) - -LOG = logging.getLogger(__name__) - -drivers = None - - -class ImportFailureNotifier(object): - """Noisily re-raises some exception over-and-over when notify is called.""" - - def __init__(self, exception): - self.exception = exception - - def notify(self, context, message): - raise self.exception - - -def _get_drivers(): - """Instantiates and returns drivers based on the flag values.""" - global drivers - if drivers is None: - drivers = [] - for notification_driver in CONF.list_notifier_drivers: - try: - drivers.append(importutils.import_module(notification_driver)) - except ImportError as e: - drivers.append(ImportFailureNotifier(e)) - return drivers - - -def add_driver(notification_driver): - """Add a notification driver at runtime.""" - # Make sure the driver list is initialized. - _get_drivers() - if isinstance(notification_driver, basestring): - # Load and add - try: - drivers.append(importutils.import_module(notification_driver)) - except ImportError as e: - drivers.append(ImportFailureNotifier(e)) - else: - # Driver is already loaded; just add the object. - drivers.append(notification_driver) - - -def _object_name(obj): - name = [] - if hasattr(obj, '__module__'): - name.append(obj.__module__) - if hasattr(obj, '__name__'): - name.append(obj.__name__) - else: - name.append(obj.__class__.__name__) - return '.'.join(name) - - -def remove_driver(notification_driver): - """Remove a notification driver at runtime.""" - # Make sure the driver list is initialized. - _get_drivers() - removed = False - if notification_driver in drivers: - # We're removing an object. Easy. - drivers.remove(notification_driver) - removed = True - else: - # We're removing a driver by name. Search for it. - for driver in drivers: - if _object_name(driver) == notification_driver: - drivers.remove(driver) - removed = True - - if not removed: - raise ValueError("Cannot remove; %s is not in list" % - notification_driver) - - -def notify(context, message): - """Passes notification to multiple notifiers in a list.""" - for driver in _get_drivers(): - try: - driver.notify(context, message) - except Exception as e: - LOG.exception(_("Problem '%(e)s' attempting to send to " - "notification driver %(driver)s."), locals()) - - -def _reset_drivers(): - """Used by unit tests to reset the drivers.""" - global drivers - drivers = None diff --git a/openstack/common/plugin/plugin.py b/openstack/common/plugin/plugin.py index 9f06342..dc41f3d 100644 --- a/openstack/common/plugin/plugin.py +++ b/openstack/common/plugin/plugin.py @@ -14,7 +14,6 @@ # under the License. from openstack.common import log as logging -from openstack.common.notifier import list_notifier LOG = logging.getLogger(__name__) diff --git a/openstack/common/plugin/pluginmanager.py b/openstack/common/plugin/pluginmanager.py index d9b6bc3..b10ce46 100644 --- a/openstack/common/plugin/pluginmanager.py +++ b/openstack/common/plugin/pluginmanager.py @@ -19,7 +19,7 @@ import pkg_resources from openstack.common import cfg from openstack.common import log as logging -from openstack.common.notifier import list_notifier +from openstack.common.notifier import api as notifier_api CONF = cfg.CONF @@ -53,17 +53,6 @@ class PluginManager(object): self._service_name = service_name self.plugins = [] - def _force_use_list_notifier(self): - if (CONF.notification_driver != - 'openstack.common.notifier.list_notifier'): - if not hasattr(CONF, "list_notifier_drivers"): - CONF.list_notifier_drivers = [] - old_notifier = CONF.notification_driver - drvstring = 'openstack.common.notifier.list_notifier' - CONF.notification_driver = drvstring - if old_notifier: - list_notifier.add_driver(old_notifier) - def load_plugins(self): self.plugins = [] @@ -77,16 +66,10 @@ class PluginManager(object): LOG.error(_("Failed to load plugin %(plug)s: %(exc)s") % {'plug': entrypoint, 'exc': exc}) - # See if we need to turn on the list notifier - for plugin in self.plugins: - if plugin.notifiers: - self._force_use_list_notifier() - break - # Register individual notifiers. for plugin in self.plugins: for notifier in plugin.notifiers: - list_notifier.add_driver(notifier) + notifier_api.add_driver(notifier) def plugin_extension_factory(self, ext_mgr): for plugin in self.plugins: diff --git a/openstack/common/policy.py b/openstack/common/policy.py index 7d6eff9..fd1d7d3 100644 --- a/openstack/common/policy.py +++ b/openstack/common/policy.py @@ -131,6 +131,13 @@ def enforce(match_list, target_dict, credentials_dict, exc=None, class Brain(object): """Implements policy checking.""" + + _checks = {} + + @classmethod + def _register(cls, name, func): + cls._checks[name] = func + @classmethod def load_json(cls, data, default_rule=None): """Init a brain using json instead of a rules dictionary.""" @@ -138,6 +145,11 @@ class Brain(object): return cls(rules=rules_dict, default_rule=default_rule) def __init__(self, rules=None, default_rule=None): + if self.__class__ != Brain: + LOG.warning(_("Inheritance-based rules are deprecated; use " + "the default brain instead of %s.") % + self.__class__.__name__) + self.rules = rules or {} self.default_rule = default_rule @@ -151,15 +163,24 @@ class Brain(object): LOG.exception(_("Failed to understand rule %(match)r") % locals()) # If the rule is invalid, fail closed return False + + func = None try: - f = getattr(self, '_check_%s' % match_kind) + old_func = getattr(self, '_check_%s' % match_kind) except AttributeError: - if not self._check_generic(match, target_dict, cred_dict): - return False + func = self._checks.get(match_kind, self._checks.get(None, None)) else: - if not f(match_value, target_dict, cred_dict): - return False - return True + LOG.warning(_("Inheritance-based rules are deprecated; update " + "_check_%s") % match_kind) + func = (lambda brain, kind, value, target, cred: + old_func(value, target, cred)) + + if not func: + LOG.error(_("No handler for matches of kind %s") % match_kind) + # Fail closed + return False + + return func(self, match_kind, match_value, target_dict, cred_dict) def check(self, match_list, target_dict, cred_dict): """Checks authorization of some rules against credentials. @@ -183,58 +204,97 @@ class Brain(object): return True return False - def _check_rule(self, match, target_dict, cred_dict): - """Recursively checks credentials based on the brains rules.""" - try: - new_match_list = self.rules[match] - except KeyError: - if self.default_rule and match != self.default_rule: - new_match_list = ('rule:%s' % self.default_rule,) - else: - return False - return self.check(new_match_list, target_dict, cred_dict) +class HttpBrain(Brain): + """A brain that can check external urls for policy. - def _check_role(self, match, target_dict, cred_dict): - """Check that there is a matching role in the cred dict.""" - return match.lower() in [x.lower() for x in cred_dict['roles']] + Posts json blobs for target and credentials. - def _check_generic(self, match, target_dict, cred_dict): - """Check an individual match. + Note that this brain is deprecated; the http check is registered + by default. + """ - Matches look like: + pass - tenant:%(tenant_id)s - role:compute:admin - """ +def register(name, func=None): + """ + Register a function as a policy check. + + :param name: Gives the name of the check type, e.g., 'rule', + 'role', etc. If name is None, a default function + will be registered. + :param func: If given, provides the function to register. If not + given, returns a function taking one argument to + specify the function to register, allowing use as a + decorator. + """ - # TODO(termie): do dict inspection via dot syntax - match = match % target_dict - key, value = match.split(':', 1) - if key in cred_dict: - return value == cred_dict[key] - return False + # Perform the actual decoration by registering the function. + # Returns the function for compliance with the decorator + # interface. + def decorator(func): + # Register the function + Brain._register(name, func) + return func + + # If the function is given, do the registration + if func: + return decorator(func) + + return decorator + + +@register("rule") +def _check_rule(brain, match_kind, match, target_dict, cred_dict): + """Recursively checks credentials based on the brains rules.""" + try: + new_match_list = brain.rules[match] + except KeyError: + if brain.default_rule and match != brain.default_rule: + new_match_list = ('rule:%s' % brain.default_rule,) + else: + return False + return brain.check(new_match_list, target_dict, cred_dict) -class HttpBrain(Brain): - """A brain that can check external urls for policy. - Posts json blobs for target and credentials. +@register("role") +def _check_role(brain, match_kind, match, target_dict, cred_dict): + """Check that there is a matching role in the cred dict.""" + return match.lower() in [x.lower() for x in cred_dict['roles']] + + +@register('http') +def _check_http(brain, match_kind, match, target_dict, cred_dict): + """Check http: rules by calling to a remote server. + + This example implementation simply verifies that the response is + exactly 'True'. A custom brain using response codes could easily + be implemented. """ + url = 'http:' + (match % target_dict) + data = {'target': jsonutils.dumps(target_dict), + 'credentials': jsonutils.dumps(cred_dict)} + post_data = urllib.urlencode(data) + f = urllib2.urlopen(url, post_data) + return f.read() == "True" - def _check_http(self, match, target_dict, cred_dict): - """Check http: rules by calling to a remote server. - This example implementation simply verifies that the response is - exactly 'True'. A custom brain using response codes could easily - be implemented. +@register(None) +def _check_generic(brain, match_kind, match, target_dict, cred_dict): + """Check an individual match. - """ - url = match % target_dict - data = {'target': jsonutils.dumps(target_dict), - 'credentials': jsonutils.dumps(cred_dict)} - post_data = urllib.urlencode(data) - f = urllib2.urlopen(url, post_data) - return f.read() == "True" + Matches look like: + + tenant:%(tenant_id)s + role:compute:admin + + """ + + # TODO(termie): do dict inspection via dot syntax + match = match % target_dict + if match_kind in cred_dict: + return match == cred_dict[match_kind] + return False diff --git a/openstack/common/rpc/common.py b/openstack/common/rpc/common.py index 2e15c3e..d2a1cf1 100644 --- a/openstack/common/rpc/common.py +++ b/openstack/common/rpc/common.py @@ -108,7 +108,7 @@ class Connection(object): """ raise NotImplementedError() - def create_consumer(self, conf, topic, proxy, fanout=False): + def create_consumer(self, topic, proxy, fanout=False): """Create a consumer on this connection. A consumer is associated with a message queue on the backend message @@ -117,7 +117,6 @@ class Connection(object): off of the queue will determine which method gets called on the proxy object. - :param conf: An openstack.common.cfg configuration object. :param topic: This is a name associated with what to consume from. Multiple instances of a service may consume from the same topic. For example, all instances of nova-compute consume @@ -133,7 +132,7 @@ class Connection(object): """ raise NotImplementedError() - def create_worker(self, conf, topic, proxy, pool_name): + def create_worker(self, topic, proxy, pool_name): """Create a worker on this connection. A worker is like a regular consumer of messages directed to a @@ -143,7 +142,6 @@ class Connection(object): be asked to process it. Load is distributed across the members of the pool in round-robin fashion. - :param conf: An openstack.common.cfg configuration object. :param topic: This is a name associated with what to consume from. Multiple instances of a service may consume from the same topic. diff --git a/openstack/common/setup.py b/openstack/common/setup.py index 59f255d..628f5e3 100644 --- a/openstack/common/setup.py +++ b/openstack/common/setup.py @@ -52,7 +52,6 @@ def canonicalize_emails(changelog, mapping): # Get requirements from the first file that exists def get_reqs_from_files(requirements_files): - reqs_in = [] for requirements_file in requirements_files: if os.path.exists(requirements_file): return open(requirements_file, 'r').read().split('\n') @@ -144,8 +143,8 @@ def _get_git_next_version_suffix(branch_name): # where the bit after the last . is the short sha, and the bit between # the last and second to last is the revno count (revno, sha) = post_version.split(".")[-2:] - first_half = "%(milestonever)s~%(datestamp)s" % locals() - second_half = "%(revno_prefix)s%(revno)s.%(sha)s" % locals() + first_half = "%s~%s" % (milestonever, datestamp) + second_half = "%s%s.%s" % (revno_prefix, revno, sha) return ".".join((first_half, second_half)) diff --git a/openstack/common/timeutils.py b/openstack/common/timeutils.py index 4416a3b..ae300e4 100644 --- a/openstack/common/timeutils.py +++ b/openstack/common/timeutils.py @@ -106,3 +106,21 @@ def advance_time_seconds(seconds): def clear_time_override(): """Remove the overridden time.""" utcnow.override_time = None + + +def marshall_now(now=None): + """Make an rpc-safe datetime with microseconds. + + Note: tzinfo is stripped, but not required for relative times.""" + if not now: + now = utcnow() + return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, + minute=now.minute, second=now.second, + microsecond=now.microsecond) + + +def unmarshall_time(tyme): + """Unmarshall a datetime dict.""" + return datetime.datetime(day=tyme['day'], month=tyme['month'], + year=tyme['year'], hour=tyme['hour'], minute=tyme['minute'], + second=tyme['second'], microsecond=tyme['microsecond']) |