From e09a12138494e78055f03afa68da661befaa16c4 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Wed, 5 Sep 2012 11:33:40 +0100 Subject: Sync latest cfg from openstack-common Syncs the following changes from stable/folsom: 4dc2043 cfg: clean up None value handling 513bd3a Allow set_default and set_override to use None 0a36c92 Tilde expansion for --config-file and --config-dir 038d597 Add import_opt() method to ConfigOpts 90da88c Modifies _is_opt_registered fcn to check for duplicate opts 3af0786 cfg: Fix typo in documentation 9f93872 Update common code to support pep 1.3. bug 1014216 fb0f00e Use 'is not None' instead of '!= None' d1ec5ce Add support to include config aliases Also, exclude keystone/openstack/common from pep8 checks because it is pep8-checked in openstack-common anyway and all pep8 fixes would need to go into openstack-common first. Change-Id: Ie6354251942648c16c2be270af2b95f3f45e27ed --- keystone/openstack/common/cfg.py | 173 +++++++++++++++++++++++++-------- keystone/openstack/common/iniparser.py | 6 +- tox.ini | 2 +- 3 files changed, 139 insertions(+), 42 deletions(-) diff --git a/keystone/openstack/common/cfg.py b/keystone/openstack/common/cfg.py index 899a6c46..e0d76ff2 100644 --- a/keystone/openstack/common/cfg.py +++ b/keystone/openstack/common/cfg.py @@ -42,8 +42,8 @@ Options can be strings, integers, floats, booleans, lists or 'multi strings':: osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension', default=DEFAULT_EXTENSIONS) -Option schemas are registered with with the config manager at runtime, but -before the option is referenced:: +Option schemas are registered with the config manager at runtime, but before +the option is referenced:: class ExtensionManager(object): @@ -367,6 +367,11 @@ class ConfigFileValueError(Error): pass +def _fixpath(p): + """Apply tilde expansion and absolutization to a path.""" + return os.path.abspath(os.path.expanduser(p)) + + def _get_config_dirs(project=None): """Return a list of directors where config files may be located. @@ -384,11 +389,9 @@ def _get_config_dirs(project=None): ~/ /etc/ """ - fix_path = lambda p: os.path.abspath(os.path.expanduser(p)) - cfg_dirs = [ - fix_path(os.path.join('~', '.' + project)) if project else None, - fix_path('~'), + _fixpath(os.path.join('~', '.' + project)) if project else None, + _fixpath('~'), os.path.join('/etc', project) if project else None, '/etc' ] @@ -464,7 +467,7 @@ def _is_opt_registered(opts, opt): :raises: DuplicateOptError if a naming conflict is detected """ if opt.dest in opts: - if opts[opt.dest]['opt'] is not opt: + if opts[opt.dest]['opt'] != opt: raise DuplicateOptError(opt.name) return True else: @@ -494,7 +497,8 @@ class Opt(object): multi = False def __init__(self, name, dest=None, short=None, default=None, - metavar=None, help=None, secret=False, required=False): + metavar=None, help=None, secret=False, required=False, + deprecated_name=None): """Construct an Opt object. The only required parameter is the option's name. However, it is @@ -508,6 +512,7 @@ class Opt(object): :param help: an explanation of how the option is used :param secret: true iff the value should be obfuscated in log output :param required: true iff a value must be supplied for this option + :param deprecated_name: deprecated name option. Acts like an alias """ self.name = name if dest is None: @@ -520,6 +525,13 @@ class Opt(object): self.help = help self.secret = secret self.required = required + if deprecated_name is not None: + self.deprecated_name = deprecated_name.replace('-', '_') + else: + self.deprecated_name = None + + def __ne__(self, another): + return vars(self) != vars(another) def _get_from_config_parser(self, cparser, section): """Retrieves the option value from a MultiConfigParser object. @@ -531,7 +543,13 @@ class Opt(object): :param cparser: a ConfigParser object :param section: a section name """ - return cparser.get(section, self.dest) + return self._cparser_get_with_deprecated(cparser, section) + + def _cparser_get_with_deprecated(self, cparser, section): + """If cannot find option as dest try deprecated_name alias.""" + if self.deprecated_name is not None: + return cparser.get(section, [self.dest, self.deprecated_name]) + return cparser.get(section, [self.dest]) def _add_to_cli(self, parser, group=None): """Makes the option available in the command line interface. @@ -546,9 +564,11 @@ class Opt(object): container = self._get_optparse_container(parser, group) kwargs = self._get_optparse_kwargs(group) prefix = self._get_optparse_prefix('', group) - self._add_to_optparse(container, self.name, self.short, kwargs, prefix) + self._add_to_optparse(container, self.name, self.short, kwargs, prefix, + self.deprecated_name) - def _add_to_optparse(self, container, name, short, kwargs, prefix=''): + def _add_to_optparse(self, container, name, short, kwargs, prefix='', + deprecated_name=None): """Add an option to an optparse parser or group. :param container: an optparse.OptionContainer object @@ -561,6 +581,8 @@ class Opt(object): args = ['--' + prefix + name] if short: args += ['-' + short] + if deprecated_name: + args += ['--' + prefix + deprecated_name] for a in args: if container.has_option(a): raise DuplicateOptError(a) @@ -591,11 +613,9 @@ class Opt(object): dest = self.dest if group is not None: dest = group.name + '_' + dest - kwargs.update({ - 'dest': dest, - 'metavar': self.metavar, - 'help': self.help, - }) + kwargs.update({'dest': dest, + 'metavar': self.metavar, + 'help': self.help, }) return kwargs def _get_optparse_prefix(self, prefix, group): @@ -645,7 +665,8 @@ class BoolOpt(Opt): return value - return [convert_bool(v) for v in cparser.get(section, self.dest)] + return [convert_bool(v) for v in + self._cparser_get_with_deprecated(cparser, section)] def _add_to_cli(self, parser, group=None): """Extends the base class method to add the --nooptname option.""" @@ -658,7 +679,8 @@ class BoolOpt(Opt): kwargs = self._get_optparse_kwargs(group, action='store_false') prefix = self._get_optparse_prefix('no', group) kwargs["help"] = "The inverse of --" + self.name - self._add_to_optparse(container, self.name, None, kwargs, prefix) + self._add_to_optparse(container, self.name, None, kwargs, prefix, + self.deprecated_name) def _get_optparse_kwargs(self, group, action='store_true', **kwargs): """Extends the base optparse keyword dict for boolean options.""" @@ -672,7 +694,8 @@ class IntOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a integer from ConfigParser.""" - return [int(v) for v in cparser.get(section, self.dest)] + return [int(v) for v in self._cparser_get_with_deprecated(cparser, + section)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for integer options.""" @@ -686,7 +709,8 @@ class FloatOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a float from ConfigParser.""" - return [float(v) for v in cparser.get(section, self.dest)] + return [float(v) for v in + self._cparser_get_with_deprecated(cparser, section)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for float options.""" @@ -703,7 +727,8 @@ class ListOpt(Opt): def _get_from_config_parser(self, cparser, section): """Retrieve the opt value as a list from ConfigParser.""" - return [v.split(',') for v in cparser.get(section, self.dest)] + return [v.split(',') for v in + self._cparser_get_with_deprecated(cparser, section)] def _get_optparse_kwargs(self, group, **kwargs): """Extends the base optparse keyword dict for list options.""" @@ -732,6 +757,13 @@ class MultiStrOpt(Opt): return super(MultiStrOpt, self)._get_optparse_kwargs(group, action='append') + def _cparser_get_with_deprecated(self, cparser, section): + """If cannot find option as dest try deprecated_name alias.""" + if self.deprecated_name is not None: + return cparser.get(section, [self.dest, self.deprecated_name], + multi=True) + return cparser.get(section, [self.dest], multi=True) + class OptGroup(object): @@ -780,7 +812,7 @@ class OptGroup(object): if _is_opt_registered(self._opts, opt): return False - self._opts[opt.dest] = {'opt': opt, 'override': None, 'default': None} + self._opts[opt.dest] = {'opt': opt} return True @@ -846,25 +878,38 @@ class ConfigParser(iniparser.BaseParser): class MultiConfigParser(object): def __init__(self): - self.sections = {} + self.parsed = [] def read(self, config_files): read_ok = [] for filename in config_files: - parser = ConfigParser(filename, self.sections) + sections = {} + parser = ConfigParser(filename, sections) try: parser.parse() except IOError: continue - + self.parsed.insert(0, sections) read_ok.append(filename) return read_ok - def get(self, section, name): - return self.sections[section][name] + def get(self, section, names, multi=False): + rvalue = [] + for sections in self.parsed: + if section not in sections: + continue + for name in names: + if name in sections[section]: + if multi: + rvalue = sections[section][name] + rvalue + else: + return sections[section][name] + if multi and rvalue != []: + return rvalue + raise KeyError class ConfigOpts(collections.Mapping): @@ -1045,7 +1090,7 @@ class ConfigOpts(collections.Mapping): if _is_opt_registered(self._opts, opt): return False - self._opts[opt.dest] = {'opt': opt, 'override': None, 'default': None} + self._opts[opt.dest] = {'opt': opt} return True @@ -1114,6 +1159,25 @@ class ConfigOpts(collections.Mapping): for opt in opts: self.unregister_opt(opt, group, clear_cache=False) + def import_opt(self, name, module_str, group=None): + """Import an option definition from a module. + + Import a module and check that a given option is registered. + + This is intended for use with global configuration objects + like cfg.CONF where modules commonly register options with + CONF at module load time. If one module requires an option + defined by another module it can use this method to explicitly + declare the dependency. + + :param name: the name/dest of the opt + :param module_str: the name of a module to import + :param group: an option OptGroup object or group name + :raises: NoSuchOptError, NoSuchGroupError + """ + __import__(module_str) + self._get_opt_info(name, group) + @__clear_cache def set_override(self, name, override, group=None): """Override an opt value. @@ -1144,6 +1208,33 @@ class ConfigOpts(collections.Mapping): opt_info = self._get_opt_info(name, group) opt_info['default'] = default + @__clear_cache + def clear_override(self, name, group=None): + """Clear an override an opt value. + + Clear a previously set override of the command line, config file + and default values of a given option. + + :param name: the name/dest of the opt + :param group: an option OptGroup object or group name + :raises: NoSuchOptError, NoSuchGroupError + """ + opt_info = self._get_opt_info(name, group) + opt_info.pop('override', None) + + @__clear_cache + def clear_default(self, name, group=None): + """Clear an override an opt's default value. + + Clear a previously set override of the default value of given option. + + :param name: the name/dest of the opt + :param group: an option OptGroup object or group name + :raises: NoSuchOptError, NoSuchGroupError + """ + opt_info = self._get_opt_info(name, group) + opt_info.pop('default', None) + def _all_opt_infos(self): """A generator function for iteration opt infos.""" for info in self._opts.values(): @@ -1160,8 +1251,8 @@ class ConfigOpts(collections.Mapping): def _unset_defaults_and_overrides(self): """Unset any default or override on all options.""" for info, group in self._all_opt_infos(): - info['default'] = None - info['override'] = None + info.pop('default', None) + info.pop('override', None) def disable_interspersed_args(self): """Set parsing to stop on the first non-option. @@ -1207,10 +1298,10 @@ class ConfigOpts(collections.Mapping): """ dirs = [] if self.config_dir: - dirs.append(self.config_dir) + dirs.append(_fixpath(self.config_dir)) for cf in reversed(self.config_file): - dirs.append(os.path.dirname(cf)) + dirs.append(os.path.dirname(_fixpath(cf))) dirs.extend(_get_config_dirs(self.project)) @@ -1284,10 +1375,10 @@ class ConfigOpts(collections.Mapping): return self.GroupAttr(self, self._get_group(name)) info = self._get_opt_info(name, group) - default, opt, override = [info[k] for k in sorted(info.keys())] + opt = info['opt'] - if override is not None: - return override + if 'override' in info: + return info['override'] values = [] if self._cparser is not None: @@ -1315,15 +1406,15 @@ class ConfigOpts(collections.Mapping): if values: return values - if default is not None: - return default + if 'default' in info: + return info['default'] return opt.default def _substitute(self, value): """Perform string template substitution. - Substititue any template variables (e.g. $foo, ${bar}) in the supplied + Substitute any template variables (e.g. $foo, ${bar}) in the supplied string value(s) with opt values. :param value: the string value, or list of string values @@ -1391,6 +1482,8 @@ class ConfigOpts(collections.Mapping): config_dir_glob = os.path.join(self.config_dir, '*.conf') config_files += sorted(glob.glob(config_dir_glob)) + config_files = [_fixpath(p) for p in config_files] + self._cparser = MultiConfigParser() try: @@ -1408,10 +1501,10 @@ class ConfigOpts(collections.Mapping): :raises: RequiredOptError """ for info, group in self._all_opt_infos(): - default, opt, override = [info[k] for k in sorted(info.keys())] + opt = info['opt'] if opt.required: - if (default is not None or override is not None): + if ('default' in info or 'override' in info): continue if self._get(opt.name, group) is None: diff --git a/keystone/openstack/common/iniparser.py b/keystone/openstack/common/iniparser.py index 53ca0233..24128444 100644 --- a/keystone/openstack/common/iniparser.py +++ b/keystone/openstack/common/iniparser.py @@ -52,7 +52,11 @@ class BaseParser(object): else: key, value = line[:colon], line[colon + 1:] - return key.strip(), [value.strip()] + value = value.strip() + if ((value and value[0] == value[-1]) and + (value[0] == "\"" or value[0] == "'")): + value = value[1:-1] + return key.strip(), [value] def parse(self, lineiter): key = None diff --git a/tox.ini b/tox.ini index 5dd354ba..ec562420 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ commands = bash run_tests.sh -N --no-pep8 [testenv:pep8] deps = pep8==1.3.3 -commands = pep8 --exclude=vcsversion.py,*.pyc --repeat --show-source bin doc httpd keystone setup.py tests +commands = pep8 --exclude=vcsversion.py,*.pyc,openstack --repeat --show-source bin doc httpd keystone setup.py tests [tox:jenkins] downloadcache = ~/cache/pip -- cgit