Import latest openstack-common

Main change is the MultiStrOpt support which pulls in the new
iniparser module. See lp#955308.

Change-Id: I1358f09038113873e60343f00a95229002b58c1f

2 files changed, 280 insertions, 53 deletions

diff --git a/keystone/openstack/common/cfg.py b/keystone/openstack/common/cfg.py
index 51c72ecd..a83e8638 100644
--- a/keystone/openstack/common/cfg.py
+++ b/keystone/openstack/common/cfg.py
@@ -17,7 +17,9 @@
 r"""
 Configuration options which may be set on the command line or in config files.
 
-The schema for each option is defined using the Opt sub-classes e.g.
+The schema for each option is defined using the Opt sub-classes, e.g.:
+
+::
 
     common_opts = [
         cfg.StrOpt('bind_host',
@@ -28,20 +30,20 @@ The schema for each option is defined using the Opt sub-classes e.g.
                    help='Port number to listen on')
     ]
 
-Options can be strings, integers, floats, booleans, lists or 'multi strings':
+Options can be strings, integers, floats, booleans, lists or 'multi strings'::
 
     enabled_apis_opt = cfg.ListOpt('enabled_apis',
                                    default=['ec2', 'osapi_compute'],
                                    help='List of APIs to enable by default')
 
     DEFAULT_EXTENSIONS = [
-        'nova.api.openstack.contrib.standard_extensions'
+        'nova.api.openstack.compute.contrib.standard_extensions'
     ]
     osapi_compute_extension_opt = cfg.MultiStrOpt('osapi_compute_extension',
                                                   default=DEFAULT_EXTENSIONS)
 
 Option schemas are registered with with the config manager at runtime, but
-before the option is referenced:
+before the option is referenced::
 
     class ExtensionManager(object):
 
@@ -57,7 +59,7 @@ before the option is referenced:
                 ....
 
 A common usage pattern is for each option schema to be defined in the module or
-class which uses the option:
+class which uses the option::
 
     opts = ...
 
@@ -72,7 +74,7 @@ class which uses the option:
 
 An option may optionally be made available via the command line. Such options
 must registered with the config manager before the command line is parsed (for
-the purposes of --help and CLI arg validation):
+the purposes of --help and CLI arg validation)::
 
     cli_opts = [
         cfg.BoolOpt('verbose',
@@ -88,7 +90,7 @@ the purposes of --help and CLI arg validation):
     def add_common_opts(conf):
         conf.register_cli_opts(cli_opts)
 
-The config manager has a single CLI option defined by default, --config-file:
+The config manager has a single CLI option defined by default, --config-file::
 
     class ConfigOpts(object):
 
@@ -99,9 +101,9 @@ The config manager has a single CLI option defined by default, --config-file:
             ...
             self.register_cli_opt(self.config_file_opt)
 
-Option values are parsed from any supplied config files using SafeConfigParser.
-If none are specified, a default set is used e.g. glance-api.conf and
-glance-common.conf:
+Option values are parsed from any supplied config files using
+openstack.common.iniparser. If none are specified, a default set is used
+e.g. glance-api.conf and glance-common.conf::
 
     glance-api.conf:
       [DEFAULT]
@@ -116,7 +118,7 @@ are parsed in order, with values in later files overriding those in earlier
 files.
 
 The parsing of CLI args and config files is initiated by invoking the config
-manager e.g.
+manager e.g.::
 
     conf = ConfigOpts()
     conf.register_opt(BoolOpt('verbose', ...))
@@ -124,10 +126,10 @@ manager e.g.
     if conf.verbose:
         ...
 
-Options can be registered as belonging to a group:
+Options can be registered as belonging to a group::
 
-    rabbit_group = cfg.OptionGroup(name='rabbit',
-                                   title='RabbitMQ options')
+    rabbit_group = cfg.OptGroup(name='rabbit',
+                                title='RabbitMQ options')
 
     rabbit_host_opt = cfg.StrOpt('host',
                                  default='localhost',
@@ -143,7 +145,7 @@ Options can be registered as belonging to a group:
         conf.register_opt(rabbit_port_opt, group='rabbit')
 
 If no group is specified, options belong to the 'DEFAULT' section of config
-files:
+files::
 
     glance-api.conf:
       [DEFAULT]
@@ -158,13 +160,14 @@ files:
       password = guest
       virtual_host = /
 
-Command-line options in a group are automatically prefixed with the group name:
+Command-line options in a group are automatically prefixed with the
+group name::
 
     --rabbit-host localhost --rabbit-port 9999
 
 Option values in the default group are referenced as attributes/properties on
 the config manager; groups are also attributes on the config manager, with
-attributes for each of the options associated with the group:
+attributes for each of the options associated with the group::
 
     server.start(app, conf.bind_port, conf.bind_host, conf)
 
@@ -173,7 +176,7 @@ attributes for each of the options associated with the group:
         port=conf.rabbit.port,
         ...)
 
-Option values may reference other values using PEP 292 string substitution:
+Option values may reference other values using PEP 292 string substitution::
 
     opts = [
         cfg.StrOpt('state_path',
@@ -191,29 +194,40 @@ Note that interpolation can be avoided by using '$$'.
 
 For command line utilities that dispatch to other command line utilities, the
 disable_interspersed_args() method is available. If this this method is called,
-then parsing e.g.
+then parsing e.g.::
 
   script --verbose cmd --debug /tmp/mything
 
-will no longer return:
+will no longer return::
 
   ['cmd', '/tmp/mything']
 
-as the leftover arguments, but will instead return:
+as the leftover arguments, but will instead return::
 
   ['cmd', '--debug', '/tmp/mything']
 
 i.e. argument parsing is stopped at the first non-option argument.
+
+Options may be declared as secret so that their values are not leaked into
+log files:
+
+     opts = [
+        cfg.StrOpt('s3_store_access_key', secret=True),
+        cfg.StrOpt('s3_store_secret_key', secret=True),
+        ...
+     ]
+
 """
 
 import collections
-import ConfigParser
 import copy
 import optparse
 import os
 import string
 import sys
 
+from keystone.openstack.common import iniparser
+
 
 class Error(Exception):
     """Base class for cfg exceptions."""
@@ -307,9 +321,12 @@ class ConfigFileValueError(Error):
 def find_config_files(project=None, prog=None):
     """Return a list of default configuration files.
 
+    :param project: an optional project name
+    :param prog: the program name, defaulting to the basename of sys.argv[0]
+
     We default to two config files: [${project}.conf, ${prog}.conf]
 
-    And we look for those config files in the following directories:
+    And we look for those config files in the following directories::
 
       ~/.${project}/
       ~/
@@ -324,9 +341,6 @@ def find_config_files(project=None, prog=None):
     '~/.foo/bar.conf']
 
     If no project name is supplied, we only look for ${prog.conf}.
-
-    :param project: an optional project name
-    :param prog: the program name, defaulting to the basename of sys.argv[0]
     """
     if prog is None:
         prog = os.path.basename(sys.argv[0])
@@ -395,9 +409,10 @@ class Opt(object):
       help:
         an string explaining how the options value is used
     """
+    multi = False
 
-    def __init__(self, name, dest=None, short=None,
-                 default=None, metavar=None, help=None):
+    def __init__(self, name, dest=None, short=None, default=None,
+                 metavar=None, help=None, secret=False):
         """Construct an Opt object.
 
         The only required parameter is the option's name. However, it is
@@ -409,6 +424,7 @@ class Opt(object):
         :param default: the default value of the option
         :param metavar: the option argument to show in --help
         :param help: an explanation of how the option is used
+        :param secret: true iff the value should be obfuscated in log output
         """
         self.name = name
         if dest is None:
@@ -419,9 +435,10 @@ class Opt(object):
         self.default = default
         self.metavar = metavar
         self.help = help
+        self.secret = secret
 
     def _get_from_config_parser(self, cparser, section):
-        """Retrieves the option value from a ConfigParser object.
+        """Retrieves the option value from a MultiConfigParser object.
 
         This is the method ConfigOpts uses to look up the option value from
         config files. Most opt types override this method in order to perform
@@ -430,7 +447,7 @@ class Opt(object):
         :param cparser: a ConfigParser object
         :param section: a section name
         """
-        return cparser.get(section, self.dest, raw=True)
+        return cparser.get(section, self.dest)
 
     def _add_to_cli(self, parser, group=None):
         """Makes the option available in the command line interface.
@@ -532,9 +549,19 @@ class BoolOpt(Opt):
     1/0, yes/no, true/false or on/off.
     """
 
+    _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                       '0': False, 'no': False, 'false': False, 'off': False}
+
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a boolean from ConfigParser."""
-        return cparser.getboolean(section, self.dest)
+        def convert_bool(v):
+            value = self._boolean_states.get(v.lower())
+            if value is None:
+                raise ValueError('Unexpected boolean value %r' % v)
+
+            return value
+
+        return [convert_bool(v) for v in cparser.get(section, self.dest)]
 
     def _add_to_cli(self, parser, group=None):
         """Extends the base class method to add the --nooptname option."""
@@ -561,7 +588,7 @@ class IntOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a integer from ConfigParser."""
-        return cparser.getint(section, self.dest)
+        return [int(v) for v in cparser.get(section, self.dest)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for integer options."""
@@ -575,7 +602,7 @@ class FloatOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a float from ConfigParser."""
-        return cparser.getfloat(section, self.dest)
+        return [float(v) for v in cparser.get(section, self.dest)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for float options."""
@@ -592,7 +619,7 @@ class ListOpt(Opt):
 
     def _get_from_config_parser(self, cparser, section):
         """Retrieve the opt value as a list from ConfigParser."""
-        return cparser.get(section, self.dest).split(',')
+        return [v.split(',') for v in cparser.get(section, self.dest)]
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for list options."""
@@ -614,14 +641,7 @@ class MultiStrOpt(Opt):
     Multistr opt values are string opts which may be specified multiple times.
     The opt value is a list containing all the string values specified.
     """
-
-    def _get_from_config_parser(self, cparser, section):
-        """Retrieve the opt value as a multistr from ConfigParser."""
-        # FIXME(markmc): values spread across the CLI and multiple
-        #                config files should be appended
-        value = super(MultiStrOpt, self)._get_from_config_parser(cparser,
-                                                                 section)
-        return value if value is None else [value]
+    multi = True
 
     def _get_optparse_kwargs(self, group, **kwargs):
         """Extends the base optparse keyword dict for multi str options."""
@@ -688,6 +708,69 @@ class OptGroup(object):
         return self._optparse_group
 
 
+class ParseError(iniparser.ParseError):
+    def __init__(self, msg, lineno, line, filename):
+        super(ParseError, self).__init__(msg, lineno, line)
+        self.filename = filename
+
+    def __str__(self):
+        return 'at %s:%d, %s: %r' % (self.filename, self.lineno,
+                                     self.msg, self.line)
+
+
+class ConfigParser(iniparser.BaseParser):
+    def __init__(self, filename, sections):
+        super(ConfigParser, self).__init__()
+        self.filename = filename
+        self.sections = sections
+        self.section = None
+
+    def parse(self):
+        with open(self.filename) as f:
+            return super(ConfigParser, self).parse(f)
+
+    def new_section(self, section):
+        self.section = section
+        self.sections.setdefault(self.section, {})
+
+    def assignment(self, key, value):
+        if not self.section:
+            raise self.error_no_section()
+
+        self.sections[self.section].setdefault(key, [])
+        self.sections[self.section][key].append('\n'.join(value))
+
+    def parse_exc(self, msg, lineno, line=None):
+        return ParseError(msg, lineno, line, self.filename)
+
+    def error_no_section(self):
+        return self.parse_exc('Section must be started before assignment',
+                              self.lineno)
+
+
+class MultiConfigParser(object):
+    def __init__(self):
+        self.sections = {}
+
+    def read(self, config_files):
+        read_ok = []
+
+        for filename in config_files:
+            parser = ConfigParser(filename, self.sections)
+
+            try:
+                parser.parse()
+            except IOError:
+                continue
+
+            read_ok.append(filename)
+
+        return read_ok
+
+    def get(self, section, name):
+        return self.sections[section][name]
+
+
 class ConfigOpts(collections.Mapping):
 
     """
@@ -945,15 +1028,22 @@ class ConfigOpts(collections.Mapping):
         logger.log(lvl, "config files: %s", self.config_file)
         logger.log(lvl, "=" * 80)
 
+        def _sanitize(opt, value):
+            """Obfuscate values of options declared secret"""
+            return value if not opt.secret else '*' * len(str(value))
+
         for opt_name in sorted(self._opts):
-            logger.log(lvl, "%-30s = %s", opt_name, getattr(self, opt_name))
+            opt = self._get_opt_info(opt_name)['opt']
+            logger.log(lvl, "%-30s = %s", opt_name,
+                       _sanitize(opt, getattr(self, opt_name)))
 
         for group_name in self._groups:
             group_attr = self.GroupAttr(self, self._get_group(group_name))
             for opt_name in sorted(self._groups[group_name]._opts):
+                opt = self._get_opt_info(opt_name, group_name)['opt']
                 logger.log(lvl, "%-30s = %s",
                            "%s.%s" % (group_name, opt_name),
-                           getattr(group_attr, opt_name))
+                           _sanitize(opt, getattr(group_attr, opt_name)))
 
         logger.log(lvl, "*" * 80)
 
@@ -983,20 +1073,31 @@ class ConfigOpts(collections.Mapping):
         if override is not None:
             return override
 
+        values = []
         if self._cparser is not None:
             section = group.name if group is not None else 'DEFAULT'
             try:
-                return opt._get_from_config_parser(self._cparser, section)
-            except (ConfigParser.NoOptionError,
-                    ConfigParser.NoSectionError):
+                value = opt._get_from_config_parser(self._cparser, section)
+            except KeyError:
                 pass
-            except ValueError, ve:
+            except ValueError as ve:
                 raise ConfigFileValueError(str(ve))
+            else:
+                if not opt.multi:
+                    # No need to continue since the last value wins
+                    return value[-1]
+                values.extend(value)
 
         name = name if group is None else group.name + '_' + name
-        value = self._cli_values.get(name, None)
+        value = self._cli_values.get(name)
         if value is not None:
-            return value
+            if not opt.multi:
+                return value
+
+            return value + values
+
+        if values:
+            return values
 
         if default is not None:
             return default
@@ -1066,12 +1167,12 @@ class ConfigOpts(collections.Mapping):
 
         :raises: ConfigFilesNotFoundError, ConfigFileParseError
         """
-        self._cparser = ConfigParser.SafeConfigParser()
+        self._cparser = MultiConfigParser()
 
         try:
             read_ok = self._cparser.read(config_files)
-        except ConfigParser.ParsingError, cpe:
-            raise ConfigFileParseError(cpe.filename, cpe.message)
+        except iniparser.ParseError as pe:
+            raise ConfigFileParseError(pe.filename, str(pe))
 
         if read_ok != config_files:
             not_read_ok = filter(lambda f: f not in read_ok, config_files)
diff --git a/keystone/openstack/common/iniparser.py b/keystone/openstack/common/iniparser.py
new file mode 100644
index 00000000..53ca0233
--- /dev/null
+++ b/keystone/openstack/common/iniparser.py
@@ -0,0 +1,126 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack LLC.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class ParseError(Exception):
+    def __init__(self, message, lineno, line):
+        self.msg = message
+        self.line = line
+        self.lineno = lineno
+
+    def __str__(self):
+        return 'at line %d, %s: %r' % (self.lineno, self.msg, self.line)
+
+
+class BaseParser(object):
+    lineno = 0
+    parse_exc = ParseError
+
+    def _assignment(self, key, value):
+        self.assignment(key, value)
+        return None, []
+
+    def _get_section(self, line):
+        if line[-1] != ']':
+            return self.error_no_section_end_bracket(line)
+        if len(line) <= 2:
+            return self.error_no_section_name(line)
+
+        return line[1:-1]
+
+    def _split_key_value(self, line):
+        colon = line.find(':')
+        equal = line.find('=')
+        if colon < 0 and equal < 0:
+            return self.error_invalid_assignment(line)
+
+        if colon < 0 or (equal >= 0 and equal < colon):
+            key, value = line[:equal], line[equal + 1:]
+        else:
+            key, value = line[:colon], line[colon + 1:]
+
+        return key.strip(), [value.strip()]
+
+    def parse(self, lineiter):
+        key = None
+        value = []
+
+        for line in lineiter:
+            self.lineno += 1
+
+            line = line.rstrip()
+            if not line:
+                # Blank line, ends multi-line values
+                if key:
+                    key, value = self._assignment(key, value)
+                continue
+            elif line[0] in (' ', '\t'):
+                # Continuation of previous assignment
+                if key is None:
+                    self.error_unexpected_continuation(line)
+                else:
+                    value.append(line.lstrip())
+                continue
+
+            if key:
+                # Flush previous assignment, if any
+                key, value = self._assignment(key, value)
+
+            if line[0] == '[':
+                # Section start
+                section = self._get_section(line)
+                if section:
+                    self.new_section(section)
+            elif line[0] in '#;':
+                self.comment(line[1:].lstrip())
+            else:
+                key, value = self._split_key_value(line)
+                if not key:
+                    return self.error_empty_key(line)
+
+        if key:
+            # Flush previous assignment, if any
+            self._assignment(key, value)
+
+    def assignment(self, key, value):
+        """Called when a full assignment is parsed"""
+        raise NotImplementedError()
+
+    def new_section(self, section):
+        """Called when a new section is started"""
+        raise NotImplementedError()
+
+    def comment(self, comment):
+        """Called when a comment is parsed"""
+        pass
+
+    def error_invalid_assignment(self, line):
+        raise self.parse_exc("No ':' or '=' found in assignment",
+                             self.lineno, line)
+
+    def error_empty_key(self, line):
+        raise self.parse_exc('Key cannot be empty', self.lineno, line)
+
+    def error_unexpected_continuation(self, line):
+        raise self.parse_exc('Unexpected continuation line',
+                             self.lineno, line)
+
+    def error_no_section_end_bracket(self, line):
+        raise self.parse_exc('Invalid section (must end with ])',
+                             self.lineno, line)
+
+    def error_no_section_name(self, line):
+        raise self.parse_exc('Empty section name', self.lineno, line)