Merge "Support for directory source of config files"

2 files changed, 157 insertions, 56 deletions

diff --git a/openstack/common/cfg.py b/openstack/common/cfg.py
index 3602fee..c492ef1 100644
--- a/openstack/common/cfg.py
+++ b/openstack/common/cfg.py
@@ -90,16 +90,21 @@ the purposes of --help and CLI arg validation)::
     def add_common_opts(conf):
         conf.register_cli_opts(cli_opts)
 
-The config manager has a single CLI option defined by default, --config-file::
+The config manager has two CLI options defined by default, --config-file
+and --config-dir::
 
     class ConfigOpts(object):
 
-        config_file_opt = MultiStrOpt('config-file',
-                                      ...
-
         def __init__(self, ...):
-            ...
-            self.register_cli_opt(self.config_file_opt)
+
+            opts = [
+                MultiStrOpt('config-file',
+                        ...),
+                StrOpt('config-dir',
+                       ...),
+            ]
+
+            self.register_cli_opts(opts)
 
 Option values are parsed from any supplied config files using
 openstack.common.iniparser. If none are specified, a default set is used
@@ -221,6 +226,7 @@ log files:
 
 import collections
 import copy
+import glob
 import functools
 import optparse
 import os
@@ -825,14 +831,26 @@ class ConfigOpts(collections.Mapping):
 
         self.__cache = {}
 
-        self.register_cli_opt(
-            MultiStrOpt('config-file',
-                        default=self.default_config_files,
-                        metavar='PATH',
-                        help='Path to a config file to use. Multiple config '
-                             'files can be specified, with values in later '
-                             'files taking precedence. The default files used '
-                             'are: %s' % (self.default_config_files, )))
+        opts = [
+             MultiStrOpt('config-file',
+                         default=self.default_config_files,
+                         metavar='PATH',
+                         help='Path to a config file to use. Multiple config '
+                              'files can be specified, with values in later '
+                              'files taking precedence. The default files '
+                              ' used are: %s' %
+                              (self.default_config_files, )),
+            StrOpt('config-dir',
+                   metavar='DIR',
+                   help='Path to a config directory to pull *.conf '
+                        'files from. This file set is sorted, so as to '
+                        'provide a predictable parse order if individual '
+                        'options are over-ridden. The set is parsed after '
+                        'the file(s), if any, specified via --config-file, '
+                        'hence over-ridden options in the directory take '
+                        'precedence.'),
+            ]
+        self.register_cli_opts(opts)
 
     def __clear_cache(f):
         @functools.wraps(f)
@@ -853,6 +871,10 @@ class ConfigOpts(collections.Mapping):
         The object may be called multiple times, each time causing the previous
         set of values to be overwritten.
 
+        If the --config-dir option is set, any *.conf files from this
+        directory are pulled in, after all the file(s) specified by the
+        --config-file option.
+
         :params args: command line arguments (defaults to sys.argv[1:])
         :returns: the list of arguments left over after parsing options
         :raises: SystemExit, ConfigFilesNotFoundError, ConfigFileParseError
@@ -865,8 +887,14 @@ class ConfigOpts(collections.Mapping):
 
         self._cli_values = vars(values)
 
-        if self.config_file:
-            self._parse_config_files(self.config_file)
+        def _list_config_dir():
+            return sorted(glob.glob(os.path.join(self.config_dir, '*.conf')))
+
+        from_file = list(self.config_file)
+
+        from_dir = _list_config_dir() if self.config_dir else []
+
+        self._parse_config_files(from_file + from_dir)
 
         return args
 
diff --git a/tests/unit/test_cfg.py b/tests/unit/test_cfg.py
index da78254..1d8bdca 100644
--- a/tests/unit/test_cfg.py
+++ b/tests/unit/test_cfg.py
@@ -15,6 +15,7 @@
 #    under the License.
 
 import os
+import shutil
 import sys
 import StringIO
 import tempfile
@@ -72,6 +73,7 @@ class BaseTestCase(unittest.TestCase):
                                usage='%prog FOO BAR',
                                default_config_files=[])
         self.tempfiles = []
+        self.tempdirs = []
         self.stubs = stubout.StubOutForTesting()
 
     def tearDown(self):
@@ -80,7 +82,7 @@ class BaseTestCase(unittest.TestCase):
 
     def create_tempfiles(self, files):
         for (basename, contents) in files:
-            (fd, path) = tempfile.mkstemp(prefix=basename)
+            (fd, path) = tempfile.mkstemp(prefix=basename, suffix='.conf')
             self.tempfiles.append(path)
             try:
                 os.write(fd, contents)
@@ -91,6 +93,8 @@ class BaseTestCase(unittest.TestCase):
     def remove_tempfiles(self):
         for p in self.tempfiles:
             os.remove(p)
+        for d in self.tempdirs:
+            shutil.rmtree(d, ignore_errors=True)
 
 
 class UsageTestCase(BaseTestCase):
@@ -209,8 +213,8 @@ class CliOptsTestCase(BaseTestCase):
         self.assertTrue('1.0' in sys.stdout.getvalue())
 
     def test_config_file(self):
-        paths = self.create_tempfiles([('1.conf', '[DEFAULT]'),
-                                       ('2.conf', '[DEFAULT]')])
+        paths = self.create_tempfiles([('1', '[DEFAULT]'),
+                                       ('2', '[DEFAULT]')])
 
         self.conf(['--config-file', paths[0], '--config-file', paths[1]])
 
@@ -234,7 +238,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_str_default(self):
         self.conf.register_opt(StrOpt('foo', default='bar'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n')])
 
         self.conf(['--config-file', paths[0]])
@@ -245,7 +249,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_str_value(self):
         self.conf.register_opt(StrOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = bar\n')])
 
@@ -257,10 +261,10 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_str_value_override(self):
         self.conf.register_cli_opt(StrOpt('foo'))
 
-        paths = self.create_tempfiles([('1.conf',
+        paths = self.create_tempfiles([('1',
                                         '[DEFAULT]\n'
                                         'foo = baar\n'),
-                                       ('2.conf',
+                                       ('2',
                                         '[DEFAULT]\n'
                                         'foo = baaar\n')])
 
@@ -274,7 +278,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_bool_default(self):
         self.conf.register_opt(BoolOpt('foo', default=False))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n')])
 
         self.conf(['--config-file', paths[0]])
@@ -285,7 +289,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_bool_value(self):
         self.conf.register_opt(BoolOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = true\n')])
 
@@ -297,10 +301,10 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_bool_value_override(self):
         self.conf.register_cli_opt(BoolOpt('foo'))
 
-        paths = self.create_tempfiles([('1.conf',
+        paths = self.create_tempfiles([('1',
                                         '[DEFAULT]\n'
                                         'foo = 0\n'),
-                                       ('2.conf',
+                                       ('2',
                                         '[DEFAULT]\n'
                                         'foo = yes\n')])
 
@@ -314,7 +318,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_int_default(self):
         self.conf.register_opt(IntOpt('foo', default=666))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n')])
 
         self.conf(['--config-file', paths[0]])
@@ -325,7 +329,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_int_value(self):
         self.conf.register_opt(IntOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = 666\n')])
 
@@ -337,10 +341,10 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_int_value_override(self):
         self.conf.register_cli_opt(IntOpt('foo'))
 
-        paths = self.create_tempfiles([('1.conf',
+        paths = self.create_tempfiles([('1',
                                         '[DEFAULT]\n'
                                         'foo = 66\n'),
-                                       ('2.conf',
+                                       ('2',
                                         '[DEFAULT]\n'
                                         'foo = 666\n')])
 
@@ -354,7 +358,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_float_default(self):
         self.conf.register_opt(FloatOpt('foo', default=6.66))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n')])
 
         self.conf(['--config-file', paths[0]])
@@ -365,7 +369,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_float_value(self):
         self.conf.register_opt(FloatOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = 6.66\n')])
 
@@ -377,10 +381,10 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_float_value_override(self):
         self.conf.register_cli_opt(FloatOpt('foo'))
 
-        paths = self.create_tempfiles([('1.conf',
+        paths = self.create_tempfiles([('1',
                                         '[DEFAULT]\n'
                                         'foo = 6.6\n'),
-                                       ('2.conf',
+                                       ('2',
                                         '[DEFAULT]\n'
                                         'foo = 6.66\n')])
 
@@ -394,7 +398,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_list_default(self):
         self.conf.register_opt(ListOpt('foo', default=['bar']))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n')])
 
         self.conf(['--config-file', paths[0]])
@@ -405,7 +409,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_list_value(self):
         self.conf.register_opt(ListOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = bar\n')])
 
@@ -417,10 +421,10 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_list_value_override(self):
         self.conf.register_cli_opt(ListOpt('foo'))
 
-        paths = self.create_tempfiles([('1.conf',
+        paths = self.create_tempfiles([('1',
                                         '[DEFAULT]\n'
                                         'foo = bar,bar\n'),
-                                       ('2.conf',
+                                       ('2',
                                         '[DEFAULT]\n'
                                         'foo = b,a,r\n')])
 
@@ -434,7 +438,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_multistr_default(self):
         self.conf.register_opt(MultiStrOpt('foo', default=['bar']))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n')])
 
         self.conf(['--config-file', paths[0]])
@@ -445,7 +449,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_multistr_value(self):
         self.conf.register_opt(MultiStrOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = bar\n')])
 
@@ -457,10 +461,10 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_multistr_values_append(self):
         self.conf.register_cli_opt(MultiStrOpt('foo'))
 
-        paths = self.create_tempfiles([('1.conf',
+        paths = self.create_tempfiles([('1',
                                         '[DEFAULT]\n'
                                         'foo = bar1\n'),
-                                       ('2.conf',
+                                       ('2',
                                         '[DEFAULT]\n'
                                         'foo = bar2\n'
                                         'foo = bar3\n')])
@@ -476,7 +480,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_multiple_opts(self):
         self.conf.register_opts([StrOpt('foo'), StrOpt('bar')])
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = bar\n'
                                         'bar = foo\n')])
@@ -491,7 +495,7 @@ class ConfigFileOptsTestCase(BaseTestCase):
     def test_conf_file_raw_value(self):
         self.conf.register_opt(StrOpt('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = bar-%08x\n')])
 
@@ -538,7 +542,7 @@ class OptGroupsTestCase(BaseTestCase):
         self.conf.register_group(OptGroup('blaa'))
         self.conf.register_opt(StrOpt('foo'), group='blaa')
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[blaa]\n'
                                         'foo = bar\n')])
 
@@ -558,7 +562,7 @@ class MappingInterfaceTestCase(BaseTestCase):
 
         self.assertTrue('foo' in self.conf)
         self.assertTrue('config_file' in self.conf)
-        self.assertEquals(len(self.conf), 2)
+        self.assertEquals(len(self.conf), 3)
         self.assertEquals(self.conf['foo'], 'bar')
         self.assertEquals(self.conf.get('foo'), 'bar')
         self.assertTrue('bar' in self.conf.values())
@@ -629,7 +633,7 @@ class TemplateSubstitutionTestCase(BaseTestCase):
     def test_str_sub_default_from_config_file(self):
         self._prep_test_str_sub(bar_default='$foo')
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = blaa\n')])
 
@@ -654,7 +658,7 @@ class TemplateSubstitutionTestCase(BaseTestCase):
     def test_str_sub_arg_from_config_file(self):
         self._prep_test_str_sub()
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = blaa\n')])
 
@@ -665,7 +669,7 @@ class TemplateSubstitutionTestCase(BaseTestCase):
     def test_str_sub_config_file_from_default(self):
         self._prep_test_str_sub(foo_default='blaa')
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'bar = $foo\n')])
 
@@ -676,7 +680,7 @@ class TemplateSubstitutionTestCase(BaseTestCase):
     def test_str_sub_config_file_from_arg(self):
         self._prep_test_str_sub()
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'bar = $foo\n')])
 
@@ -687,7 +691,7 @@ class TemplateSubstitutionTestCase(BaseTestCase):
     def test_str_sub_config_file_from_config_file(self):
         self._prep_test_str_sub()
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'bar = $foo\n'
                                         'foo = blaa\n')])
@@ -707,6 +711,74 @@ class TemplateSubstitutionTestCase(BaseTestCase):
         self.assertTrue(hasattr(self.conf.ba, 'r'))
         self.assertEquals(self.conf.ba.r, 'blaa')
 
+    def test_config_dir(self):
+        snafu_group = OptGroup('snafu')
+        self.conf.register_group(snafu_group)
+        self.conf.register_cli_opt(StrOpt('foo'))
+        self.conf.register_cli_opt(StrOpt('bell'), group=snafu_group)
+
+        dir = tempfile.mkdtemp()
+        self.tempdirs.append(dir)
+
+        paths = self.create_tempfiles([(os.path.join(dir, '00-test'),
+                                        '[DEFAULT]\n'
+                                        'foo = bar-00\n'
+                                        '[snafu]\n'
+                                        'bell = whistle-00\n'),
+                                       (os.path.join(dir, '02-test'),
+                                        '[snafu]\n'
+                                        'bell = whistle-02\n'
+                                        '[DEFAULT]\n'
+                                        'foo = bar-02\n'),
+                                       (os.path.join(dir, '01-test'),
+                                        '[DEFAULT]\n'
+                                        'foo = bar-01\n')])
+
+        self.conf(['--foo', 'bar',
+                   '--config-dir', os.path.dirname(paths[0])])
+
+        self.assertTrue(hasattr(self.conf, 'foo'))
+        self.assertEquals(self.conf.foo, 'bar-02')
+        self.assertTrue(hasattr(self.conf, 'snafu'))
+        self.assertTrue(hasattr(self.conf.snafu, 'bell'))
+        self.assertEquals(self.conf.snafu.bell, 'whistle-02')
+
+    def test_config_dir_file_precedence(self):
+        snafu_group = OptGroup('snafu')
+        self.conf.register_group(snafu_group)
+        self.conf.register_cli_opt(StrOpt('foo'))
+        self.conf.register_cli_opt(StrOpt('bell'), group=snafu_group)
+
+        dir = tempfile.mkdtemp()
+        self.tempdirs.append(dir)
+
+        paths = self.create_tempfiles([(os.path.join(dir, '00-test'),
+                                        '[DEFAULT]\n'
+                                        'foo = bar-00\n'),
+                                       ('01-test',
+                                        '[snafu]\n'
+                                        'bell = whistle-01\n'
+                                        '[DEFAULT]\n'
+                                        'foo = bar-01\n'),
+                                       ('03-test',
+                                        '[snafu]\n'
+                                        'bell = whistle-03\n'
+                                        '[DEFAULT]\n'
+                                        'foo = bar-03\n'),
+                                       (os.path.join(dir, '02-test'),
+                                        '[DEFAULT]\n'
+                                        'foo = bar-02\n')])
+
+        self.conf(['--foo', 'bar',
+                   '--config-file', paths[1],
+                   '--config-dir', os.path.dirname(paths[0]),
+                   '--config-file', paths[2], ])
+
+        self.assertTrue(hasattr(self.conf, 'foo'))
+        self.assertEquals(self.conf.foo, 'bar-02')
+        self.assertTrue(hasattr(self.conf, 'snafu'))
+        self.assertTrue(hasattr(self.conf.snafu, 'bell'))
+        self.assertEquals(self.conf.snafu.bell, 'whistle-03')
 
 class ReparseTestCase(BaseTestCase):
 
@@ -714,7 +786,7 @@ class ReparseTestCase(BaseTestCase):
         self.conf.register_group(OptGroup('blaa'))
         self.conf.register_cli_opt(StrOpt('foo', default='r'), group='blaa')
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[blaa]\n'
                                         'foo = b\n')])
 
@@ -868,7 +940,7 @@ class SadPathTestCase(BaseTestCase):
         self._do_test_bad_cli_value(FloatOpt)
 
     def test_conf_file_not_found(self):
-        paths = self.create_tempfiles([('test.conf', '')])
+        paths = self.create_tempfiles([('test', '')])
         os.remove(paths[0])
         self.tempfiles.remove(paths[0])
 
@@ -876,7 +948,7 @@ class SadPathTestCase(BaseTestCase):
                           self.conf, ['--config-file', paths[0]])
 
     def test_conf_file_broken(self):
-        paths = self.create_tempfiles([('test.conf', 'foo')])
+        paths = self.create_tempfiles([('test', 'foo')])
 
         self.assertRaises(ConfigFileParseError,
                           self.conf, ['--config-file', paths[0]])
@@ -884,7 +956,7 @@ class SadPathTestCase(BaseTestCase):
     def _do_test_conf_file_bad_value(self, opt_class):
         self.conf.register_opt(opt_class('foo'))
 
-        paths = self.create_tempfiles([('test.conf',
+        paths = self.create_tempfiles([('test',
                                         '[DEFAULT]\n'
                                         'foo = bar\n')])
 
@@ -964,6 +1036,7 @@ class OptDumpingTestCase(BaseTestCase):
                 "'--blaa-key', 'admin', '--passwd', 'hush']",
                 "config files: []",
                 "=" * 80,
+                "config_dir                     = None",
                 "config_file                    = []",
                 "foo                            = this",
                 "passwd                         = ****",