diff options
| -rw-r--r-- | doc/source/execution.rst | 9 | ||||
| -rw-r--r-- | jenkins_jobs/config.py | 8 | ||||
| -rw-r--r-- | jenkins_jobs/local_yaml.py | 5 | ||||
| -rw-r--r-- | jenkins_jobs/parser.py | 6 | ||||
| -rw-r--r-- | jenkins_jobs/utils.py | 3 | ||||
| -rw-r--r-- | tests/localyaml/fixtures/custom_retain_anchors.yaml | 8 | ||||
| -rw-r--r-- | tests/localyaml/fixtures/custom_retain_anchors_include001.yaml | 10 | ||||
| -rw-r--r-- | tests/localyaml/test_localyaml.py | 37 |
8 files changed, 82 insertions, 4 deletions
diff --git a/doc/source/execution.rst b/doc/source/execution.rst index f8adc683..7434be72 100644 --- a/doc/source/execution.rst +++ b/doc/source/execution.rst @@ -57,6 +57,15 @@ job_builder section so user can be sure which instance was updated. User may click the link to go directly to that job. False by default. +**retain_anchors** + (Optional) If set to True, YAML anchors will be retained across files, + allowing jobs to be composed from bits of YAML defined in separate files. + Note this means that the order of processing files matters - `jenkins-jobs` + loads files in alphabetical order (all files in a dir are loaded before any + files in subdirs). For example, if your anchors are in a file named `foo.yml` + they will be accessible in `qux.yml` but not in `bar.yml`. They will also be + accessible in `mydir/bar.yml` and `mydir/qux.yml`. False by default. + jenkins section ^^^^^^^^^^^^^^^ diff --git a/jenkins_jobs/config.py b/jenkins_jobs/config.py index 7b0c2a1d..ac15852d 100644 --- a/jenkins_jobs/config.py +++ b/jenkins_jobs/config.py @@ -41,6 +41,7 @@ recursive=False exclude=.* allow_duplicates=False allow_empty_variables=False +retain_anchors=False # other named sections could be used in addition to the implicit [jenkins] # if you have multiple jenkins servers. @@ -300,6 +301,13 @@ class JJBConfig(object): config.has_option('job_builder', 'allow_empty_variables') and config.getboolean('job_builder', 'allow_empty_variables')) + # retain anchors across files? + retain_anchors = False + if config and config.has_option('job_builder', 'retain_anchors'): + retain_anchors = config.getboolean('job_builder', + 'retain_anchors') + self.yamlparser['retain_anchors'] = retain_anchors + def validate(self): # Inform the user as to what is likely to happen, as they may specify # a real jenkins instance in test mode to get the plugin info to check diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index 493dac07..a55587b9 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -564,8 +564,9 @@ class LazyLoader(CustomLoader): return self._cls.from_yaml(self._loader, node) -def load(stream, **kwargs): - LocalAnchorLoader.reset_anchors() +def load(stream, retain_anchors=False, **kwargs): + if not retain_anchors: + LocalAnchorLoader.reset_anchors() return yaml.load(stream, functools.partial(LocalLoader, **kwargs)) diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index 69f1115a..8be0823a 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -98,7 +98,7 @@ class YamlParser(object): for path in fn: if not hasattr(path, 'read') and os.path.isdir(path): files_to_process.extend([os.path.join(path, f) - for f in os.listdir(path) + for f in sorted(os.listdir(path)) if (f.endswith('.yml') or f.endswith('.yaml'))]) else: @@ -134,7 +134,9 @@ class YamlParser(object): def _parse_fp(self, fp): # wrap provided file streams to ensure correct encoding used - data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path) + data = local_yaml.load(utils.wrap_stream(fp), + self.jjb_config.yamlparser['retain_anchors'], + search_path=self.path) if data: if not isinstance(data, list): raise JenkinsJobsException( diff --git a/jenkins_jobs/utils.py b/jenkins_jobs/utils.py index 385f87ba..4ce8ad2c 100644 --- a/jenkins_jobs/utils.py +++ b/jenkins_jobs/utils.py @@ -50,6 +50,9 @@ def recurse_path(root, excludes=None): relative = [e for e in excludes if os.path.sep in e and not os.path.isabs(e)] for root, dirs, files in os.walk(basepath, topdown=True): + # sort in-place to ensure dirnames are visited in alphabetical order + # a predictable order makes it easier to use the retain_anchors option + dirs.sort() dirs[:] = [ d for d in dirs if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns]) diff --git a/tests/localyaml/fixtures/custom_retain_anchors.yaml b/tests/localyaml/fixtures/custom_retain_anchors.yaml new file mode 100644 index 00000000..aa329235 --- /dev/null +++ b/tests/localyaml/fixtures/custom_retain_anchors.yaml @@ -0,0 +1,8 @@ +- project: + name: retain_anchors + jobs: + - retain_anchors + +- job-template: + name: retain_anchors + <<: *retain_anchors_defaults diff --git a/tests/localyaml/fixtures/custom_retain_anchors_include001.yaml b/tests/localyaml/fixtures/custom_retain_anchors_include001.yaml new file mode 100644 index 00000000..05242dd2 --- /dev/null +++ b/tests/localyaml/fixtures/custom_retain_anchors_include001.yaml @@ -0,0 +1,10 @@ +- retain_anchors_wrapper_defaults: &retain_anchors_wrapper_defaults + name: 'retain_anchors_wrapper_defaults' + wrappers: + - timeout: + timeout: 180 + fail: true + +- retain_anchors_defaults: &retain_anchors_defaults + name: 'retain_anchors_defaults' + <<: *retain_anchors_wrapper_defaults diff --git a/tests/localyaml/test_localyaml.py b/tests/localyaml/test_localyaml.py index 28460da6..788bb3b2 100644 --- a/tests/localyaml/test_localyaml.py +++ b/tests/localyaml/test_localyaml.py @@ -15,6 +15,7 @@ # under the License. import os +import yaml from testtools import ExpectedException from yaml.composer import ComposerError @@ -80,3 +81,39 @@ class TestCaseLocalYamlIncludeAnchors(base.BaseTestCase): jjb_config.validate() j = YamlParser(jjb_config) j.load_files([os.path.join(self.fixtures_path, f) for f in files]) + + +class TestCaseLocalYamlRetainAnchors(base.BaseTestCase): + + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + + def test_retain_anchors_default(self): + """ + Verify that anchors are NOT retained across files by default. + """ + + files = ["custom_retain_anchors_include001.yaml", + "custom_retain_anchors.yaml"] + + jjb_config = JJBConfig() + # use the default value for retain_anchors + jjb_config.validate() + j = YamlParser(jjb_config) + with ExpectedException(yaml.composer.ComposerError, + "found undefined alias.*"): + j.load_files([os.path.join(self.fixtures_path, f) for f in files]) + + def test_retain_anchors_enabled(self): + """ + Verify that anchors are retained across files if retain_anchors is + enabled in the config. + """ + + files = ["custom_retain_anchors_include001.yaml", + "custom_retain_anchors.yaml"] + + jjb_config = JJBConfig() + jjb_config.yamlparser['retain_anchors'] = True + jjb_config.validate() + j = YamlParser(jjb_config) + j.load_files([os.path.join(self.fixtures_path, f) for f in files]) |