From bf88fe32c1e2a1f69a620aeb58ec8202c2703c0b Mon Sep 17 00:00:00 2001 From: Justin Stoller Date: Tue, 10 Nov 2015 15:40:43 -0800 Subject: Allow deep formatting of macro parameters Prior to this patch macros serialized into a yaml document, a simple string substitution was applied and then they were re-loaded into python. This created several problems: - newlines are lost because the yaml loading was not respecting the newline settings of the original strings (Story: 1387060) - {obj:} notation was not working for macro parameters (Story: 2000254) - multiline strings passed into scripts will create a yaml syntax error when being re-loaded (unticketed) This patch resolves these problems by replacing the previous interpolation code with the more robust `deep_format` method. A side effect of this patch is that there are now fewer disparate code paths for interpolating variables. Change-Id: Ia06c98286537a50dc52c6e5e000b32ebc9e8ede2 Signed-off-by: Justin Stoller --- jenkins_jobs/registry.py | 10 ++-- .../fixtures/builder/multi-line-strings001.xml | 35 +++++++++++++ .../fixtures/builder/multi-line-strings001.yaml | 27 ++++++++++ tests/macros/fixtures/scm/obj-in-scm-macro001.xml | 59 ++++++++++++++++++++++ tests/macros/fixtures/scm/obj-in-scm-macro001.yaml | 26 ++++++++++ 5 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 tests/macros/fixtures/builder/multi-line-strings001.xml create mode 100644 tests/macros/fixtures/builder/multi-line-strings001.yaml create mode 100644 tests/macros/fixtures/scm/obj-in-scm-macro001.xml create mode 100644 tests/macros/fixtures/scm/obj-in-scm-macro001.yaml diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index 087bc744..d1c9845a 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -19,10 +19,9 @@ import logging import operator import pkg_resources import re -import yaml from jenkins_jobs.errors import JenkinsJobsException -from jenkins_jobs.formatter import CustomFormatter +from jenkins_jobs.formatter import deep_format logger = logging.getLogger(__name__) @@ -152,16 +151,15 @@ class ModuleRegistry(object): if template_data: # Template data contains values that should be interpolated # into the component definition - s = yaml.dump(component_data, default_flow_style=False) allow_empty_variables = self.global_config \ and self.global_config.has_section('job_builder') \ and self.global_config.has_option( 'job_builder', 'allow_empty_variables') \ and self.global_config.getboolean( 'job_builder', 'allow_empty_variables') - s = CustomFormatter( - allow_empty_variables).format(s, **template_data) - component_data = yaml.load(s) + + component_data = deep_format( + component_data, template_data, allow_empty_variables) else: # The component is a simple string name, eg "run-tests" name = component diff --git a/tests/macros/fixtures/builder/multi-line-strings001.xml b/tests/macros/fixtures/builder/multi-line-strings001.xml new file mode 100644 index 00000000..d1c7294a --- /dev/null +++ b/tests/macros/fixtures/builder/multi-line-strings001.xml @@ -0,0 +1,35 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + # Set environment variables +ENV_VAR_ONE=one +ENV_VAR_TWO=2 + +DIFFERENT_ENV_VAR=different + + +# Debug output +echo -e "\n\n\n" +sort | env +echo -e "\n\n\n" + +# Test Command +bundle install +bundle exec rake test + + + + + + + diff --git a/tests/macros/fixtures/builder/multi-line-strings001.yaml b/tests/macros/fixtures/builder/multi-line-strings001.yaml new file mode 100644 index 00000000..427acd0b --- /dev/null +++ b/tests/macros/fixtures/builder/multi-line-strings001.yaml @@ -0,0 +1,27 @@ +- builder: + name: basic-builder + builders: + - shell: | + # Set environment variables + {env_setup} + + # Debug output + echo -e "\n\n\n" + sort | env + echo -e "\n\n\n" + + # Test Command + {test_command} + +- job: + name: my-job + builders: + - basic-builder: + env_setup: | + ENV_VAR_ONE=one + ENV_VAR_TWO=2 + + DIFFERENT_ENV_VAR=different + test_command: | + bundle install + bundle exec rake test diff --git a/tests/macros/fixtures/scm/obj-in-scm-macro001.xml b/tests/macros/fixtures/scm/obj-in-scm-macro001.xml new file mode 100644 index 00000000..accf2d1e --- /dev/null +++ b/tests/macros/fixtures/scm/obj-in-scm-macro001.xml @@ -0,0 +1,59 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + 2 + + + origin + +refs/heads/*:refs/remotes/origin/* + git://github.com/me/my-project.git + + + + + main-branch + + + other-branch + + + + + false + false + false + false + true + Default + + + + + + true + + true + false + + + + + false + true + false + 10 + + + + + + + diff --git a/tests/macros/fixtures/scm/obj-in-scm-macro001.yaml b/tests/macros/fixtures/scm/obj-in-scm-macro001.yaml new file mode 100644 index 00000000..2bb48602 --- /dev/null +++ b/tests/macros/fixtures/scm/obj-in-scm-macro001.yaml @@ -0,0 +1,26 @@ +- scm: + name: multi-branch-git + scm: + - git: + url: 'git://github.com/{git_user}/{git_project}.git' + skip-tag: true + branches: '{obj:git_branches}' + wipe-workspace: false + shallow-clone: true + fastpoll: true + clean: + before: true + after: true + submodule: + recursive: true + + +- job: + name: my-job + scm: + - multi-branch-git: + git_user: 'me' + git_project: 'my-project' + git_branches: + - main-branch + - other-branch -- cgit