Allow deep formatting of macro parameters

Prior to this patch macros serialized into a yaml document, a simple string
substitution was applied and then they were re-loaded into python. This
created several problems:
 - newlines are lost because the yaml loading was not respecting the
   newline settings of the original strings (Story: 1387060)
 - {obj:} notation was not working for macro parameters (Story: 2000254)
 - multiline strings passed into scripts will create a yaml syntax error
   when being re-loaded (unticketed)

This patch resolves these problems by replacing the previous interpolation
code with the more robust `deep_format` method. A side effect of this patch
is that there are now fewer disparate code paths for interpolating variables.

Change-Id: Ia06c98286537a50dc52c6e5e000b32ebc9e8ede2
Signed-off-by: Justin Stoller <justin.stoller@gmail.com>

5 files changed, 151 insertions, 6 deletions

diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py
index 087bc744..d1c9845a 100644
--- a/jenkins_jobs/registry.py
+++ b/jenkins_jobs/registry.py
@@ -19,10 +19,9 @@ import logging
 import operator
 import pkg_resources
 import re
-import yaml
 
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.formatter import CustomFormatter
+from jenkins_jobs.formatter import deep_format
 
 logger = logging.getLogger(__name__)
 
@@ -152,16 +151,15 @@ class ModuleRegistry(object):
             if template_data:
                 # Template data contains values that should be interpolated
                 # into the component definition
-                s = yaml.dump(component_data, default_flow_style=False)
                 allow_empty_variables = self.global_config \
                     and self.global_config.has_section('job_builder') \
                     and self.global_config.has_option(
                         'job_builder', 'allow_empty_variables') \
                     and self.global_config.getboolean(
                         'job_builder', 'allow_empty_variables')
-                s = CustomFormatter(
-                    allow_empty_variables).format(s, **template_data)
-                component_data = yaml.load(s)
+
+                component_data = deep_format(
+                    component_data, template_data, allow_empty_variables)
         else:
             # The component is a simple string name, eg "run-tests"
             name = component
diff --git a/tests/macros/fixtures/builder/multi-line-strings001.xml b/tests/macros/fixtures/builder/multi-line-strings001.xml
new file mode 100644
index 00000000..d1c7294a
--- /dev/null
+++ b/tests/macros/fixtures/builder/multi-line-strings001.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command># Set environment variables
+ENV_VAR_ONE=one
+ENV_VAR_TWO=2
+
+DIFFERENT_ENV_VAR=different
+
+
+# Debug output
+echo -e &quot;\n\n\n&quot;
+sort | env
+echo -e &quot;\n\n\n&quot;
+
+# Test Command
+bundle install
+bundle exec rake test
+
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>
diff --git a/tests/macros/fixtures/builder/multi-line-strings001.yaml b/tests/macros/fixtures/builder/multi-line-strings001.yaml
new file mode 100644
index 00000000..427acd0b
--- /dev/null
+++ b/tests/macros/fixtures/builder/multi-line-strings001.yaml
@@ -0,0 +1,27 @@
+- builder:
+    name: basic-builder
+    builders:
+      - shell: |
+          # Set environment variables
+          {env_setup}
+
+          # Debug output
+          echo -e "\n\n\n"
+          sort | env
+          echo -e "\n\n\n"
+
+          # Test Command
+          {test_command}
+
+- job:
+    name: my-job
+    builders:
+      - basic-builder:
+          env_setup: |
+            ENV_VAR_ONE=one
+            ENV_VAR_TWO=2
+
+            DIFFERENT_ENV_VAR=different
+          test_command: |
+            bundle install
+            bundle exec rake test
diff --git a/tests/macros/fixtures/scm/obj-in-scm-macro001.xml b/tests/macros/fixtures/scm/obj-in-scm-macro001.xml
new file mode 100644
index 00000000..accf2d1e
--- /dev/null
+++ b/tests/macros/fixtures/scm/obj-in-scm-macro001.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name>origin</name>
+        <refspec>+refs/heads/*:refs/remotes/origin/*</refspec>
+        <url>git://github.com/me/my-project.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>main-branch</name>
+      </hudson.plugins.git.BranchSpec>
+      <hudson.plugins.git.BranchSpec>
+        <name>other-branch</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <excludedUsers/>
+    <buildChooser class="hudson.plugins.git.util.DefaultBuildChooser"/>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <authorOrCommitter>false</authorOrCommitter>
+    <wipeOutWorkspace>false</wipeOutWorkspace>
+    <pruneBranches>false</pruneBranches>
+    <remotePoll>true</remotePoll>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <relativeTargetDir/>
+    <reference/>
+    <gitConfigName/>
+    <gitConfigEmail/>
+    <skipTag>true</skipTag>
+    <scmName/>
+    <useShallowClone>true</useShallowClone>
+    <ignoreNotifyCommit>false</ignoreNotifyCommit>
+    <extensions>
+      <hudson.plugins.git.extensions.impl.CleanCheckout/>
+      <hudson.plugins.git.extensions.impl.CleanBeforeCheckout/>
+      <hudson.plugins.git.extensions.impl.SubmoduleOption>
+        <disableSubmodules>false</disableSubmodules>
+        <recursiveSubmodules>true</recursiveSubmodules>
+        <trackingSubmodules>false</trackingSubmodules>
+        <timeout>10</timeout>
+      </hudson.plugins.git.extensions.impl.SubmoduleOption>
+    </extensions>
+  </scm>
+  <builders/>
+  <publishers/>
+  <buildWrappers/>
+</project>
diff --git a/tests/macros/fixtures/scm/obj-in-scm-macro001.yaml b/tests/macros/fixtures/scm/obj-in-scm-macro001.yaml
new file mode 100644
index 00000000..2bb48602
--- /dev/null
+++ b/tests/macros/fixtures/scm/obj-in-scm-macro001.yaml
@@ -0,0 +1,26 @@
+- scm:
+    name: multi-branch-git
+    scm:
+      - git:
+          url: 'git://github.com/{git_user}/{git_project}.git'
+          skip-tag: true
+          branches: '{obj:git_branches}'
+          wipe-workspace: false
+          shallow-clone: true
+          fastpoll: true
+          clean:
+            before: true
+            after: true
+          submodule:
+              recursive: true
+
+
+- job:
+    name: my-job
+    scm:
+      - multi-branch-git:
+          git_user: 'me'
+          git_project: 'my-project'
+          git_branches:
+            - main-branch
+            - other-branch