Add tests for YamlParser and patch 2.6 minidom

Test full build job XML produced by the YamlParser object which exposed
differences in how the XML was being written on python 2.6 versus newer
versions of python.

Differences were whitespace only, but made testing and validating the
xml outputted across multiple versions of python difficult.

Code now patches the python 2.6 writexml method on the Element class in
the xml.dom.minidom module, with a version of the method taken from
python 2.7.5.

Removed the now obsolete regex that was reformatting the output on
python 2.6.

Change-Id: I26c100b6adfbcb9b197bb06cd162855adaaf24c5

5 files changed, 323 insertions, 8 deletions

diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py
index 2a89653f..0c969043 100644
--- a/jenkins_jobs/builder.py
+++ b/jenkins_jobs/builder.py
@@ -16,6 +16,7 @@
 # Manage jobs in Jenkins server
 
 import os
+import sys
 import hashlib
 import yaml
 import json
@@ -34,6 +35,41 @@ logger = logging.getLogger(__name__)
 MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
 
 
+# Python 2.6's minidom toprettyxml produces broken output by adding extraneous
+# whitespace around data. This patches the broken implementation with one taken
+# from 2.7
+def writexml(self, writer, indent="", addindent="", newl=""):
+    # indent = current indentation
+    # addindent = indentation to add to higher levels
+    # newl = newline string
+    writer.write(indent + "<" + self.tagName)
+
+    attrs = self._get_attributes()
+    a_names = attrs.keys()
+    a_names.sort()
+
+    for a_name in a_names:
+        writer.write(" %s=\"" % a_name)
+        minidom._write_data(writer, attrs[a_name].value)
+        writer.write("\"")
+    if self.childNodes:
+        writer.write(">")
+        if (len(self.childNodes) == 1 and
+                self.childNodes[0].nodeType == minidom.Node.TEXT_NODE):
+            self.childNodes[0].writexml(writer, '', '', '')
+        else:
+            writer.write(newl)
+            for node in self.childNodes:
+                node.writexml(writer, indent + addindent, addindent, newl)
+            writer.write(indent)
+        writer.write("</%s>%s" % (self.tagName, newl))
+    else:
+        writer.write("/>%s" % (newl))
+
+if sys.hexversion < 0x02070000:
+    minidom.Element.writexml = writexml
+
+
 def deep_format(obj, paramdict):
     """Apply the paramdict via str.format() to all string objects found within
        the supplied obj. Lists and dicts are traversed recursively."""
@@ -239,8 +275,8 @@ class YamlParser(object):
 
     def getXMLForJob(self, data):
         kind = data.get('project-type', 'freestyle')
-        data["description"] = data.get("description", "") + \
-            self.get_managed_string()
+        data["description"] = (data.get("description", "") +
+                               self.get_managed_string()).lstrip()
         for ep in pkg_resources.iter_entry_points(
                 group='jenkins_jobs.projects', name=kind):
             Mod = ep.load()
@@ -375,14 +411,9 @@ class XmlJob(object):
     def md5(self):
         return hashlib.md5(self.output()).hexdigest()
 
-    # Pretty printing ideas from
-    # http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python
-    pretty_text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL)
-
     def output(self):
         out = minidom.parseString(XML.tostring(self.xml))
-        out = out.toprettyxml(indent='  ', encoding='utf-8')
-        return self.pretty_text_re.sub('>\g<1></', out)
+        return out.toprettyxml(indent='  ', encoding='utf-8')
 
 
 class CacheStorage(object):
diff --git a/tests/yamlparser/__init__.py b/tests/yamlparser/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/yamlparser/__init__.py
diff --git a/tests/yamlparser/fixtures/complete001.xml b/tests/yamlparser/fixtures/complete001.xml
new file mode 100644
index 00000000..1a00ddfd
--- /dev/null
+++ b/tests/yamlparser/fixtures/complete001.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name>origin</name>
+        <refspec>+refs/heads/*:refs/remotes/origin/*</refspec>
+        <url>ssh://jenkins@review.openstack.org:29418/openstack-infra/jenkins-job-builder.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>origin/**</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <excludedUsers/>
+    <buildChooser class="com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTriggerBuildChooser"/>
+    <disableSubmodules>false</disableSubmodules>
+    <recursiveSubmodules>false</recursiveSubmodules>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <authorOrCommitter>false</authorOrCommitter>
+    <clean>true</clean>
+    <wipeOutWorkspace>true</wipeOutWorkspace>
+    <pruneBranches>true</pruneBranches>
+    <remotePoll>false</remotePoll>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <relativeTargetDir/>
+    <reference/>
+    <gitConfigName/>
+    <gitConfigEmail/>
+    <skipTag>false</skipTag>
+    <scmName/>
+    <useShallowClone>false</useShallowClone>
+    <browser class="hudson.plugins.git.browser.GitWeb">
+      <url>http://review.openstack.org/gitweb?p=openstack-infra/jenkins-job-builder.git</url>
+    </browser>
+  </scm>
+  <triggers class="vector">
+    <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+      <spec/>
+      <gerritProjects>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+          <compareType>PLAIN</compareType>
+          <pattern>openstack-infra/jenkins-job-builder</pattern>
+          <branches>
+            <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+              <compareType>ANT</compareType>
+              <pattern>**</pattern>
+            </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.Branch>
+          </branches>
+        </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.data.GerritProject>
+      </gerritProjects>
+      <skipVote>
+        <onSuccessful>false</onSuccessful>
+        <onFailed>false</onFailed>
+        <onUnstable>false</onUnstable>
+        <onNotBuilt>false</onNotBuilt>
+      </skipVote>
+      <silentMode>false</silentMode>
+      <escapeQuotes>true</escapeQuotes>
+      <noNameAndEmailParameters>false</noNameAndEmailParameters>
+      <dynamicTriggerConfiguration>False</dynamicTriggerConfiguration>
+      <triggerConfigURL/>
+      <triggerOnEvents>
+        <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.events.PluginPatchsetCreatedEvent/>
+      </triggerOnEvents>
+      <gerritBuildSuccessfulVerifiedValue>1</gerritBuildSuccessfulVerifiedValue>
+      <gerritBuildFailedVerifiedValue>-1</gerritBuildFailedVerifiedValue>
+      <buildStartMessage/>
+      <buildFailureMessage>This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.</buildFailureMessage>
+      <buildSuccessfulMessage/>
+      <buildUnstableMessage/>
+      <customUrl/>
+    </com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
+  </triggers>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/usr/bin/env python
+#
+print(&quot;Doing something cool with python&quot;)
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Doing somethiung cool&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/zsh
+echo &quot;Doing somethin cool with zsh&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Ant>
+          <targets>target1 target2</targets>
+          <antName>default</antName>
+        </hudson.tasks.Ant>
+        <EnvInjectBuilder>
+          <info>
+            <propertiesFilePath>example.prop</propertiesFilePath>
+            <propertiesContent>EXAMPLE=foo-bar</propertiesContent>
+          </info>
+        </EnvInjectBuilder>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+    <com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+      <includes>file1,file2*.txt</includes>
+      <excludes>file2bad.txt</excludes>
+      <flatten>false</flatten>
+      <includeAntExcludes>false</includeAntExcludes>
+      <relativeTo>userContent</relativeTo>
+      <hudsonHomeRelative>false</hudsonHomeRelative>
+    </com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+  </buildWrappers>
+</project>
diff --git a/tests/yamlparser/fixtures/complete001.yaml b/tests/yamlparser/fixtures/complete001.yaml
new file mode 100644
index 00000000..686b495a
--- /dev/null
+++ b/tests/yamlparser/fixtures/complete001.yaml
@@ -0,0 +1,92 @@
+- wrapper:
+    name: timeout-wrapper
+    wrappers:
+      - timeout:
+          fail: true
+          elastic-percentage: 150
+          elastic-default-timeout: 90
+          type: elastic
+
+- wrapper:
+    name: pre-scm-shell-ant
+    wrappers:
+      - pre-scm-buildstep:
+          - shell: |
+              #!/bin/bash
+              echo "Doing somethiung cool"
+          - shell: |
+              #!/bin/zsh
+              echo "Doing somethin cool with zsh"
+          - ant: "target1 target2"
+            ant-name: "Standard Ant"
+          - inject:
+               properties-file: example.prop
+               properties-content: EXAMPLE=foo-bar
+
+- wrapper:
+    name: copy-files
+    wrappers:
+      - copy-to-slave:
+          includes:
+            - file1
+            - file2*.txt
+          excludes:
+            - file2bad.txt
+
+- trigger:
+    name: gerrit-review
+    triggers:
+      - gerrit:
+         triggerOnPatchsetUploadedEvent: true
+         triggerOnChangeMergedEvent: false
+         triggerOnRefUpdatedEvent: false
+         triggerOnCommentAddedEvent: false
+         overrideVotes: true
+         gerritBuildSuccessfulVerifiedValue: 1
+         gerritBuildFailedVerifiedValue: -1
+         projects:
+           - projectCompareType: 'PLAIN'
+             projectPattern: '{project_pattern}'
+             branchCompareType: 'ANT'
+             branchPattern: '**'
+         failureMessage: 'This change was unable to be automatically merged with the current state of the repository. Please rebase your change and upload a new patchset.'
+
+- scm:
+    name: gerrit-scm
+    scm:
+      - git:
+         url: ssh://jenkins@review.openstack.org:29418/{project_pattern}.git
+         branches:
+          - origin/**
+         name: origin
+         prune: true
+         clean: true
+         browser: gitweb
+         browser-url: http://review.openstack.org/gitweb?p={project_pattern}.git
+         choosing-strategy: gerrit
+
+- project:
+    name: test
+    version:
+        - 1.2
+    jobs:
+        - 'build_myproject_{version}'
+
+- job-template:
+    name: 'build_myproject_{version}'
+    scm:
+      - gerrit-scm:
+         project_pattern: openstack-infra/jenkins-job-builder
+    triggers:
+      - gerrit-review:
+         project_pattern: openstack-infra/jenkins-job-builder
+    wrappers:
+      - timeout-wrapper
+      - pre-scm-shell-ant
+      - copy-files
+    builders:
+      - shell: |
+         #!/usr/bin/env python
+         #
+         print("Doing something cool with python")
+
diff --git a/tests/yamlparser/test_yamlparser.py b/tests/yamlparser/test_yamlparser.py
new file mode 100644
index 00000000..b70b7bbf
--- /dev/null
+++ b/tests/yamlparser/test_yamlparser.py
@@ -0,0 +1,55 @@
+# Joint copyright:
+#  - Copyright 2012,2013 Wikimedia Foundation
+#  - Copyright 2012,2013 Antoine "hashar" Musso
+#  - Copyright 2013 Arnaud Fabre
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+from testtools import TestCase
+from testscenarios.testcase import TestWithScenarios
+from tests.base import get_scenarios, BaseTestCase
+import doctest
+import testtools
+from jenkins_jobs.builder import YamlParser
+
+
+class TestCaseModuleYamlInclude(TestWithScenarios, TestCase, BaseTestCase):
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+    scenarios = get_scenarios(fixtures_path)
+
+    def test_yaml_snippet(self):
+        if not self.xml_filename or not self.yaml_filename:
+            return
+
+        xml_filepath = os.path.join(self.fixtures_path, self.xml_filename)
+        expected_xml = u"%s" % open(xml_filepath, 'r').read()
+
+        yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
+
+        parser = YamlParser()
+        parser.parse(yaml_filepath)
+
+        # Generate the XML tree
+        parser.generateXML()
+
+        # Prettify generated XML
+        pretty_xml = parser.jobs[0].output()
+
+        self.assertThat(
+            pretty_xml,
+            testtools.matchers.DocTestMatches(expected_xml,
+                                              doctest.ELLIPSIS |
+                                              doctest.NORMALIZE_WHITESPACE |
+                                              doctest.REPORT_NDIFF)
+        )