Fix multibyte character problem

Fixed problem that causes a UnicodeEncodeError when use multi-byte characters in the job.yml.

Change-Id: Ie715c827a794e73fae11cdca079ea80cfb8c280d

42 files changed, 61 insertions, 40 deletions

diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py
index 5d7a81f9..2a89653f 100644
--- a/jenkins_jobs/builder.py
+++ b/jenkins_jobs/builder.py
@@ -381,7 +381,7 @@ class XmlJob(object):
 
     def output(self):
         out = minidom.parseString(XML.tostring(self.xml))
-        out = out.toprettyxml(indent='  ')
+        out = out.toprettyxml(indent='  ', encoding='utf-8')
         return self.pretty_text_re.sub('>\g<1></', out)
 
 
diff --git a/tests/base.py b/tests/base.py
index 34c6ba28..3daeb77e 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -17,6 +17,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import codecs
 import os
 import re
 import doctest
@@ -61,7 +62,7 @@ class BaseTestCase(object):
     def __read_content(self):
         # Read XML content, assuming it is unicode encoded
         xml_filepath = os.path.join(self.fixtures_path, self.xml_filename)
-        xml_content = u"%s" % open(xml_filepath, 'r').read()
+        xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()
 
         yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
         with file(yaml_filepath, 'r') as yaml_file:
@@ -83,7 +84,8 @@ class BaseTestCase(object):
         pub.gen_xml(parser, xml_project, yaml_content)
 
         # Prettify generated XML
-        pretty_xml = XmlJob(xml_project, 'fixturejob').output()
+        pretty_xml = unicode(XmlJob(xml_project, 'fixturejob').output(),
+                             'utf-8')
 
         self.assertThat(
             pretty_xml,
diff --git a/tests/builders/fixtures/maven-target-doc.xml b/tests/builders/fixtures/maven-target-doc.xml
index 55173a2b..b8432598 100644
--- a/tests/builders/fixtures/maven-target-doc.xml
+++ b/tests/builders/fixtures/maven-target-doc.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <builders>
     <hudson.tasks.Maven>
diff --git a/tests/builders/fixtures/maven-target001.xml b/tests/builders/fixtures/maven-target001.xml
index dbb6490f..328eb0e8 100644
--- a/tests/builders/fixtures/maven-target001.xml
+++ b/tests/builders/fixtures/maven-target001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <builders>
     <hudson.tasks.Maven>
diff --git a/tests/builders/fixtures/trigger-builds001.xml b/tests/builders/fixtures/trigger-builds001.xml
index 5afabc73..f083ed86 100644
--- a/tests/builders/fixtures/trigger-builds001.xml
+++ b/tests/builders/fixtures/trigger-builds001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <builders>
     <hudson.plugins.parameterizedtrigger.TriggerBuilder>
diff --git a/tests/general/fixtures/assigned-node001.xml b/tests/general/fixtures/assigned-node001.xml
index 37a87ba9..ebbf8c7c 100644
--- a/tests/general/fixtures/assigned-node001.xml
+++ b/tests/general/fixtures/assigned-node001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <actions/>
   <keepDependencies>false</keepDependencies>
diff --git a/tests/general/fixtures/assigned-node002.xml b/tests/general/fixtures/assigned-node002.xml
index 01aff283..1505db51 100644
--- a/tests/general/fixtures/assigned-node002.xml
+++ b/tests/general/fixtures/assigned-node002.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <actions/>
   <keepDependencies>false</keepDependencies>
diff --git a/tests/notifications/fixtures/http-endpoint001.xml b/tests/notifications/fixtures/http-endpoint001.xml
index cc3ddc18..79ba0ef8 100644
--- a/tests/notifications/fixtures/http-endpoint001.xml
+++ b/tests/notifications/fixtures/http-endpoint001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <properties>
     <com.tikal.hudson.plugins.notification.HudsonNotificationProperty>
diff --git a/tests/parameters/fixtures/dynamic-choice-param001.xml b/tests/parameters/fixtures/dynamic-choice-param001.xml
index 7220a733..f057e8e4 100644
--- a/tests/parameters/fixtures/dynamic-choice-param001.xml
+++ b/tests/parameters/fixtures/dynamic-choice-param001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <properties>
     <hudson.model.ParametersDefinitionProperty>
diff --git a/tests/properties/fixtures/batch-task.xml b/tests/properties/fixtures/batch-task.xml
index 9866cc66..9740f8a2 100644
--- a/tests/properties/fixtures/batch-task.xml
+++ b/tests/properties/fixtures/batch-task.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <properties>
     <hudson.plugins.batch__task.BatchTaskProperty>
diff --git a/tests/properties/fixtures/throttle001.xml b/tests/properties/fixtures/throttle001.xml
index de321a37..073a9f8f 100644
--- a/tests/properties/fixtures/throttle001.xml
+++ b/tests/properties/fixtures/throttle001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <properties>
     <hudson.plugins.throttleconcurrents.ThrottleJobProperty>
diff --git a/tests/publishers/fixtures/README b/tests/publishers/fixtures/README
index dc505aba..225f630f 100644
--- a/tests/publishers/fixtures/README
+++ b/tests/publishers/fixtures/README
@@ -10,5 +10,5 @@ Each yaml file MUST have a corresponding xml file.
 
 Once the YAML file has been parsed, it is prettify using python minidom
 which also means that:
-- your XML file must start with: <?xml version="1.0" ?>
+- your XML file must start with: <?xml version="1.0" encoding="utf-8"?>
 - self closing elements do not contains space eg: <element/>
diff --git a/tests/publishers/fixtures/archive001.xml b/tests/publishers/fixtures/archive001.xml
index 73c52460..24bc4d62 100644
--- a/tests/publishers/fixtures/archive001.xml
+++ b/tests/publishers/fixtures/archive001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
     <publishers>
         <hudson.tasks.ArtifactArchiver>
diff --git a/tests/publishers/fixtures/blame001.xml b/tests/publishers/fixtures/blame001.xml
index ef2ff74d..d07c9503 100644
--- a/tests/publishers/fixtures/blame001.xml
+++ b/tests/publishers/fixtures/blame001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.blame__upstream__commiters.BlameUpstreamCommitersPublisher/>
diff --git a/tests/publishers/fixtures/build001.xml b/tests/publishers/fixtures/build001.xml
index 60ada7f5..fc7718b5 100644
--- a/tests/publishers/fixtures/build001.xml
+++ b/tests/publishers/fixtures/build001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.build__publisher.BuildPublisher>
diff --git a/tests/publishers/fixtures/cloverphp001.xml b/tests/publishers/fixtures/cloverphp001.xml
index f572450d..6af0e77a 100644
--- a/tests/publishers/fixtures/cloverphp001.xml
+++ b/tests/publishers/fixtures/cloverphp001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <org.jenkinsci.plugins.cloverphp.CloverPHPPublisher>
diff --git a/tests/publishers/fixtures/cloverphp002.xml b/tests/publishers/fixtures/cloverphp002.xml
index 5f4db9f3..33ad6610 100644
--- a/tests/publishers/fixtures/cloverphp002.xml
+++ b/tests/publishers/fixtures/cloverphp002.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <org.jenkinsci.plugins.cloverphp.CloverPHPPublisher>
diff --git a/tests/publishers/fixtures/cloverphp003.xml b/tests/publishers/fixtures/cloverphp003.xml
index a9255397..4fc565d6 100644
--- a/tests/publishers/fixtures/cloverphp003.xml
+++ b/tests/publishers/fixtures/cloverphp003.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <org.jenkinsci.plugins.cloverphp.CloverPHPPublisher>
diff --git a/tests/publishers/fixtures/description-setter.xml b/tests/publishers/fixtures/description-setter001.xml
index f73d5c00..1cbb41fd 100644
--- a/tests/publishers/fixtures/description-setter.xml
+++ b/tests/publishers/fixtures/description-setter001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
diff --git a/tests/publishers/fixtures/description-setter.yaml b/tests/publishers/fixtures/description-setter001.yaml
index f1adf0fb..f1adf0fb 100644
--- a/tests/publishers/fixtures/description-setter.yaml
+++ b/tests/publishers/fixtures/description-setter001.yaml
diff --git a/tests/publishers/fixtures/description-setter002.xml b/tests/publishers/fixtures/description-setter002.xml
new file mode 100644
index 00000000..23f5d912
--- /dev/null
+++ b/tests/publishers/fixtures/description-setter002.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <publishers>
+    <hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
+      <regexp>.*(&lt;a href=.*a&gt;)</regexp>
+      <regexpForFailed>.*(&lt;a href=.*a&gt;)</regexpForFailed>
+      <description>こんにちは</description>
+      <descriptionForFailed>さようなら</descriptionForFailed>
+      <setForMatrix>true</setForMatrix>
+    </hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
+  </publishers>
+</project>
diff --git a/tests/publishers/fixtures/description-setter002.yaml b/tests/publishers/fixtures/description-setter002.yaml
new file mode 100644
index 00000000..7e6e7018
--- /dev/null
+++ b/tests/publishers/fixtures/description-setter002.yaml
@@ -0,0 +1,7 @@
+publishers:
+  - description-setter:
+      regexp: ".*(<a href=.*a>)"
+      regexp-for-failed: ".*(<a href=.*a>)"
+      description: "こんにちは"
+      description-for-failed: "さようなら"
+      set-for-matrix: true
diff --git a/tests/publishers/fixtures/emotional-jenkins.xml b/tests/publishers/fixtures/emotional-jenkins.xml
index 0630e84e..c1469043 100644
--- a/tests/publishers/fixtures/emotional-jenkins.xml
+++ b/tests/publishers/fixtures/emotional-jenkins.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <org.jenkinsci.plugins.emotional__jenkins.EmotionalJenkinsPublisher/>
diff --git a/tests/publishers/fixtures/git001.xml b/tests/publishers/fixtures/git001.xml
index 63dcdfda..866b0575 100644
--- a/tests/publishers/fixtures/git001.xml
+++ b/tests/publishers/fixtures/git001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.git.GitPublisher>
diff --git a/tests/publishers/fixtures/github-notifier.xml b/tests/publishers/fixtures/github-notifier.xml
index a4501b40..3f20f33b 100644
--- a/tests/publishers/fixtures/github-notifier.xml
+++ b/tests/publishers/fixtures/github-notifier.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <com.cloudbees.jenkins.GitHubCommitNotifier/>
diff --git a/tests/publishers/fixtures/mavendeploy001.xml b/tests/publishers/fixtures/mavendeploy001.xml
index 6db0030e..327dddc0 100644
--- a/tests/publishers/fixtures/mavendeploy001.xml
+++ b/tests/publishers/fixtures/mavendeploy001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.maven.RedeployPublisher>
diff --git a/tests/publishers/fixtures/pipeline001.xml b/tests/publishers/fixtures/pipeline001.xml
index cb448fbb..4b9295c4 100644
--- a/tests/publishers/fixtures/pipeline001.xml
+++ b/tests/publishers/fixtures/pipeline001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <au.com.centrumsystems.hudson.plugin.buildpipeline.trigger.BuildPipelineTrigger>
diff --git a/tests/publishers/fixtures/pipeline002.xml b/tests/publishers/fixtures/pipeline002.xml
index 33eca2ab..a5c7dc9f 100644
--- a/tests/publishers/fixtures/pipeline002.xml
+++ b/tests/publishers/fixtures/pipeline002.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <au.com.centrumsystems.hudson.plugin.buildpipeline.trigger.BuildPipelineTrigger>
diff --git a/tests/publishers/fixtures/plot001.xml b/tests/publishers/fixtures/plot001.xml
index 382d5b71..abd36d0a 100644
--- a/tests/publishers/fixtures/plot001.xml
+++ b/tests/publishers/fixtures/plot001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.plot.PlotPublisher>
diff --git a/tests/publishers/fixtures/plot002.xml b/tests/publishers/fixtures/plot002.xml
index c67956aa..1eeffb17 100644
--- a/tests/publishers/fixtures/plot002.xml
+++ b/tests/publishers/fixtures/plot002.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.plot.PlotPublisher>
diff --git a/tests/publishers/fixtures/plot003.xml b/tests/publishers/fixtures/plot003.xml
index d13c93bd..51c4ca94 100644
--- a/tests/publishers/fixtures/plot003.xml
+++ b/tests/publishers/fixtures/plot003.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <hudson.plugins.plot.PlotPublisher>
diff --git a/tests/publishers/fixtures/scp001.xml b/tests/publishers/fixtures/scp001.xml
index 78af6776..fafa32f6 100644
--- a/tests/publishers/fixtures/scp001.xml
+++ b/tests/publishers/fixtures/scp001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <be.certipost.hudson.plugin.SCPRepositoryPublisher>
diff --git a/tests/publishers/fixtures/stash001.xml b/tests/publishers/fixtures/stash001.xml
index 9260c641..a07da18a 100644
--- a/tests/publishers/fixtures/stash001.xml
+++ b/tests/publishers/fixtures/stash001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <org.jenkinsci.plugins.stashNotifier.StashNotifier>
diff --git a/tests/publishers/fixtures/xunit001.xml b/tests/publishers/fixtures/xunit001.xml
index 2535b624..7d6e5b92 100644
--- a/tests/publishers/fixtures/xunit001.xml
+++ b/tests/publishers/fixtures/xunit001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <publishers>
     <xunit>
diff --git a/tests/scm/fixtures/git-shallow-clone01.xml b/tests/scm/fixtures/git-shallow-clone01.xml
index d66862cd..dd001541 100644
--- a/tests/scm/fixtures/git-shallow-clone01.xml
+++ b/tests/scm/fixtures/git-shallow-clone01.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <scm class="hudson.plugins.git.GitSCM">
     <configVersion>2</configVersion>
diff --git a/tests/scm/fixtures/git-shallow-clone02.xml b/tests/scm/fixtures/git-shallow-clone02.xml
index c1735f21..57fe8d7a 100644
--- a/tests/scm/fixtures/git-shallow-clone02.xml
+++ b/tests/scm/fixtures/git-shallow-clone02.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <scm class="hudson.plugins.git.GitSCM">
     <configVersion>2</configVersion>
diff --git a/tests/scm/fixtures/git-shallow-clone03.xml b/tests/scm/fixtures/git-shallow-clone03.xml
index c1735f21..57fe8d7a 100644
--- a/tests/scm/fixtures/git-shallow-clone03.xml
+++ b/tests/scm/fixtures/git-shallow-clone03.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <scm class="hudson.plugins.git.GitSCM">
     <configVersion>2</configVersion>
diff --git a/tests/scm/fixtures/gitlab.xml b/tests/scm/fixtures/gitlab.xml
index 4623bf9e..be772a9d 100644
--- a/tests/scm/fixtures/gitlab.xml
+++ b/tests/scm/fixtures/gitlab.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <scm class="hudson.plugins.git.GitSCM">
     <configVersion>2</configVersion>
diff --git a/tests/scm/fixtures/repo001.xml b/tests/scm/fixtures/repo001.xml
index bed34c01..94be6b5c 100644
--- a/tests/scm/fixtures/repo001.xml
+++ b/tests/scm/fixtures/repo001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <scm class="hudson.plugins.repo.RepoScm">
     <manifestRepositoryUrl>https://example.com/project/</manifestRepositoryUrl>
diff --git a/tests/triggers/fixtures/gerrit001.xml b/tests/triggers/fixtures/gerrit001.xml
index 53fadaae..ebde0064 100644
--- a/tests/triggers/fixtures/gerrit001.xml
+++ b/tests/triggers/fixtures/gerrit001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <triggers class="vector">
     <com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritTrigger>
diff --git a/tests/triggers/fixtures/github-pull-request.xml b/tests/triggers/fixtures/github-pull-request.xml
index 63f0ef8c..e4e19b0d 100644
--- a/tests/triggers/fixtures/github-pull-request.xml
+++ b/tests/triggers/fixtures/github-pull-request.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <triggers class="vector">
     <org.jenkinsci.plugins.ghprb.GhprbTrigger>
diff --git a/tests/wrappers/fixtures/env-file001.xml b/tests/wrappers/fixtures/env-file001.xml
index d6cb26a5..0fc51598 100644
--- a/tests/wrappers/fixtures/env-file001.xml
+++ b/tests/wrappers/fixtures/env-file001.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <project>
   <buildWrappers>
     <hudson.plugins.envfile.EnvFileBuildWrapper>