Use yaml local tags to support including files

Add support for local tags which are application specific to allow
including of other yaml files or code from scripts. Allows for code to
be maintained and tested as seperate files, as well as reduces
duplication of yaml code that cannot be macro'ed or easily templated by
including it from a common file.

Adds support for the following tags:
  'include' - load file as yaml code
  'include-raw' - load file as data (for scripts)
  'include-raw-escaped' - load file as data with escaping braces '{}'
      as default for use with job-templates

Use configuration file options to provide a search path for the files.

- Test behaviour of yaml tags independent of any XML generation
  by comparing json result of yaml parsing to verify that certain
  tags do/don't recall the yaml.load() method.
- Add examples for the include tags via addition tests for YamlParser
  class

Inspired by
http://stackoverflow.com/questions/528281/how-can-i-include-an-yaml-file-inside-another

Change-Id: Ib90a07043112d4739d6529ceddbc9817668bcec0

30 files changed, 755 insertions, 24 deletions

diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst
index f640719c..c84c89e3 100644
--- a/doc/source/configuration.rst
+++ b/doc/source/configuration.rst
@@ -302,6 +302,12 @@ For example:
 .. literalinclude::  /../../tests/yamlparser/fixtures/custom_distri.yaml
 
 
+Custom Yaml Tags
+----------------
+
+.. automodule:: jenkins_jobs.local_yaml
+
+
 Modules
 -------
 
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
index 302e0a49..5d2312d3 100644
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -65,6 +65,11 @@ job_builder section
   When this option is set to True, that behavior changes and it will only
   overwrite the description if you specified it in the yaml. False by default.
 
+**include_path**
+  (Optional) Can be set to a ':' delimited list of paths, which jenkins
+  job builder will search for any files specified by the custom application
+  yaml tags 'include', 'include-raw' and 'include-raw-escaped'.
+
 jenkins section
 ^^^^^^^^^^^^^^^
 
diff --git a/etc/jenkins_jobs.ini-sample b/etc/jenkins_jobs.ini-sample
index 6ea02b54..62fe9a87 100644
--- a/etc/jenkins_jobs.ini-sample
+++ b/etc/jenkins_jobs.ini-sample
@@ -1,6 +1,7 @@
 [job_builder]
 ignore_cache=True
 keep_descriptions=False
+include_path=.:scripts:~/git/
 
 [jenkins]
 user=jenkins
diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py
index ef0cb0a2..5b86a677 100644
--- a/jenkins_jobs/builder.py
+++ b/jenkins_jobs/builder.py
@@ -31,6 +31,7 @@ import copy
 import itertools
 import fnmatch
 from jenkins_jobs.errors import JenkinsJobsException
+import local_yaml
 
 logger = logging.getLogger(__name__)
 MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
@@ -122,9 +123,15 @@ class YamlParser(object):
         self.jobs = []
         self.config = config
         self.registry = ModuleRegistry(self.config)
+        self.path = ["."]
+        if self.config:
+            if config.has_section('job_builder') and \
+                    config.has_option('job_builder', 'include_path'):
+                self.path = config.get('job_builder',
+                                       'include_path').split(':')
 
     def parse_fp(self, fp):
-        data = yaml.load(fp)
+        data = local_yaml.load(fp, search_path=self.path)
         if data:
             if not isinstance(data, list):
                 raise JenkinsJobsException(
diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py
new file mode 100644
index 00000000..f67e52bd
--- /dev/null
+++ b/jenkins_jobs/local_yaml.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python
+# Copyright (C) 2013 Hewlett-Packard.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Provides local yaml parsing classes and extend yaml module
+
+"""Custom application specific yamls tags are supported to provide
+enhancements when reading yaml configuration.
+
+These allow inclusion of arbitrary files as a method of having blocks of data
+managed separately to the yaml job configurations. A specific usage of this is
+inlining scripts contained in separate files, although such tags may also be
+used to simplify usage of macros or job templates.
+
+The tag ``!include`` will treat the following string as file which should be
+parsed as yaml configuration data.
+
+Example:
+
+    .. literalinclude:: /../../tests/localyaml/fixtures/include001.yaml
+
+
+The tag ``!include-raw`` will treat the following file as a data blob, which
+should be read into the calling yaml construct without any further parsing.
+Any data in a file included through this tag, will be treated as string data.
+
+Example:
+
+    .. literalinclude:: /../../tests/localyaml/fixtures/include-raw001.yaml
+
+
+The tag ``!include-raw-escape`` treats the given file as a data blob, which
+should be escaped before being read in as string data. This allows
+job-templates to use this tag to include scripts from files without
+needing to escape braces in the original file.
+
+
+Example:
+
+    .. literalinclude::
+        /../../tests/localyaml/fixtures/include-raw-escaped001.yaml
+
+"""
+
+import functools
+import logging
+import re
+import os
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class LocalLoader(yaml.Loader):
+    """Subclass for yaml.Loader which handles the local tags 'include',
+    'include-raw' and 'include-raw-escaped' to specify a file to include data
+    from and whether to parse it as additional yaml, treat it as a data blob
+    or additionally escape the data contained. These are specified in yaml
+    files by "!include path/to/file.yaml".
+
+    Constructor access a list of search paths to look under for the given
+    file following each tag, taking the first match found. Search path by
+    default will include the same directory as the yaml file and the current
+    working directory.
+
+
+    Loading::
+
+        # use the load function provided in this module
+        import local_yaml
+        data = local_yaml.load(open(fn))
+
+
+        # Loading by providing the alternate class to the default yaml load
+        from local_yaml import LocalLoader
+        data = yaml.load(open(fn), LocalLoader)
+
+        # Loading with a search path
+        from local_yaml import LocalLoader
+        import functools
+        data = yaml.load(open(fn), functools.partial(LocalLoader,
+                        search_path=['path']))
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        # make sure to pop off any local settings before passing to
+        # the parent constructor as any unknown args may cause errors.
+        self.search_path = set()
+        if 'search_path' in kwargs:
+            for p in kwargs.pop('search_path'):
+                logger.debug("Adding '{0}' to search path for include tags"
+                             .format(p))
+                self.search_path.add(os.path.normpath(p))
+
+        if 'escape_callback' in kwargs:
+            self._escape = kwargs.pop('escape_callback')
+
+        super(LocalLoader, self).__init__(*args, **kwargs)
+
+        # Add tag constructors
+        self.add_constructor('!include', self._include_tag)
+        self.add_constructor('!include-raw', self._include_raw_tag)
+        self.add_constructor('!include-raw-escape',
+                             self._include_raw_escape_tag)
+
+        if isinstance(self.stream, file):
+            self.search_path.add(os.path.normpath(
+                os.path.dirname(self.stream.name)))
+        self.search_path.add(os.path.normpath(os.path.curdir))
+
+    def _find_file(self, filename):
+        for dirname in self.search_path:
+            candidate = os.path.expanduser(os.path.join(dirname, filename))
+            if os.path.isfile(candidate):
+                logger.info("Including file '{0}' from path '{0}'"
+                            .format(filename, dirname))
+                return candidate
+        return filename
+
+    def _include_tag(self, loader, node):
+        filename = self._find_file(loader.construct_yaml_str(node))
+        with open(filename, 'r') as f:
+            data = yaml.load(f, functools.partial(LocalLoader,
+                                                  search_path=self.search_path
+                                                  ))
+        return data
+
+    def _include_raw_tag(self, loader, node):
+        filename = self._find_file(loader.construct_yaml_str(node))
+        try:
+            with open(filename, 'r') as f:
+                data = f.read()
+        except:
+            logger.error("Failed to include file using search path: '{0}'"
+                         .format(':'.join(self.search_path)))
+            raise
+        return data
+
+    def _include_raw_escape_tag(self, loader, node):
+        return self._escape(self._include_raw_tag(loader, node))
+
+    def _escape(self, data):
+        return re.sub(r'({|})', r'\1\1', data)
+
+
+def load(stream, **kwargs):
+    return yaml.load(stream, functools.partial(LocalLoader, **kwargs))
diff --git a/tests/base.py b/tests/base.py
index 2b73e2f2..364889b4 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -22,11 +22,12 @@ import logging
 import os
 import re
 import doctest
+import json
 import operator
 import testtools
 import xml.etree.ElementTree as XML
 from ConfigParser import ConfigParser
-import yaml
+import jenkins_jobs.local_yaml as yaml
 from jenkins_jobs.builder import XmlJob, YamlParser, ModuleRegistry
 from jenkins_jobs.modules import (project_flow,
                                   project_matrix,
@@ -34,30 +35,33 @@ from jenkins_jobs.modules import (project_flow,
                                   project_multijob)
 
 
-def get_scenarios(fixtures_path):
+def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml'):
     """Returns a list of scenarios, each scenario being described
-    by two parameters (yaml and xml filenames).
-        - content of the fixture .xml file (aka expected)
+    by two parameters (yaml and xml filenames by default).
+        - content of the fixture output file (aka expected)
     """
     scenarios = []
     files = os.listdir(fixtures_path)
-    yaml_files = [f for f in files if re.match(r'.*\.yaml$', f)]
+    input_files = [f for f in files if re.match(r'.*\.{0}$'.format(in_ext), f)]
 
-    for yaml_filename in yaml_files:
-        xml_candidate = re.sub(r'\.yaml$', '.xml', yaml_filename)
-        # Make sure the yaml file has a xml counterpart
-        if xml_candidate not in files:
+    for input_filename in input_files:
+        output_candidate = re.sub(r'\.{0}$'.format(in_ext),
+                                  '.{0}'.format(out_ext), input_filename)
+        # Make sure the input file has a output counterpart
+        if output_candidate not in files:
             raise Exception(
-                "No XML file named '%s' to match "
-                "YAML file '%s'" % (xml_candidate, yaml_filename))
-        conf_candidate = re.sub(r'\.yaml$', '.conf', yaml_filename)
+                "No {0} file named '{1}' to match {2} file '{3}'"
+                .format(out_ext.toupper(), output_candidate,
+                        in_ext.toupper(), input_filename))
+
+        conf_candidate = re.sub(r'\.yaml$', '.conf', input_filename)
         # If present, add the configuration file
         if conf_candidate not in files:
             conf_candidate = None
 
-        scenarios.append((yaml_filename, {
-            'yaml_filename': yaml_filename,
-            'xml_filename': xml_candidate,
+        scenarios.append((input_filename, {
+            'in_filename': input_filename,
+            'out_filename': output_candidate,
             'conf_filename': conf_candidate,
         }))
 
@@ -74,22 +78,22 @@ class BaseTestCase(object):
 
     logging.basicConfig()
 
-    def __read_content(self):
+    def _read_content(self):
         # Read XML content, assuming it is unicode encoded
-        xml_filepath = os.path.join(self.fixtures_path, self.xml_filename)
+        xml_filepath = os.path.join(self.fixtures_path, self.out_filename)
         xml_content = u"%s" % codecs.open(xml_filepath, 'r', 'utf-8').read()
 
-        yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
+        yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
         with file(yaml_filepath, 'r') as yaml_file:
             yaml_content = yaml.load(yaml_file)
 
         return (yaml_content, xml_content)
 
     def test_yaml_snippet(self):
-        if not self.xml_filename or not self.yaml_filename:
+        if not self.out_filename or not self.in_filename:
             return
 
-        yaml_content, expected_xml = self.__read_content()
+        yaml_content, expected_xml = self._read_content()
         project = None
         if ('project-type' in yaml_content):
             if (yaml_content['project-type'] == "maven"):
@@ -126,13 +130,13 @@ class BaseTestCase(object):
 
 class SingleJobTestCase(BaseTestCase):
     def test_yaml_snippet(self):
-        if not self.xml_filename or not self.yaml_filename:
+        if not self.out_filename or not self.in_filename:
             return
 
-        xml_filepath = os.path.join(self.fixtures_path, self.xml_filename)
+        xml_filepath = os.path.join(self.fixtures_path, self.out_filename)
         expected_xml = u"%s" % open(xml_filepath, 'r').read()
 
-        yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
+        yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
 
         if self.conf_filename:
             config = ConfigParser()
@@ -159,3 +163,23 @@ class SingleJobTestCase(BaseTestCase):
                                               doctest.NORMALIZE_WHITESPACE |
                                               doctest.REPORT_NDIFF)
         )
+
+
+class JsonTestCase(BaseTestCase):
+
+    def test_yaml_snippet(self):
+        if not self.out_filename or not self.in_filename:
+            return
+
+        yaml_content, expected_json = self._read_content()
+
+        pretty_json = json.dumps(yaml_content, indent=4,
+                                 separators=(',', ': '))
+
+        self.assertThat(
+            pretty_json,
+            testtools.matchers.DocTestMatches(expected_json,
+                                              doctest.ELLIPSIS |
+                                              doctest.NORMALIZE_WHITESPACE |
+                                              doctest.REPORT_NDIFF)
+        )
diff --git a/tests/localyaml/__init__.py b/tests/localyaml/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/localyaml/__init__.py
diff --git a/tests/localyaml/fixtures/include-raw-escaped001.json b/tests/localyaml/fixtures/include-raw-escaped001.json
new file mode 100644
index 00000000..17098585
--- /dev/null
+++ b/tests/localyaml/fixtures/include-raw-escaped001.json
@@ -0,0 +1,24 @@
+[
+    {
+        "template-job": {
+            "builders": [
+                {
+                    "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
+                },
+                {
+                    "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n    # this next section is executed as one\n    echo \"${{VAR3}}\"\n    exit 0\n}}\n\n"
+                }
+            ],
+            "name": "test-job-include-raw-{num}"
+        }
+    },
+    {
+        "project": {
+            "num": 1,
+            "jobs": [
+                "test-job-include-raw-{num}"
+            ],
+            "name": "test-job-template-1"
+        }
+    }
+]
diff --git a/tests/localyaml/fixtures/include-raw-escaped001.yaml b/tests/localyaml/fixtures/include-raw-escaped001.yaml
new file mode 100644
index 00000000..38397b5d
--- /dev/null
+++ b/tests/localyaml/fixtures/include-raw-escaped001.yaml
@@ -0,0 +1,13 @@
+- template-job:
+    name: test-job-include-raw-{num}
+    builders:
+      - shell:
+          !include-raw-escape include-raw001-hello-world.sh
+      - shell:
+          !include-raw-escape include-raw001-vars.sh
+
+- project:
+    name: test-job-template-1
+    num: 1
+    jobs:
+      - 'test-job-include-raw-{num}'
diff --git a/tests/localyaml/fixtures/include-raw001-hello-world.sh b/tests/localyaml/fixtures/include-raw001-hello-world.sh
new file mode 100644
index 00000000..74ba06c1
--- /dev/null
+++ b/tests/localyaml/fixtures/include-raw001-hello-world.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Sample script showing how the yaml include-raw tag can be used
+# to inline scripts that are maintained outside of the jenkins
+# job yaml configuration.
+
+echo "hello world"
+
+exit 0
diff --git a/tests/localyaml/fixtures/include-raw001-vars.sh b/tests/localyaml/fixtures/include-raw001-vars.sh
new file mode 100644
index 00000000..46c544e7
--- /dev/null
+++ b/tests/localyaml/fixtures/include-raw001-vars.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# sample script to check that brackets aren't escaped
+# when using the include-raw application yaml tag
+
+VAR1="hello"
+VAR2="world"
+VAR3="${VAR1} ${VAR2}"
+
+[[ -n "${VAR3}" ]] && {
+    # this next section is executed as one
+    echo "${VAR3}"
+    exit 0
+}
+
diff --git a/tests/localyaml/fixtures/include-raw001.json b/tests/localyaml/fixtures/include-raw001.json
new file mode 100644
index 00000000..7f2ac38b
--- /dev/null
+++ b/tests/localyaml/fixtures/include-raw001.json
@@ -0,0 +1,15 @@
+[
+    {
+        "job": {
+            "builders": [
+                {
+                    "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n"
+                },
+                {
+                     "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n    # this next section is executed as one\n    echo \"${VAR3}\"\n    exit 0\n}\n\n"
+                }
+            ],
+            "name": "test-job-include-raw-1"
+        }
+    }
+]
diff --git a/tests/localyaml/fixtures/include-raw001.yaml b/tests/localyaml/fixtures/include-raw001.yaml
new file mode 100644
index 00000000..ba2f8ef0
--- /dev/null
+++ b/tests/localyaml/fixtures/include-raw001.yaml
@@ -0,0 +1,7 @@
+- job:
+    name: test-job-include-raw-1
+    builders:
+      - shell:
+          !include-raw include-raw001-hello-world.sh
+      - shell:
+          !include-raw include-raw001-vars.sh
diff --git a/tests/localyaml/fixtures/include001.json b/tests/localyaml/fixtures/include001.json
new file mode 100644
index 00000000..93713f8e
--- /dev/null
+++ b/tests/localyaml/fixtures/include001.json
@@ -0,0 +1,43 @@
+[
+    {
+        "job": {
+            "builders": [
+                {
+                    "copyartifact": {
+                        "filter": "*.tar.gz",
+                        "project": "foo",
+                        "parameter-filters": "PUBLISH=true",
+                        "target": "/home/foo",
+                        "flatten": true,
+                        "optional": true,
+                        "which-build": "last-successful"
+                    }
+                },
+                {
+                    "copyartifact": {
+                        "project": "bar",
+                        "parameter-filters": "PUBLISH=true",
+                        "target": "/home/foo",
+                        "build-number": 123,
+                        "which-build": "specific-build",
+                        "filter": "*.tar.gz",
+                        "flatten": true,
+                        "optional": true
+                    }
+                },
+                {
+                    "copyartifact": {
+                        "filter": "*.tar.gz",
+                        "project": "baz",
+                        "parameter-filters": "PUBLISH=true",
+                        "target": "/home/foo",
+                        "flatten": true,
+                        "optional": true,
+                        "which-build": "upstream-build"
+                    }
+                }
+            ],
+            "name": "test-job-1"
+        }
+    }
+]
diff --git a/tests/localyaml/fixtures/include001.yaml b/tests/localyaml/fixtures/include001.yaml
new file mode 100644
index 00000000..e29a2e9d
--- /dev/null
+++ b/tests/localyaml/fixtures/include001.yaml
@@ -0,0 +1,4 @@
+- job:
+    name: test-job-1
+    builders:
+      !include include001.yaml.inc
diff --git a/tests/localyaml/fixtures/include001.yaml.inc b/tests/localyaml/fixtures/include001.yaml.inc
new file mode 100644
index 00000000..294598bb
--- /dev/null
+++ b/tests/localyaml/fixtures/include001.yaml.inc
@@ -0,0 +1,17 @@
+- copyartifact: &copytarball
+    project: foo
+    filter: "*.tar.gz"
+    target: /home/foo
+    which-build: last-successful
+    optional: true
+    flatten: true
+    parameter-filters: PUBLISH=true
+- copyartifact:
+    <<: *copytarball
+    project: bar
+    which-build: specific-build
+    build-number: 123
+- copyartifact:
+    <<: *copytarball
+    project: baz
+    which-build: upstream-build
diff --git a/tests/localyaml/test_localyaml.py b/tests/localyaml/test_localyaml.py
new file mode 100644
index 00000000..2f4eb51d
--- /dev/null
+++ b/tests/localyaml/test_localyaml.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 Darragh Bailey
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+from testtools import TestCase
+from testscenarios.testcase import TestWithScenarios
+from tests.base import get_scenarios, JsonTestCase
+
+
+class TestCaseLocalYamlInclude(TestWithScenarios, TestCase, JsonTestCase):
+    """
+    Verify application specific tags independently of any changes to
+    modules XML parsing behaviour
+    """
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+    scenarios = get_scenarios(fixtures_path, 'yaml', 'json')
diff --git a/tests/yamlparser/fixtures/include-raw-escape001-echo-vars.sh b/tests/yamlparser/fixtures/include-raw-escape001-echo-vars.sh
new file mode 100644
index 00000000..31e8743d
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw-escape001-echo-vars.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# test script containing some variables to show how you can include scripts
+# into job template definitions provided you use the include-raw-escaped tag
+
+MSG="hello world"
+
+[[ -n "${MSG}" ]] && {
+    # this next section is executed as one
+    echo "${MSG}"
+    exit 0
+}
diff --git a/tests/yamlparser/fixtures/include-raw-escape001.conf b/tests/yamlparser/fixtures/include-raw-escape001.conf
new file mode 100644
index 00000000..28027f4b
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw-escape001.conf
@@ -0,0 +1,2 @@
+[job_builder]
+include_path=tests/localyaml/fixtures:.
diff --git a/tests/yamlparser/fixtures/include-raw-escape001.xml b/tests/yamlparser/fixtures/include-raw-escape001.xml
new file mode 100644
index 00000000..f24be4da
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw-escape001.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <disabled>false</disabled>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <quietPeriod>1</quietPeriod>
+  <assignedNode>my-test-node</assignedNode>
+  <canRoam>false</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+#
+# test script containing some variables to show how you can include scripts
+# into job template definitions provided you use the include-raw-escaped tag
+
+MSG=&quot;hello world&quot;
+
+[[ -n &quot;${MSG}&quot; ]] &amp;&amp; {
+    # this next section is executed as one
+    echo &quot;${MSG}&quot;
+    exit 0
+}
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>
diff --git a/tests/yamlparser/fixtures/include-raw-escape001.yaml b/tests/yamlparser/fixtures/include-raw-escape001.yaml
new file mode 100644
index 00000000..abb5b93b
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw-escape001.yaml
@@ -0,0 +1,22 @@
+# vim: sw=4 ts=4 et
+- job-template:
+    name: 'test-job-{num}'
+    project-type: freestyle
+    node: my-test-node
+    disabled: false
+    quiet-period: 1
+    properties:
+      - inject:
+          keep-build-variables: true
+          keep-system-variables: true
+    builders:
+      - shell:
+          !include-raw-escape include-raw-escape001-echo-vars.sh
+
+
+- project:
+    name: test-job-template-1
+    num: 1
+    jobs:
+      - 'test-job-{num}'
+
diff --git a/tests/yamlparser/fixtures/include-raw001.xml b/tests/yamlparser/fixtures/include-raw001.xml
new file mode 100644
index 00000000..9713f14d
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw001.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/bin/bash
+#
+# Sample script showing how the yaml include-raw tag can be used
+# to inline scripts that are maintained outside of the jenkins
+# job yaml configuration.
+
+echo &quot;hello world&quot;
+
+exit 0
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>
diff --git a/tests/yamlparser/fixtures/include-raw001.yaml b/tests/yamlparser/fixtures/include-raw001.yaml
new file mode 100644
index 00000000..a89f61ea
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw001.yaml
@@ -0,0 +1,10 @@
+- job:
+    name: test-job-2
+    properties:
+      - inject:
+          keep-build-variables: true
+          keep-system-variables: true
+    builders:
+      - shell:
+          !include-raw ../../localyaml/fixtures/include-raw001-hello-world.sh
+
diff --git a/tests/yamlparser/fixtures/include-raw002-cool.sh b/tests/yamlparser/fixtures/include-raw002-cool.sh
new file mode 100644
index 00000000..6b72b6ae
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw002-cool.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "Doing somethiung cool"
diff --git a/tests/yamlparser/fixtures/include-raw002-cool.zsh b/tests/yamlparser/fixtures/include-raw002-cool.zsh
new file mode 100644
index 00000000..709c762b
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw002-cool.zsh
@@ -0,0 +1,2 @@
+#!/bin/zsh
+echo "Doing somethin cool with zsh"
diff --git a/tests/yamlparser/fixtures/include-raw002.xml b/tests/yamlparser/fixtures/include-raw002.xml
new file mode 100644
index 00000000..25e97ab3
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw002.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Doing somethiung cool&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/zsh
+echo &quot;Doing somethin cool with zsh&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Ant>
+          <targets>target1 target2</targets>
+          <antName>default</antName>
+        </hudson.tasks.Ant>
+        <EnvInjectBuilder>
+          <info>
+            <propertiesFilePath>example.prop</propertiesFilePath>
+            <propertiesContent>EXAMPLE=foo-bar</propertiesContent>
+          </info>
+        </EnvInjectBuilder>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+    <com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+      <includes>file1,file2*.txt</includes>
+      <excludes>file2bad.txt</excludes>
+      <flatten>false</flatten>
+      <includeAntExcludes>false</includeAntExcludes>
+      <relativeTo>userContent</relativeTo>
+      <hudsonHomeRelative>false</hudsonHomeRelative>
+    </com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+  </buildWrappers>
+</project>
diff --git a/tests/yamlparser/fixtures/include-raw002.yaml b/tests/yamlparser/fixtures/include-raw002.yaml
new file mode 100644
index 00000000..771f720d
--- /dev/null
+++ b/tests/yamlparser/fixtures/include-raw002.yaml
@@ -0,0 +1,44 @@
+# vim: sw=4 ts=4 et
+- wrapper:
+    name: timeout-wrapper
+    wrappers:
+      - timeout:
+          fail: true
+          elastic-percentage: 150
+          elastic-default-timeout: 90
+          type: elastic
+
+- wrapper:
+    name: pre-scm-shell-ant
+    wrappers:
+      - pre-scm-buildstep:
+          - shell:
+              !include-raw include-raw002-cool.sh
+          - shell:
+              !include-raw include-raw002-cool.zsh
+          - ant: "target1 target2"
+            ant-name: "Standard Ant"
+          - inject:
+               properties-file: example.prop
+               properties-content: EXAMPLE=foo-bar
+
+- wrapper:
+    name: copy-files
+    wrappers:
+      - copy-to-slave:
+          includes:
+            - file1
+            - file2*.txt
+          excludes:
+            - file2bad.txt
+
+
+- job:
+    name: test-job-3
+    wrappers:
+        !include include001.yaml.inc
+    properties:
+      - inject:
+          keep-build-variables: true
+          keep-system-variables: true
+
diff --git a/tests/yamlparser/fixtures/include001.xml b/tests/yamlparser/fixtures/include001.xml
new file mode 100644
index 00000000..25e97ab3
--- /dev/null
+++ b/tests/yamlparser/fixtures/include001.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders/>
+  <publishers/>
+  <buildWrappers>
+    <hudson.plugins.build__timeout.BuildTimeoutWrapper>
+      <timeoutMinutes>3</timeoutMinutes>
+      <failBuild>true</failBuild>
+      <writingDescription>false</writingDescription>
+      <timeoutPercentage>150</timeoutPercentage>
+      <timeoutMinutesElasticDefault>90</timeoutMinutesElasticDefault>
+      <timeoutType>elastic</timeoutType>
+    </hudson.plugins.build__timeout.BuildTimeoutWrapper>
+    <org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+      <buildSteps>
+        <hudson.tasks.Shell>
+          <command>#!/bin/bash
+echo &quot;Doing somethiung cool&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Shell>
+          <command>#!/bin/zsh
+echo &quot;Doing somethin cool with zsh&quot;
+</command>
+        </hudson.tasks.Shell>
+        <hudson.tasks.Ant>
+          <targets>target1 target2</targets>
+          <antName>default</antName>
+        </hudson.tasks.Ant>
+        <EnvInjectBuilder>
+          <info>
+            <propertiesFilePath>example.prop</propertiesFilePath>
+            <propertiesContent>EXAMPLE=foo-bar</propertiesContent>
+          </info>
+        </EnvInjectBuilder>
+      </buildSteps>
+    </org.jenkinsci.plugins.preSCMbuildstep.PreSCMBuildStepsWrapper>
+    <com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+      <includes>file1,file2*.txt</includes>
+      <excludes>file2bad.txt</excludes>
+      <flatten>false</flatten>
+      <includeAntExcludes>false</includeAntExcludes>
+      <relativeTo>userContent</relativeTo>
+      <hudsonHomeRelative>false</hudsonHomeRelative>
+    </com.michelin.cio.hudson.plugins.copytoslave.CopyToSlaveBuildWrapper>
+  </buildWrappers>
+</project>
diff --git a/tests/yamlparser/fixtures/include001.yaml b/tests/yamlparser/fixtures/include001.yaml
new file mode 100644
index 00000000..85a54525
--- /dev/null
+++ b/tests/yamlparser/fixtures/include001.yaml
@@ -0,0 +1,46 @@
+# vim: sw=4 ts=4 et
+- wrapper:
+    name: timeout-wrapper
+    wrappers:
+      - timeout:
+          fail: true
+          elastic-percentage: 150
+          elastic-default-timeout: 90
+          type: elastic
+
+- wrapper:
+    name: pre-scm-shell-ant
+    wrappers:
+      - pre-scm-buildstep:
+          - shell: |
+              #!/bin/bash
+              echo "Doing somethiung cool"
+          - shell: |
+              #!/bin/zsh
+              echo "Doing somethin cool with zsh"
+          - ant: "target1 target2"
+            ant-name: "Standard Ant"
+          - inject:
+               properties-file: example.prop
+               properties-content: EXAMPLE=foo-bar
+
+- wrapper:
+    name: copy-files
+    wrappers:
+      - copy-to-slave:
+          includes:
+            - file1
+            - file2*.txt
+          excludes:
+            - file2bad.txt
+
+
+- job:
+    name: test-job-1
+    wrappers:
+        !include include001.yaml.inc
+    properties:
+      - inject:
+          keep-build-variables: true
+          keep-system-variables: true
+
diff --git a/tests/yamlparser/fixtures/include001.yaml.inc b/tests/yamlparser/fixtures/include001.yaml.inc
new file mode 100644
index 00000000..312284f4
--- /dev/null
+++ b/tests/yamlparser/fixtures/include001.yaml.inc
@@ -0,0 +1,3 @@
+- timeout-wrapper
+- pre-scm-shell-ant
+- copy-files