Basic folder support

Allows specifying a folder attribute for each job generated, which in
turn is used when creating or uploading to place the job under the
requested folder.

The job name is expanded after defaults are applied, to support the
attribute being defined within a set of defaults applied to a number of
jobs.

This in turn allows for multiple jobs with the same basename to exist,
provided they are targeted at different folders.

Does not support creating the folders if they do not exist.

Change-Id: I8c2157c4c81087cc972a048d1b88d5f08ac65361

3 files changed, 54 insertions, 32 deletions

diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py
index 238be932..7626d461 100644
--- a/jenkins_jobs/builder.py
+++ b/jenkins_jobs/builder.py
@@ -65,18 +65,43 @@ class JenkinsManager(object):
         self._view_list = None
         self._jjb_config = jjb_config
 
+    def _setup_output(self, output, item, config_xml=False):
+        output_dir = output
+        output_fn = os.path.join(output, item)
+        if '/' in item:
+            # in item folder
+            output_fn = os.path.join(output, os.path.normpath(item))
+            output_dir = os.path.dirname(output_fn)
+
+        # if in a folder, re-adding name to the directory here
+        if config_xml:
+            output_dir = os.path.join(
+                output_dir, os.path.basename(item))
+            output_fn = os.path.join(output_dir, 'config.xml')
+
+        if output_dir != output:
+            logger.info("Creating directory %s" % output_dir)
+            try:
+                os.makedirs(output_dir)
+            except OSError:
+                if not os.path.isdir(output_dir):
+                    raise
+
+        return output_fn
+
     @property
     def jobs(self):
         if self._jobs is None:
             # populate jobs
-            self._jobs = self.jenkins.get_jobs()
+            self._jobs = self.jenkins.get_all_jobs()
 
         return self._jobs
 
     @property
     def job_list(self):
         if self._job_list is None:
-            self._job_list = set(job['name'] for job in self.jobs)
+            # python-jenkins uses 'fullname' for folder/name combination
+            self._job_list = set(job['fullname'] for job in self.jobs)
         return self._job_list
 
     def update_job(self, job_name, xml):
@@ -154,17 +179,18 @@ class JenkinsManager(object):
         if keep is None:
             keep = []
         for job in jobs:
-            if job['name'] not in keep:
-                if self.is_managed(job['name']):
+            # python-jenkins stores the folder and name as 'fullname'
+            if job['fullname'] not in keep:
+                if self.is_managed(job['fullname']):
                     logger.info("Removing obsolete jenkins job {0}"
-                                .format(job['name']))
-                    self.delete_job(job['name'])
+                                .format(job['fullname']))
+                    self.delete_job(job['fullname'])
                     deleted_jobs += 1
                 else:
                     logger.info("Not deleting unmanaged jenkins job %s",
-                                job['name'])
+                                job['fullname'])
             else:
-                logger.debug("Keeping job %s", job['name'])
+                logger.debug("Keeping job %s", job['fullname'])
         return deleted_jobs
 
     def delete_jobs(self, jobs):
@@ -231,17 +257,8 @@ class JenkinsManager(object):
                         raise
                     continue
 
-                if config_xml:
-                    output_dir = os.path.join(output, job.name)
-                    logger.info("Creating directory %s" % output_dir)
-                    try:
-                        os.makedirs(output_dir)
-                    except OSError:
-                        if not os.path.isdir(output_dir):
-                            raise
-                    output_fn = os.path.join(output_dir, 'config.xml')
-                else:
-                    output_fn = os.path.join(output, job.name)
+                output_fn = self._setup_output(output, job.name, config_xml)
+
                 logger.debug("Writing XML to '{0}'".format(output_fn))
                 with io.open(output_fn, 'w', encoding='utf-8') as f:
                     f.write(job.output().decode('utf-8'))
@@ -383,17 +400,8 @@ class JenkinsManager(object):
                         raise
                     continue
 
-                if config_xml:
-                    output_dir = os.path.join(output, view.name)
-                    logger.info("Creating directory %s" % output_dir)
-                    try:
-                        os.makedirs(output_dir)
-                    except OSError:
-                        if not os.path.isdir(output_dir):
-                            raise
-                    output_fn = os.path.join(output_dir, 'config.xml')
-                else:
-                    output_fn = os.path.join(output, view.name)
+                output_fn = self._setup_output(output, view.name, config_xml)
+
                 logger.debug("Writing XML to '{0}'".format(output_fn))
                 with io.open(output_fn, 'w', encoding='utf-8') as f:
                     f.write(view.output().decode('utf-8'))
diff --git a/jenkins_jobs/modules/general.py b/jenkins_jobs/modules/general.py
index 658d3c14..0bfca537 100644
--- a/jenkins_jobs/modules/general.py
+++ b/jenkins_jobs/modules/general.py
@@ -56,6 +56,12 @@ Example:
       Path for a custom workspace. Defaults to Jenkins default
       configuration.
 
+    * **folder**:
+      The folder attribute provides an alternative to using '<path>/<name>' as
+      the job name to specify which Jenkins folder to upload the job to.
+      Requires the `CloudBees Folders Plugin.
+      <https://wiki.jenkins-ci.org/display/JENKINS/CloudBees+Folders+Plugin>`_
+
     * **child-workspace**:
       Path for a child custom workspace. Defaults to Jenkins default
       configuration. This parameter is only valid for matrix type jobs.
@@ -103,7 +109,6 @@ Example:
     * **raw**:
       If present, this section should contain a single **xml** entry. This XML
       will be inserted at the top-level of the :ref:`Job` definition.
-
 """
 
 import logging
diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py
index 5dee3cac..1033b9fd 100644
--- a/jenkins_jobs/parser.py
+++ b/jenkins_jobs/parser.py
@@ -226,6 +226,12 @@ class YamlParser(object):
             for macro in self.data.get(component_type, {}).values():
                 self._macro_registry.register(component_type, macro)
 
+    def _getfullname(self, data):
+        if 'folder' in data:
+            return "%s/%s" % (data['folder'], data['name'])
+
+        return data['name']
+
     def expandYaml(self, registry, jobs_glob=None):
         changed = True
         while changed:
@@ -240,15 +246,17 @@ class YamlParser(object):
             self._macro_registry.expand_macros(default)
         for job in self.data.get('job', {}).values():
             self._macro_registry.expand_macros(job)
+            job = self._applyDefaults(job)
+            job['name'] = self._getfullname(job)
             if jobs_glob and not matches(job['name'], jobs_glob):
                 logger.debug("Ignoring job {0}".format(job['name']))
                 continue
             logger.debug("Expanding job '{0}'".format(job['name']))
-            job = self._applyDefaults(job)
             self._formatDescription(job)
             self.jobs.append(job)
 
         for view in self.data.get('view', {}).values():
+            view['name'] = self._getfullname(view)
             logger.debug("Expanding view '{0}'".format(view['name']))
             self._formatDescription(view)
             self.views.append(view)
@@ -400,6 +408,7 @@ class YamlParser(object):
                     "Failure formatting template '%s', containing '%s' with "
                     "params '%s'", template_name, template, params)
                 raise
+            expanded['name'] = self._getfullname(expanded)
 
             self._macro_registry.expand_macros(expanded, params)
             job_name = expanded.get('name')