path: root/jenkins_jobs/parser.py

#!/usr/bin/env python
# Copyright (C) 2015 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# Manage JJB yaml feature implementation

import copy
import fnmatch
import io
import itertools
import logging
import re
import os

from jenkins_jobs.constants import MAGIC_MANAGE_STRING
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.formatter import deep_format
import jenkins_jobs.local_yaml as local_yaml
from jenkins_jobs import utils

__all__ = [
    "YamlParser"
]

logger = logging.getLogger(__name__)


def matches(what, glob_patterns):
    """
    Checks if the given string, ``what``, matches any of the glob patterns in
    the iterable, ``glob_patterns``

    :arg str what: String that we want to test if it matches a pattern
    :arg iterable glob_patterns: glob patterns to match (list, tuple, set,
    etc.)
    """
    return any(fnmatch.fnmatch(what, glob_pattern)
               for glob_pattern in glob_patterns)


def combination_matches(combination, match_combinations):
    """
    Checks if the given combination is matches for any of the given combination
    globs, being those a set of combinations where if a key is missing, it's
    considered matching

    (key1=2, key2=3)

    would match the combination match:
    (key2=3)

    but not:
    (key1=2, key2=2)
    """
    for cmatch in match_combinations:
        for key, val in combination.items():
            if cmatch.get(key, val) != val:
                break
        else:
            return True
    return False


class YamlParser(object):
    def __init__(self, jjb_config=None):
        self.data = {}
        self.jobs = []
        self.views = []

        self.jjb_config = jjb_config
        self.keep_desc = jjb_config.yamlparser['keep_descriptions']
        self.path = jjb_config.yamlparser['include_path']

    def load_files(self, fn):

        # handle deprecated behavior, and check that it's not a file like
        # object as these may implement the '__iter__' attribute.
        if not hasattr(fn, '__iter__') or hasattr(fn, 'read'):
            logger.warning(
                'Passing single elements for the `fn` argument in '
                'Builder.load_files is deprecated. Please update your code '
                'to use a list as support for automatic conversion will be '
                'removed in a future version.')
            fn = [fn]

        files_to_process = []
        for path in fn:
            if not hasattr(path, 'read') and os.path.isdir(path):
                files_to_process.extend([os.path.join(path, f)
                                         for f in sorted(os.listdir(path))
                                         if (f.endswith('.yml') or
                                             f.endswith('.yaml'))])
            else:
                files_to_process.append(path)

        # symlinks used to allow loading of sub-dirs can result in duplicate
        # definitions of macros and templates when loading all from top-level
        unique_files = []
        for f in files_to_process:
            if hasattr(f, 'read'):
                unique_files.append(f)
                continue
            rpf = os.path.realpath(f)
            if rpf not in unique_files:
                unique_files.append(rpf)
            else:
                logger.warning("File '%s' already added as '%s', ignoring "
                               "reference to avoid duplicating yaml "
                               "definitions." % (f, rpf))

        for in_file in unique_files:
            # use of ask-for-permissions instead of ask-for-forgiveness
            # performs better when low use cases.
            if hasattr(in_file, 'name'):
                fname = in_file.name
            else:
                fname = in_file
            logger.debug("Parsing YAML file {0}".format(fname))
            if hasattr(in_file, 'read'):
                self._parse_fp(in_file)
            else:
                self.parse(in_file)

    def _parse_fp(self, fp):
        # wrap provided file streams to ensure correct encoding used
        data = local_yaml.load(utils.wrap_stream(fp),
                               self.jjb_config.yamlparser['retain_anchors'],
                               search_path=self.path)
        if data:
            if not isinstance(data, list):
                raise JenkinsJobsException(
                    "The topmost collection in file '{fname}' must be a list,"
                    " not a {cls}".format(fname=getattr(fp, 'name', fp),
                                          cls=type(data)))
            for item in data:
                cls, dfn = next(iter(item.items()))
                group = self.data.get(cls, {})
                if len(item.items()) > 1:
                    n = None
                    for k, v in item.items():
                        if k == "name":
                            n = v
                            break
                    # Syntax error
                    raise JenkinsJobsException("Syntax error, for item "
                                               "named '{0}'. Missing indent?"
                                               .format(n))
                # allow any entry to specify an id that can also be used
                _id = dfn.get('id', dfn['name'])
                if _id in group:
                    self._handle_dups(
                        "Duplicate entry found in '{0}: '{1}' already "
                        "defined".format(fp.name, _id))
                group[_id] = dfn
                self.data[cls] = group

    def parse(self, fn):
        with io.open(fn, 'r', encoding='utf-8') as fp:
            self._parse_fp(fp)

    def _handle_dups(self, message):

        if not self.jjb_config.yamlparser['allow_duplicates']:
            logger.error(message)
            raise JenkinsJobsException(message)
        else:
            logger.warning(message)

    def _getJob(self, name):
        job = self.data.get('job', {}).get(name, None)
        if not job:
            return job
        return self._applyDefaults(job)

    def _getJobGroup(self, name):
        return self.data.get('job-group', {}).get(name, None)

    def _getJobTemplate(self, name):
        job = self.data.get('job-template', {}).get(name, None)
        if not job:
            return job
        return self._applyDefaults(job)

    def _applyDefaults(self, data, override_dict=None):
        if override_dict is None:
            override_dict = {}

        whichdefaults = data.get('defaults', 'global')
        defaults = copy.deepcopy(self.data.get('defaults',
                                 {}).get(whichdefaults, {}))
        if defaults == {} and whichdefaults != 'global':
            raise JenkinsJobsException("Unknown defaults set: '{0}'"
                                       .format(whichdefaults))

        for key in override_dict.keys():
            if key in defaults.keys():
                defaults[key] = override_dict[key]

        newdata = {}
        newdata.update(defaults)
        newdata.update(data)
        return newdata

    def _formatDescription(self, job):
        if self.keep_desc:
            description = job.get("description", None)
        else:
            description = job.get("description", '')
        if description is not None:
            job["description"] = description + \
                self._get_managed_string().lstrip()

    def _getfullname(self, data):
        if 'folder' in data:
            return "%s/%s" % (data['folder'], data['name'])

        return data['name']

    def expandYaml(self, registry, jobs_glob=None):
        changed = True
        while changed:
            changed = False
            for module in registry.modules:
                if hasattr(module, 'handle_data'):
                    if module.handle_data(self.data):
                        changed = True

        for job in self.data.get('job', {}).values():
            job = self._applyDefaults(job)
            job['name'] = self._getfullname(job)

            if jobs_glob and not matches(job['name'], jobs_glob):
                logger.debug("Ignoring job {0}".format(job['name']))
                continue
            logger.debug("Expanding job '{0}'".format(job['name']))
            self._formatDescription(job)
            self.jobs.append(job)

        for view in self.data.get('view', {}).values():
            view['name'] = self._getfullname(view)

            if jobs_glob and not matches(view['name'], jobs_glob):
                logger.debug("Ignoring view {0}".format(view['name']))
                continue

            logger.debug("Expanding view '{0}'".format(view['name']))
            self._formatDescription(view)
            self.views.append(view)

        for project in self.data.get('project', {}).values():
            logger.debug("Expanding project '{0}'".format(project['name']))
            # use a set to check for duplicate job references in projects
            seen = set()
            for jobspec in project.get('jobs', []):
                if isinstance(jobspec, dict):
                    # Singleton dict containing dict of job-specific params
                    jobname, jobparams = next(iter(jobspec.items()))
                    if not isinstance(jobparams, dict):
                        jobparams = {}
                else:
                    jobname = jobspec
                    jobparams = {}
                job = self._getJob(jobname)
                if job:
                    # Just naming an existing defined job
                    if jobname in seen:
                        self._handle_dups("Duplicate job '{0}' specified "
                                          "for project '{1}'"
                                          .format(jobname, project['name']))
                    seen.add(jobname)
                    continue
                # see if it's a job group
                group = self._getJobGroup(jobname)
                if group:
                    for group_jobspec in group['jobs']:
                        if isinstance(group_jobspec, dict):
                            group_jobname, group_jobparams = \
                                next(iter(group_jobspec.items()))
                            if not isinstance(group_jobparams, dict):
                                group_jobparams = {}
                        else:
                            group_jobname = group_jobspec
                            group_jobparams = {}
                        job = self._getJob(group_jobname)
                        if job:
                            if group_jobname in seen:
                                self._handle_dups(
                                    "Duplicate job '{0}' specified for "
                                    "project '{1}'".format(group_jobname,
                                                           project['name']))
                            seen.add(group_jobname)
                            continue
                        template = self._getJobTemplate(group_jobname)
                        # Allow a group to override parameters set by a project
                        d = type(project)(project)
                        d.update(jobparams)
                        d.update(group)
                        d.update(group_jobparams)
                        # Except name, since the group's name is not useful
                        d['name'] = project['name']
                        if template:
                            self._expandYamlForTemplateJob(d, template,
                                                           jobs_glob)
                    continue
                # see if it's a template
                template = self._getJobTemplate(jobname)
                if template:
                    d = type(project)(project)
                    d.update(jobparams)
                    self._expandYamlForTemplateJob(d, template, jobs_glob)
                else:
                    raise JenkinsJobsException("Failed to find suitable "
                                               "template named '{0}'"
                                               .format(jobname))

            for viewspec in project.get('views', []):
                if isinstance(viewspec, dict):
                    # Singleton dict containing dict of view-specific params
                    viewname, viewparams = next(iter(viewspec.items()))
                    if not isinstance(viewparams, dict):
                        viewparams = {}
                else:
                    viewname = viewspec
                    viewparams = {}
                view = self._getView(viewname)
                if view:
                    # Just naming an existing defined view
                    if viewname in seen:
                        self._handle_dups("Duplicate view '{0}' specified "
                                          "for project '{1}'"
                                          .format(viewname, project['name']))
                    seen.add(viewname)
                    continue
                # see if it's a view group
                group = self._getViewGroup(viewname)
                if group:
                    for group_viewspec in group['views']:
                        if isinstance(group_viewspec, dict):
                            group_viewname, group_viewparams = \
                                next(iter(group_viewspec.items()))
                            if not isinstance(group_viewparams, dict):
                                group_viewparams = {}
                        else:
                            group_viewname = group_viewspec
                            group_viewparams = {}
                        view = self._getView(group_viewname)
                        if view:
                            if group_viewname in seen:
                                self._handle_dups(
                                    "Duplicate view '{0}' specified for "
                                    "project '{1}'".format(group_viewname,
                                                           project['name']))
                            seen.add(group_viewname)
                            continue
                        template = self._getViewTemplate(group_viewname)
                        # Allow a group to override parameters set by a project
                        d = type(project)(project)
                        d.update(viewparams)
                        d.update(group)
                        d.update(group_viewparams)
                        # Except name, since the group's name is not useful
                        d['name'] = project['name']
                        if template:
                            self._expandYamlForTemplateView(
                                d, template, jobs_glob)
                    continue
                # see if it's a template
                template = self._getViewTemplate(viewname)
                if template:
                    d = type(project)(project)
                    d.update(viewparams)
                    self._expandYamlForTemplateView(d, template, jobs_glob)
                else:
                    raise JenkinsJobsException("Failed to find suitable "
                                               "template named '{0}'"
                                               .format(viewname))

        # check for duplicate generated jobs
        seen = set()
        # walk the list in reverse so that last definition wins
        for job in self.jobs[::-1]:
            if job['name'] in seen:
                self._handle_dups("Duplicate definitions for job '{0}' "
                                  "specified".format(job['name']))
                self.jobs.remove(job)
            seen.add(job['name'])

        # check for duplicate generated views
        seen_views = set()
        # walk the list in reverse so that last definition wins
        for view in self.views[::-1]:
            if view['name'] in seen_views:
                self._handle_dups("Duplicate definitions for view '{0}' "
                                  "specified".format(view['name']))
                self.views.remove(view)
            seen_views.add(view['name'])

        return self.jobs, self.views

    def _expandYamlForTemplateJob(self, project, template, jobs_glob=None):
        dimensions = []
        template_name = template['name']
        # reject keys that are not useful during yaml expansion
        for k in ['jobs']:
            project.pop(k)
        excludes = project.pop('exclude', [])
        for (k, v) in project.items():
            tmpk = '{{{0}}}'.format(k)
            if tmpk not in template_name:
                continue
            if type(v) == list:
                dimensions.append(zip([k] * len(v), v))
        # XXX somewhat hackish to ensure we actually have a single
        # pass through the loop
        if len(dimensions) == 0:
            dimensions = [(("", ""),)]

        for values in itertools.product(*dimensions):
            params = copy.deepcopy(project)
            params = self._applyDefaults(params, template)
            params['template-name'] = re.sub(r'({|})', r'\1\1', template_name)

            try:
                expanded_values = {}
                for (k, v) in values:
                    if isinstance(v, dict):
                        inner_key = next(iter(v))
                        expanded_values[k] = inner_key
                        expanded_values.update(v[inner_key])
                    else:
                        expanded_values[k] = v
            except TypeError:
                project_name = project.pop('name')
                logger.error(
                    "Exception thrown while expanding template '%s' for "
                    "project '%s', with expansion arguments of:\n%s\n"
                    "Original project input variables for template:\n%s\n"
                    "Most likely the inputs have items indented incorrectly "
                    "to describe how they should be applied.\n\nNote yaml "
                    "'null' is mapped to python's 'None'", template_name,
                    project_name,
                    "".join(local_yaml.dump({k: v}, default_flow_style=False)
                            for (k, v) in values),
                    local_yaml.dump(project, default_flow_style=False))
                raise

            params.update(expanded_values)
            try:
                params = deep_format(params, params)
            except Exception:
                logging.error(
                    "Failure formatting params '%s' with itself", params)
                raise
            if combination_matches(params, excludes):
                logger.debug('Excluding combination %s', str(params))
                continue

            for key in template.keys():
                if key not in params:
                    params[key] = template[key]

            try:
                expanded = deep_format(
                    template, params,
                    self.jjb_config.yamlparser['allow_empty_variables'])
            except Exception:
                logging.error(
                    "Failure formatting template '%s', containing '%s' with "
                    "params '%s'", template_name, template, params)
                raise
            expanded['name'] = self._getfullname(expanded)

            job_name = expanded.get('name')
            if jobs_glob and not matches(job_name, jobs_glob):
                continue

            self._formatDescription(expanded)
            self.jobs.append(expanded)

    def _get_managed_string(self):
        # The \n\n is not hard coded, because they get stripped if the
        # project does not otherwise have a description.
        return "\n\n" + MAGIC_MANAGE_STRING

    # Views related
    def _getView(self, name):
        view = self.data.get('view', {}).get(name, None)
        if not view:
            return view
        return self._applyDefaults(view)

    def _getViewGroup(self, name):
        return self.data.get('view-group', {}).get(name, None)

    def _getViewTemplate(self, name):
        view = self.data.get('view-template', {}).get(name, None)
        if not view:
            return view
        return self._applyDefaults(view)

    def _expandYamlForTemplateView(self, project, template, views_glob=None):
        dimensions = []
        template_name = template['name']
        # reject keys that are not useful during yaml expansion
        for k in ['views']:
            project.pop(k)
        excludes = project.pop('exclude', [])
        for (k, v) in project.items():
            tmpk = '{{{0}}}'.format(k)
            if tmpk not in template_name:
                continue
            if type(v) == list:
                dimensions.append(zip([k] * len(v), v))
        # XXX somewhat hackish to ensure we actually have a single
        # pass through the loop
        if len(dimensions) == 0:
            dimensions = [(("", ""),)]

        for values in itertools.product(*dimensions):
            params = copy.deepcopy(project)
            params = self._applyDefaults(params, template)

            expanded_values = {}
            for (k, v) in values:
                if isinstance(v, dict):
                    inner_key = next(iter(v))
                    expanded_values[k] = inner_key
                    expanded_values.update(v[inner_key])
                else:
                    expanded_values[k] = v

            params.update(expanded_values)
            params = deep_format(params, params)
            if combination_matches(params, excludes):
                logger.debug('Excluding combination %s', str(params))
                continue

            for key in template.keys():
                if key not in params:
                    params[key] = template[key]

            params['template-name'] = template_name
            expanded = deep_format(
                template, params,
                self.jjb_config.yamlparser['allow_empty_variables'])

            view_name = expanded.get('name')
            if views_glob and not matches(view_name, views_glob):
                continue

            self._formatDescription(expanded)
            self.views.append(expanded)