Differential D667

Use datagrepper to fetch jobs
ClosedPublic
Actions

Authored by mkrizek on Nov 30 2015, 2:19 PM.

Details

Reviewers
tflink
Commits
rTRGRa83a15805ab9: Use datagrepper to fetch jobs
Test Plan

Ran fedmsg-hub, fedmsgs came all the way to the StreamTrigger.
Ran python cli.py -s 1448496000 -e 1448534412 (supports also -d). Worked.
Ran python cli.py -f jobs.csv (supports also -d). Worked.

Diff Detail

Repository
rTRGR taskotron-trigger
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.
mkrizek retitled this revision from to Cleanups.Nov 30 2015, 2:19 PM
mkrizek updated this object.
mkrizek edited the test plan for this revision. (Show Details)
mkrizek added a reviewer: tflink.
mkrizek updated this revision to Diff 1728.Nov 30 2015, 2:27 PM
  • Cleanup
  • Use datagrepper to fetch jobs instead of jobs.csv
mkrizek retitled this revision from Cleanups to Use datagrepper to fetch jobs instead of jobs.csv.Nov 30 2015, 2:28 PM
mkrizek edited the test plan for this revision. (Show Details)
tflink requested changes to this revision.Nov 30 2015, 3:18 PM

I don't see any issues in the code but I'd like to see datagrepper support in addition to csv files and maintaining the ability to log jobs to csv. It's an easy way to see what's going on at the trigger level and when fedmsgs have triggered job scheduling

This revision now requires changes to proceed.Nov 30 2015, 3:18 PM
mkrizek updated this revision to Diff 1735.Dec 1 2015, 1:59 PM

Use jobs file as well

mkrizek retitled this revision from Use datagrepper to fetch jobs instead of jobs.csv to Use datagrepper to fetch jobs.Dec 2 2015, 9:18 AM
mkrizek edited the test plan for this revision. (Show Details)
mkrizek updated this revision to Diff 1748.Dec 4 2015, 6:07 PM

Fix tests

tflink accepted this revision.Dec 7 2015, 6:31 PM
This revision is now accepted and ready to land.Dec 7 2015, 6:31 PM
Closed by commit rTRGRa83a15805ab9: Use datagrepper to fetch jobs (authored by mkrizek). · Explain WhyDec 8 2015, 10:12 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

Diff 1762

View Options

conf/trigger.cfg.example

1[buildbot] 1​[buildbot]
2url = http://localhost:8080/buildbot/change_hook 2​url = http://localhost:8080/buildbot/change_hook
3 3​
4[trigger] 4​[trigger]
5koji_build_completed_tasks = rpmlint 5​koji_build_completed_tasks = rpmlint
6koji_tag_changed_tasks = upgradepath,depcheck 6​koji_tag_changed_tasks = upgradepath,depcheck
7compose_completed_tasks = 7​compose_completed_tasks =
8valid_arches = x86_64 8​valid_arches = x86_64
9koji_url = http://koji.fedoraproject.org/kojihub 9​koji_url = http://koji.fedoraproject.org/kojihub
10​datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw
10execdb_server = http://localhost:5003 11​execdb_server = http://localhost:5003
11type = StreamTrigger 12​type = StreamTrigger
12job_logging = False 13​job_logging = False
13joblog_file = /var/log/taskotron-trigger/jobs.csv 14​joblog_file = /var/log/taskotron-trigger/jobs.csv
14fuse_delay = 1000 ; seconds 15​fuse_delay = 1000 ; seconds
View Options

jobtriggers/__init__.py

1​import itertools
1import fedmsg.consumers 2​import fedmsg.consumers
2 ​
3from . import config, triggers, utils 3​from . import config, triggers, utils
4 4​
5​
6​class TriggerMsgError(Exception):
7​ pass
8​
9​
5class JobTrigger(fedmsg.consumers.FedmsgConsumer): 10​class JobTrigger(fedmsg.consumers.FedmsgConsumer):
6 def __init__(self, *args, **kw): 11​ def __init__(self, *args, **kw):
7 super(JobTrigger, self).__init__(*args, **kw) 12​ super(JobTrigger, self).__init__(*args, **kw)
8 self.trigger = triggers.get_trigger(config.trigger_type) 13​ self.trigger = triggers.get_trigger(config.trigger_type)
9 14​
10 def trigger_tasks(self, item, item_type, tasks, arches): 15​ def trigger_tasks(self, item, item_type, tasks, arches):
11 for task in tasks: 16​ for task, arch in itertools.product(tasks, arches):
12 for arch in arches: 17​ self.log.info('triggering %s for %s on %s', task, item, arch)
13 self.log.info('triggering %s for %s on %s', task, item, arch) ​
14 18​
15 if config.job_logging: 19​ if config.job_logging:
16 try: 20​ try:
17 utils.log_job(config.joblog_file, item, item_type, task, arch) 21​ utils.log_job(config.joblog_file, item, item_type, task, arch)
18 except IOError, e: 22​ except IOError, e:
19 self.log.warning('Cannot log a job into %s', config.joblog_file) 23​ self.log.exception(e)
20 24​
21 output = self.trigger.trigger_job(item, item_type, task, arch) 25​ output = self.trigger.trigger_job(item, item_type, task, arch)
22 for line in output: 26​ for line in output:
23 self.log.info(line) 27​ self.log.info(line)
View Options

jobtriggers/compose_complete_msg.py

1import fedmsg 1​import fedmsg
2import fedmsg.encoding 2​import fedmsg.encoding
3 3​
4from . import config 4​from . import config
5from . import JobTrigger 5​from . import JobTrigger
6 6​
7ITEM_TYPE = 'compose' 7​ITEM_TYPE = 'compose'
8 8​
9​
9class ComposeCompletedJobTrigger(JobTrigger): 10​class ComposeCompletedJobTrigger(JobTrigger):
10 topic = "org.fedoraproject.prod.compose.*" 11​ topic = "org.fedoraproject.prod.compose.*"
11 jsonify = False 12​ jsonify = False
12 config_key = 'taskotron.composecompletedjobtrigger.enabled' 13​ config_key = 'taskotron.composecompletedjobtrigger.enabled'
13 14​
14 def __init__(self, *args, **kw): 15​ def __init__(self, *args, **kw):
15 super(ComposeCompletedJobTrigger, self).__init__(*args, **kw) 16​ super(ComposeCompletedJobTrigger, self).__init__(*args, **kw)
16 17​
Show All 18 Lines
View Options

jobtriggers/config.py

1import ConfigParser 1​import ConfigParser
2import io 2​import io
3 3​
4defaults = """ 4​defaults = """
5[buildbot] 5​[buildbot]
6url = http://localhost:8080/buildbot/change_hook 6​url = http://localhost:8080/buildbot/change_hook
7 7​
8[trigger] 8​[trigger]
9koji_build_completed_tasks = rpmlint 9​koji_build_completed_tasks = rpmlint
10koji_tag_changed_tasks = upgradepath,depcheck 10​koji_tag_changed_tasks = upgradepath,depcheck
11compose_completed_tasks = 11​compose_completed_tasks =
12valid_arches = x86_64 12​valid_arches = x86_64
13koji_url = http://koji.fedoraproject.org/kojihub 13​koji_url = http://koji.fedoraproject.org/kojihub
14​datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw
14execdb_server = http://localhost:5003 15​execdb_server = http://localhost:5003
15type = StreamTrigger 16​type = StreamTrigger
16job_logging = False 17​job_logging = False
17joblog_file = /var/log/taskotron-trigger/jobs.cvs 18​joblog_file = /var/log/taskotron-trigger/jobs.csv
18fuse_delay = 1000 ; seconds 19​fuse_delay = 1000 ; seconds
19""" 20​"""
20 21​
21config = ConfigParser.ConfigParser() 22​config = ConfigParser.ConfigParser()
22config.readfp(io.BytesIO(defaults)) 23​config.readfp(io.BytesIO(defaults))
23config.read(['/etc/taskotron/trigger.cfg', './conf/trigger.cfg']) 24​config.read(['/etc/taskotron/trigger.cfg', './conf/trigger.cfg'])
24 25​
25 26​
26buildbot_url = config.get('buildbot', 'url') 27​buildbot_url = config.get('buildbot', 'url')
27 28​
28koji_build_completed_tasks = [e.strip() for e in config.get('trigger', 'koji_build_completed_tasks').split(',') if e] 29​koji_build_completed_tasks = [e.strip() for e in config.get('trigger', 'koji_build_completed_tasks').split(',') if e]
29koji_tag_changed_tasks = [e.strip() for e in config.get('trigger', 'koji_tag_changed_tasks').split(',') if e] 30​koji_tag_changed_tasks = [e.strip() for e in config.get('trigger', 'koji_tag_changed_tasks').split(',') if e]
30compose_completed_tasks = [e.strip() for e in config.get('trigger', 'compose_completed_tasks').split(',') if e] 31​compose_completed_tasks = [e.strip() for e in config.get('trigger', 'compose_completed_tasks').split(',') if e]
31valid_arches = [e.strip() for e in config.get('trigger', 'valid_arches').split(',') if e] 32​valid_arches = [e.strip() for e in config.get('trigger', 'valid_arches').split(',') if e]
32koji_url = config.get('trigger', 'koji_url') 33​koji_url = config.get('trigger', 'koji_url')
34​datagrepper_url = config.get('trigger', 'datagrepper_url')
33execdb_server = config.get('trigger', 'execdb_server') 35​execdb_server = config.get('trigger', 'execdb_server')
34trigger_type = config.get('trigger', 'type') 36​trigger_type = config.get('trigger', 'type')
35job_logging = True if config.get('trigger', 'job_logging') == 'True' else False 37​job_logging = True if config.get('trigger', 'job_logging') == 'True' else False
36joblog_file = config.get('trigger', 'joblog_file') 38​joblog_file = config.get('trigger', 'joblog_file')
37fuse_delay = int(config.get('trigger', 'fuse_delay')) # seconds 39​fuse_delay = int(config.get('trigger', 'fuse_delay')) # seconds
View Options

jobtriggers/jobrunner.py

1import csv 1​import csv
2import argparse 2​import argparse
3​import itertools
4​import logging
5​from copy import deepcopy
3 6​
4from . import triggers, config 7​import requests
8​from . import config
9​from . import koji_build_msg
10​from . import koji_tag_msg
11​from . import TriggerMsgError
12​from .triggers import get_trigger
5 13​
6 14​
7TRIGGER = triggers.get_trigger(config.trigger_type) 15​trigger = get_trigger(config.trigger_type)
8 16​
17​# set up logging
18​logger = logging.getLogger('Jobrunner')
19​sh = logging.StreamHandler()
20​formatter = logging.Formatter(fmt='%(asctime)s - %(name)s - %(levelname)s: %(message)s',
21​ datefmt='%Y-%m-%d %H:%M:%S')
22​sh.setFormatter(formatter)
23​logger.addHandler(sh)
9 24​
10def run_jobs(jobsfile): 25​
26​def trigger_task(item, item_type, task, arch):
27​ logger.debug('triggering %s for %s on %s' % (task, item, arch))
28​
29​ output = trigger.trigger_job(item, item_type, task, arch)
30​ for line in output:
31​ logger.info(line)
32​
33​
34​def trigger_tasks(item, item_type, tasks, arches):
35​ for task, arch in itertools.product(tasks, arches):
36​ trigger_task(item, item_type, task, arch)
37​
38​
39​def file_jobs(jobsfile):
11 with open(jobsfile, 'r') as csvfile: 40​ with open(jobsfile, 'r') as csvfile:
12 jobs = csv.reader(csvfile, delimiter=';') 41​ jobs = csv.reader(csvfile, delimiter=';')
13 for job in jobs: 42​ for job in jobs:
14 item, item_type, task, arch = job[1], job[2], job[3], job[4] 43​ trigger_task(job[1], job[2], job[3], job[4])
15 print 'triggering %s for %s on %s' % (task, item, arch) ​
16 output = TRIGGER.trigger_job(item, item_type, task, arch) ​
17 for line in output: ​
18 print line ​
19 44​
20 45​
21def parse_args(): 46​def datagrepper_jobs(start, end):
22 parser = argparse.ArgumentParser() 47​ params = {
23 parser.add_argument('jobsfile') 48​ 'start': start,
49​ 'end': end,
50​ 'category': 'buildsys',
51​ 'topic': ['org.fedoraproject.prod.buildsys.build.state.change',
52​ 'org.fedoraproject.prod.buildsys.tag'],
53​ }
24 54​
25 args = parser.parse_args() 55​ r = requests.get(config.datagrepper_url, params=params)
56​
57​ koji_tag_items = set()
58​ for msg in r.json()['raw_messages']:
59​ if msg['topic'].endswith('build.state.change'):
60​ try:
61​ item = koji_build_msg.get_item(msg)
62​ item_type = koji_build_msg.ITEM_TYPE
63​ tasks = config.koji_build_completed_tasks
64​ trigger_tasks(item, item_type, tasks, config.valid_arches)
65​ except TriggerMsgError, e:
66​ logger.debug(e)
67​ elif msg['topic'].endswith('tag'):
68​ try:
69​ koji_tag_items.add(koji_tag_msg.get_item(msg))
70​ except TriggerMsgError, e:
71​ logger.debug(e)
72​ else:
73​ logger.warning('Got an unexpected topic %s', msg['topic'])
26 74​
27 return args 75​ item_type = koji_tag_msg.ITEM_TYPE
76​ for item in koji_tag_items:
77​ tasks = deepcopy(config.koji_tag_changed_tasks)
78​ if item.endswith('testing-pending'):
79​ tasks.remove('upgradepath')
80​ trigger_tasks(item, item_type, tasks, config.valid_arches)
28 81​
29 82​
30def main(): 83​def main():
31 args = parse_args() 84​ parser = argparse.ArgumentParser(prog='Jobrunner',
32 run_jobs(args.jobsfile) 85​ usage='Gets jobs from datagrepper from a given time '
86​ 'period and runs them.')
87​ parser.add_argument('-s', '--start', help='Start time (Unix time format)')
88​ parser.add_argument('-e', '--end', help='End time (Unix time format)')
89​ parser.add_argument('-d', '--debug', action='store_true', help='Print debug message')
90​ parser.add_argument('-f', '--jobsfile', help='A file containing jobs in csv format')
91​ args = parser.parse_args()
92​
93​ logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
94​
95​ # let's make running jobs from the file a priority
96​ if args.jobsfile:
97​ file_jobs(args.jobsfile)
98​ else:
99​ datagrepper_jobs(args.start, args.end)
View Options

jobtriggers/koji_build_msg.py

1import koji 1​import koji
2import fedmsg 2​import fedmsg
3import fedmsg.encoding 3​import fedmsg.encoding
4 4​
5from . import utils, config 5​from . import config, JobTrigger, TriggerMsgError
6from . import JobTrigger ​
7 6​
8ITEM_TYPE = 'koji_build' 7​ITEM_TYPE = 'koji_build'
9 8​
10class KojiBuildCompletedJobTrigger(JobTrigger): ​
11 topic = "org.fedoraproject.prod.buildsys.build.state.change" ​
12 jsonify = False ​
13 config_key = 'taskotron.kojibuildcompletedjobtrigger.enabled' ​
14 9​
15 def __init__(self, *args, **kw): 10​def get_item(msg):
16 super(KojiBuildCompletedJobTrigger, self).__init__(*args, **kw) 11​ release = msg['msg']['release']
12​ state = msg['msg']['new']
13​ instance = msg['msg']['instance']
17 14​
18 def consume(self, message): 15​ name = msg['msg']['name']
19 usable_message = fedmsg.encoding.loads(message.body) 16​ version = msg['msg']['version']
17​
18​ envr = "%s-%s-%s" % (name, version, release)
20 19​
21 release = usable_message['msg']['release'] 20​ if not instance == 'primary':
22 state = usable_message['msg']['new'] 21​ raise TriggerMsgError('rejecting message not from primary koji instance (%s from %s)' % (envr, instance))
23 instance = usable_message['msg']['instance'] ​
24 22​
25 name = usable_message['msg']['name'] 23​ if state != koji.BUILD_STATES['COMPLETE']:
26 version = usable_message['msg']['version'] 24​ raise TriggerMsgError('rejecting message for not being a build complete message (%s, status %s)' % (envr, state))
27 buildid = usable_message['msg']['build_id'] ​
28 25​
29 envr = "%s-%s-%s" % (name, version, release) 26​ if 'fc' not in release:
27​ raise TriggerMsgError('rejecting message for not containing "fc" in release (%s)' % envr)
30 28​
31 if not instance == 'primary': 29​ return envr
32 self.log.debug('rejecting message not from primary koji instance (%s from %s)' % (envr, instance)) ​
33 return ​
34 30​
35 if state != koji.BUILD_STATES['COMPLETE']: ​
36 self.log.debug('rejecting message for not being a build complete message (%s, status %s)' % (envr, state)) ​
37 return ​
38 31​
39 if 'fc' not in release: 32​class KojiBuildCompletedJobTrigger(JobTrigger):
40 self.log.debug('rejecting message for not containing "fc" in release (%s)' % envr) 33​ topic = "org.fedoraproject.prod.buildsys.build.state.change"
41 return 34​ jsonify = False
35​ config_key = 'taskotron.kojibuildcompletedjobtrigger.enabled'
42 36​
43 self.log.debug('should be scheduling job for %s' % envr) 37​ def __init__(self, *args, **kw):
38​ super(KojiBuildCompletedJobTrigger, self).__init__(*args, **kw)
39​
40​ def consume(self, message):
41​ msg = fedmsg.encoding.loads(message.body)
44 42​
45 self.trigger_tasks(envr, ITEM_TYPE, config.koji_build_completed_tasks, config.valid_arches) 43​ try:
44​ item = get_item(msg)
45​ self.trigger_tasks(item, ITEM_TYPE, config.koji_build_completed_tasks, config.valid_arches)
46​ except TriggerMsgError, e:
47​ self.log.debug(e)
View Options

jobtriggers/koji_tag_msg.py

1import re 1​import re
2from copy import deepcopy 2​from copy import deepcopy
3 3​
4import fedmsg 4​import fedmsg
5import fedmsg.encoding 5​import fedmsg.encoding
6import moksha.hub.reactor 6​import moksha.hub.reactor
7 7​
8from twisted.internet.task import LoopingCall 8​from twisted.internet.task import LoopingCall
9 9​
10from . import config, utils 10​from . import config, JobTrigger, TriggerMsgError
11from . import JobTrigger ​
12 11​
13ITEM_TYPE = 'koji_tag' 12​ITEM_TYPE = 'koji_tag'
14 13​
14​
15​def get_item(msg):
16​ koji_tag = msg['msg']['tag']
17​ instance = msg['msg']['instance']
18​
19​ if not instance == 'primary':
20​ raise TriggerMsgError('rejecting message not from primary koji instance (%s from %s)' % (koji_tag, instance))
21​
22​ if not koji_tag.endswith('pending'):
23​ raise TriggerMsgError('rejecting message for not being a pending tag (%s)' % koji_tag)
24​
25​ m = re.match(r'^f\d\d-.*', koji_tag)
26​ if m is None:
27​ raise TriggerMsgError('rejecting message for not starting with fedora version (e.g. f22) (%s)' % koji_tag)
28​
29​ return koji_tag
30​
31​
15class KojiTagChangedJobTrigger(JobTrigger): 32​class KojiTagChangedJobTrigger(JobTrigger):
16 topic = "org.fedoraproject.prod.buildsys.tag" 33​ topic = "org.fedoraproject.prod.buildsys.tag"
17 jsonify = False 34​ jsonify = False
18 config_key = 'taskotron.kojitagchangedjobtrigger.enabled' 35​ config_key = 'taskotron.kojitagchangedjobtrigger.enabled'
19 36​
20 def __init__(self, *args, **kw): 37​ def __init__(self, *args, **kw):
21 super(KojiTagChangedJobTrigger, self).__init__(*args, **kw) 38​ super(KojiTagChangedJobTrigger, self).__init__(*args, **kw)
22 39​
Show All 13 Lines52 if tag.endswith('testing-pending'):
36 tasks.remove('upgradepath') 53​ tasks.remove('upgradepath')
37 self.trigger_tasks(tag, ITEM_TYPE, tasks, config.valid_arches) 54​ self.trigger_tasks(tag, ITEM_TYPE, tasks, config.valid_arches)
38 finally: 55​ finally:
39 self.queued_tags = set() 56​ self.queued_tags = set()
40 else: 57​ else:
41 self.log.debug("Woke up, but there were no messages.") 58​ self.log.debug("Woke up, but there were no messages.")
42 59​
43 def consume(self, message): 60​ def consume(self, message):
44 usable_message = fedmsg.encoding.loads(message.body) 61​ msg = fedmsg.encoding.loads(message.body)
45 ​
46 koji_tag = usable_message['msg']['tag'] ​
47 instance = usable_message['msg']['instance'] ​
48 ​
49 if not instance == 'primary': ​
50 self.log.debug('rejecting message not from primary koji instance (%s from %s)' % (koji_tag, instance)) ​
51 return ​
52 ​
53 if not koji_tag.endswith('pending'): ​
54 self.log.debug('rejecting message for not being a pending tag (%s)' % koji_tag) ​
55 return ​
56 62​
57 m = re.match(r'^f\d\d-.*', koji_tag) 63​ try:
58 if m is None: 64​ item = get_item(msg)
59 self.log.debug('rejecting message for not starting with fedora version (e.g. f22) (%s)' % koji_tag) ​
60 return ​
61 65​
62 if not len(self.queued_tags): 66​ if not len(self.queued_tags):
63 self.log.info('setting fuse (%d seconds) for koji_tag %s', config.fuse_delay, koji_tag) 67​ self.log.info('setting fuse (%d seconds) for koji_tag %s', config.fuse_delay, item)
64 elif koji_tag not in self.queued_tags: 68​ elif item not in self.queued_tags:
65 self.log.info('adding koji_tag %s to the fuse queue', koji_tag) 69​ self.log.info('adding koji_tag %s to the fuse queue', item)
66 70​
67 self.queued_tags.add(koji_tag) 71​ self.queued_tags.add(item)
72​ except TriggerMsgError, e:
73​ self.log.debug(e)
View Options

jobtriggers/utils.py

1import csv 1​import csv
2import datetime 2​import datetime
3 3​
4from . import config ​
5 4​
6def log_job(logname, item, item_type, task, arch): 5​def log_job(logname, item, item_type, task, arch):
7 now = datetime.datetime.now() 6​ now = datetime.datetime.now()
8 7​
9 with open(logname, 'a') as csvfile: 8​ with open(logname, 'a') as csvfile:
10 writer = csv.writer(csvfile, delimiter=';') 9​ writer = csv.writer(csvfile, delimiter=';')
11 writer.writerow([now, item, item_type, task, arch]) 10​ writer.writerow([now, item, item_type, task, arch])
View Options

requirements.txt

1dingus==0.3.4 1​dingus == 0.3.4
2pytest>=2.4.2 2​munch == 2.0.2
3pytest-cov>=1.6 3​pytest == 2.8.3
4​pytest-cov == 2.2.0
View Options

taskotron-trigger.spec

1Name: taskotron-trigger 1​Name: taskotron-trigger
2Version: 0.3.12 2​Version: 0.3.12
3Release: 1%{?dist} 3​Release: 1%{?dist}
4Summary: Triggering Taskotron jobs via fedmsg. 4​Summary: Triggering Taskotron jobs via fedmsg.
5 5​
6License: GPLv2+ 6​License: GPLv2+
7URL: https://fedorahosted.org/fedora-qa 7​URL: https://fedorahosted.org/fedora-qa
8Source0: %{name}-%{version}.tar.gz 8​Source0: %{name}-%{version}.tar.gz
9 9​
10BuildArch: noarch 10​BuildArch: noarch
11 11​
12BuildRequires: python-setuptools ​
13Requires: koji 12​Requires: koji
14Requires: python-bunch 13​Requires: python-munch
15Requires: python-requests 14​Requires: python-requests
16Requires: python-twisted 15​Requires: python-twisted
17Requires: fedmsg 16​Requires: fedmsg
18Requires: fedmsg-hub 17​Requires: fedmsg-hub
18​BuildRequires: python-setuptools
19 19​
20%description 20​%description
21Triggering Taskotron jobs via fedmsg. 21​Triggering Taskotron jobs via fedmsg.
22 22​
23 23​
24%prep 24​%prep
25%setup -q 25​%setup -q
26 26​
▲ Show 20 Lines • Show All 93 Lines • Show Last 20 Lines
View Options

testing/functest_cli.py

1import pytest ​
2from dingus import Dingus 1​from dingus import Dingus
3 2​
4import jobtriggers.jobrunner 3​import jobtriggers.jobrunner
5from jobtriggers.utils import log_job 4​from jobtriggers.utils import log_job
6 5​
7 6​
8class TestCli(): 7​class TestCli():
9 def setup_method(self, method): 8​ def setup_method(self, method):
10 self.ref_tests = ['rpmlint', 'upgradepath'] 9​ self.ref_tests = ['rpmlint', 'upgradepath']
11 self.ref_unit_types = ['koji_build', 'bodhi_id'] 10​ self.ref_unit_types = ['koji_build', 'bodhi_id']
12 self.ref_nvr = 'xchat-2.8.8-21.fc20' 11​ self.ref_nvr = 'xchat-2.8.8-21.fc20'
13 self.ref_arch = 'x86_64' 12​ self.ref_arch = 'x86_64'
14 13​
15 def test_run_jobs(self, tmpdir, monkeypatch): 14​ def test_run_jobs(self, tmpdir, monkeypatch):
16 f = tmpdir.join('jobs.csv') 15​ f = tmpdir.join('jobs.csv')
17 filename = str(f.realpath()) 16​ filename = str(f.realpath())
18 17​
19 monkeypatch.setattr(jobtriggers.jobrunner, 'TRIGGER', Dingus()) 18​ monkeypatch.setattr(jobtriggers.jobrunner, 'trigger', Dingus())
20 19​
21 log_job(filename, self.ref_tests[0], self.ref_unit_types[0], self.ref_nvr, self.ref_arch) 20​ log_job(filename, self.ref_tests[0], self.ref_unit_types[0], self.ref_nvr, self.ref_arch)
22 log_job(filename, self.ref_tests[1], self.ref_unit_types[1], self.ref_nvr, self.ref_arch) 21​ log_job(filename, self.ref_tests[1], self.ref_unit_types[1], self.ref_nvr, self.ref_arch)
23 22​
24 jobtriggers.jobrunner.run_jobs(filename) 23​ jobtriggers.jobrunner.file_jobs(filename)
25 24​
26 trigger_calls = jobtriggers.jobrunner.TRIGGER.trigger_job.calls() 25​ trigger_calls = jobtriggers.jobrunner.trigger.trigger_job.calls()
27 26​
28 assert len(trigger_calls) == 2 27​ assert len(trigger_calls) == 2
29 28​
30 assert trigger_calls[0][1] == (self.ref_tests[0], self.ref_unit_types[0], self.ref_nvr, self.ref_arch) 29​ assert trigger_calls[0][1] == (self.ref_tests[0], self.ref_unit_types[0], self.ref_nvr, self.ref_arch)
31 assert trigger_calls[1][1] == (self.ref_tests[1], self.ref_unit_types[1], self.ref_nvr, self.ref_arch) 30​ assert trigger_calls[1][1] == (self.ref_tests[1], self.ref_unit_types[1], self.ref_nvr, self.ref_arch)
View Options

testing/test_compose_trigger.py

1import pytest 1​import pytest
2from dingus import Dingus 2​from dingus import Dingus
3from bunch import Bunch 3​from munch import Munch
4 4​
5from jobtriggers import compose_complete_msg 5​from jobtriggers import compose_complete_msg
6 6​
7 7​
8@pytest.mark.usefixtures('prepare') 8​@pytest.mark.usefixtures('prepare')
9class TestComposeCompletedJobTrigger(): 9​class TestComposeCompletedJobTrigger():
10 @pytest.fixture 10​ @pytest.fixture
11 def prepare(self, monkeypatch): 11​ def prepare(self, monkeypatch):
12 self.ref_arch = '' 12​ self.ref_arch = ''
13 self.ref_branch = '21' 13​ self.ref_branch = '21'
14 self.ref_topic = 'org.fedoraproject.prod.compose.branched.complete' 14​ self.ref_topic = 'org.fedoraproject.prod.compose.branched.complete'
15 15​
16 self._create_msg(self.ref_arch, self.ref_branch, self.ref_topic) 16​ self._create_msg(self.ref_arch, self.ref_branch, self.ref_topic)
17 17​
18 self.ref_tasks = ['compose_task1', 'compose_task2'] 18​ self.ref_tasks = ['compose_task1', 'compose_task2']
19 self.ref_arches = 'noarch' 19​ self.ref_arches = 'noarch'
20 20​
21 stub_hub = Bunch(config=Bunch(get=0)) 21​ stub_hub = Munch(config=Munch(get=0))
22 22​
23 self.helper = compose_complete_msg.ComposeCompletedJobTrigger(stub_hub) 23​ self.helper = compose_complete_msg.ComposeCompletedJobTrigger(stub_hub)
24 self.helper.trigger = Dingus() 24​ self.helper.trigger = Dingus()
25 25​
26 compose_complete_msg.config.compose_completed_tasks = self.ref_tasks 26​ compose_complete_msg.config.compose_completed_tasks = self.ref_tasks
27 compose_complete_msg.config.valid_arches = self.ref_arches 27​ compose_complete_msg.config.valid_arches = self.ref_arches
28 compose_complete_msg.config.job_logging = False 28​ compose_complete_msg.config.job_logging = False
29 29​
30 def _create_msg(self, ref_arch, ref_branch, ref_topic): 30​ def _create_msg(self, ref_arch, ref_branch, ref_topic):
31 self.ref_message = Bunch(body='{"i": 1,\ 31​ self.ref_message = Munch(body='{"i": 1,\
32 "msg": {"arch": "%s",\ 32​ "msg": {"arch": "%s",\
33 "branch": "%s",\ 33​ "branch": "%s",\
34 "log": "done"},\ 34​ "log": "done"},\
35 "topic": "%s"}' % 35​ "topic": "%s"}' %
36 (ref_arch, ref_branch, ref_topic)) 36​ (ref_arch, ref_branch, ref_topic))
37 37​
38 def test_consume(self): 38​ def test_consume(self):
39 self.helper.consume(self.ref_message) 39​ self.helper.consume(self.ref_message)
Show All 39 Lines
View Options

testing/test_koji_build_trigger.py

1import koji 1​import koji
2import pytest 2​import pytest
3from dingus import Dingus 3​from dingus import Dingus
4from bunch import Bunch 4​from munch import Munch
5 5​
6from jobtriggers import koji_build_msg 6​from jobtriggers import koji_build_msg
7 7​
8 8​
9@pytest.mark.usefixtures('prepare') 9​@pytest.mark.usefixtures('prepare')
10class TestKojiBuildCompletedJobTrigger(): 10​class TestKojiBuildCompletedJobTrigger():
11 @pytest.fixture 11​ @pytest.fixture
12 def prepare(self, monkeypatch): 12​ def prepare(self, monkeypatch):
13 self.ref_instance = 'primary' 13​ self.ref_instance = 'primary'
14 self.ref_new = koji.BUILD_STATES['COMPLETE'] 14​ self.ref_new = koji.BUILD_STATES['COMPLETE']
15 self.ref_name = 'stage' 15​ self.ref_name = 'stage'
16 self.ref_version = '4.1.1' 16​ self.ref_version = '4.1.1'
17 self.ref_release = '3.fc18' 17​ self.ref_release = '3.fc18'
18 self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release) 18​ self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release)
19 19​
20 self._create_msg(self.ref_instance, self.ref_new, self.ref_name, self.ref_release, self.ref_version) 20​ self._create_msg(self.ref_instance, self.ref_new, self.ref_name, self.ref_release, self.ref_version)
21 21​
22 self.ref_tasks = ['koji_build_task1', 'koji_build_task2'] 22​ self.ref_tasks = ['koji_build_task1', 'koji_build_task2']
23 self.ref_validarches = ['i386', 'x86_64'] 23​ self.ref_validarches = ['i386', 'x86_64']
24 24​
25 stub_hub = Bunch(config=Bunch(get=0)) 25​ stub_hub = Munch(config=Munch(get=0))
26 26​
27 self.helper = koji_build_msg.KojiBuildCompletedJobTrigger(stub_hub) 27​ self.helper = koji_build_msg.KojiBuildCompletedJobTrigger(stub_hub)
28 self.helper.trigger = Dingus() 28​ self.helper.trigger = Dingus()
29 29​
30 koji_build_msg.config.koji_build_completed_tasks = self.ref_tasks 30​ koji_build_msg.config.koji_build_completed_tasks = self.ref_tasks
31 koji_build_msg.config.valid_arches = self.ref_validarches 31​ koji_build_msg.config.valid_arches = self.ref_validarches
32 koji_build_msg.config.job_logging = False 32​ koji_build_msg.config.job_logging = False
33 33​
34 def _create_msg(self, ref_instance, ref_new, ref_name, ref_release, ref_version): 34​ def _create_msg(self, ref_instance, ref_new, ref_name, ref_release, ref_version):
35 self.ref_message = Bunch(body='{"i": 1,\ 35​ self.ref_message = Munch(body='{"i": 1,\
36 "msg": {"build_id": 221146,\ 36​ "msg": {"build_id": 221146,\
37 "instance": "%s",\ 37​ "instance": "%s",\
38 "new": %d,\ 38​ "new": %d,\
39 "name": "%s",\ 39​ "name": "%s",\
40 "release": "%s",\ 40​ "release": "%s",\
41 "version": "%s"},\ 41​ "version": "%s"},\
42 "timestamp": 1359603469.21164,\ 42​ "timestamp": 1359603469.21164,\
43 "topic": "org.fedoraproject.prod.buildsys.build.state.change",\ 43​ "topic": "org.fedoraproject.prod.buildsys.build.state.change",\
▲ Show 20 Lines • Show All 44 Lines • Show Last 20 Lines
View Options

testing/test_koji_tag_trigger.py

1import pytest 1​import pytest
2from dingus import Dingus 2​from dingus import Dingus
3from bunch import Bunch 3​from munch import Munch
4 4​
5from jobtriggers import koji_tag_msg 5​from jobtriggers import koji_tag_msg
6 6​
7 7​
8@pytest.mark.usefixtures('prepare') 8​@pytest.mark.usefixtures('prepare')
9class TestKojiTagChangedJobTrigger(): 9​class TestKojiTagChangedJobTrigger():
10 @pytest.fixture 10​ @pytest.fixture
11 def prepare(self, monkeypatch): 11​ def prepare(self, monkeypatch):
12 self.ref_instance = 'primary' 12​ self.ref_instance = 'primary'
13 self.ref_name = 'stage' 13​ self.ref_name = 'stage'
14 self.ref_version = '4.1.1' 14​ self.ref_version = '4.1.1'
15 self.ref_release = '3.fc18' 15​ self.ref_release = '3.fc18'
16 self.ref_tag = 'f18-updates-pending' 16​ self.ref_tag = 'f18-updates-pending'
17 self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release) 17​ self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release)
18 18​
19 self.ref_tasks = ['depcheck', 'upgradepath'] 19​ self.ref_tasks = ['depcheck', 'upgradepath']
20 self.ref_validarches = ['i386', 'x86_64'] 20​ self.ref_validarches = ['i386', 'x86_64']
21 21​
22 self._create_msg(self.ref_instance, self.ref_name, self.ref_release, self.ref_tag, self.ref_version) 22​ self._create_msg(self.ref_instance, self.ref_name, self.ref_release, self.ref_tag, self.ref_version)
23 23​
24 stub_hub = Bunch(config=Bunch(get=0)) 24​ stub_hub = Munch(config=Munch(get=0))
25 self.helper = koji_tag_msg.KojiTagChangedJobTrigger(stub_hub) 25​ self.helper = koji_tag_msg.KojiTagChangedJobTrigger(stub_hub)
26 26​
27 self.helper.trigger = Dingus() 27​ self.helper.trigger = Dingus()
28 28​
29 koji_tag_msg.config.koji_tag_changed_tasks = self.ref_tasks 29​ koji_tag_msg.config.koji_tag_changed_tasks = self.ref_tasks
30 koji_tag_msg.config.valid_arches = self.ref_validarches 30​ koji_tag_msg.config.valid_arches = self.ref_validarches
31 koji_tag_msg.config.job_logging = False 31​ koji_tag_msg.config.job_logging = False
32 32​
33 def _create_msg(self, ref_instance, ref_name, ref_release, ref_tag, ref_version): 33​ def _create_msg(self, ref_instance, ref_name, ref_release, ref_tag, ref_version):
34 self.ref_message = Bunch(body='{"i": 1,\ 34​ self.ref_message = Munch(body='{"i": 1,\
35 "msg": {"build_id": 221146,\ 35​ "msg": {"build_id": 221146,\
36 "instance": "%s",\ 36​ "instance": "%s",\
37 "name": "%s",\ 37​ "name": "%s",\
38 "owner": "ralph",\ 38​ "owner": "ralph",\
39 "release": "%s",\ 39​ "release": "%s",\
40 "tag": "%s",\ 40​ "tag": "%s",\
41 "tag_id": 216,\ 41​ "tag_id": 216,\
42 "user": "bodhi",\ 42​ "user": "bodhi",\
▲ Show 20 Lines • Show All 76 Lines • Show Last 20 Lines