Differential D36

Trigger jobs for all arches
ClosedPublic
Actions

Authored by mkrizek on Mar 31 2014, 1:22 PM.

Details

Reviewers
ralph
tflink
Maniphest Tasks
T109: Trigger jobs for all arches
Commits
rTRGRc8a78404ddec: Trigger jobs for all arches
Summary

This also adds temporary configuration file (settings.py) - configuration should probably be done differently but that will be handled in another ticket. Fixes T109

Test Plan

Tests included

Diff Detail

Lint
Lint Skipped
Unit
Unit Tests Skipped
mkrizek added a comment.Mar 31 2014, 1:54 PM

Note that this is against taskotron-trigger, not libtaskotron. The .arcconfig should be fixed in D37.

tflink added a comment.Apr 1 2014, 7:26 AM

Other than the question about 'noarch' in utils, looks good to me

jobtriggers/utils.py
19

What would this do and why? It looks like it would add noarch if there are no valid arches found in a build's rpms?

mkrizek updated this revision.Apr 1 2014, 8:40 AM

Remove needless condition in util.get_arches

jobtriggers/utils.py
19

Ah, this should be removed since 'noarch' is in valid_arches.

tflink accepted this revision.Apr 1 2014, 8:03 PM

Looks good to me

mkrizek closed this revision.Apr 2 2014, 8:47 AM

Closed by commit rTRGRc8a78404ddec (authored by @mkrizek).

Revision Contents

Diff 105

View Options

.gitignore

1config.ini 1​config.ini
2.*.swp 2​.*.swp
3*.pyc 3​*.pyc
4*.pyo 4​*.pyo
5​env_trigger/
6​jobtriggers.egg-info/
View Options

fedmsg.d/taskotron-trigger.py

1config = { 1​config = {
2 'taskotron.kojijobtrigger.enabled': True, 2​ 'taskotron.kojijobtrigger.enabled': True,
3 'taskotron.bodhijobtrigger.enabled': True, 3​ 'taskotron.bodhijobtrigger.enabled': True,
4 'taskotron.buildbot': { ​
5 'username': "username", ​
6 'password': "password", ​
7 'url': "http://localhost/buildbot", ​
8 }, ​
9} 4​}
View Options

jobtriggers/bodhi_msg.py

1import logging 1​import logging
2 ​
3import fedmsg 2​import fedmsg
4import fedmsg.encoding 3​import fedmsg.encoding
5import fedmsg.consumers 4​import fedmsg.consumers
6 5​
7from . import triggers 6​from . import triggers, utils, settings
7​
8 8​
9UNIT_TYPE = 'bodhi_update' 9​UNIT_TYPE = 'bodhi_update'
10 10​
11logging.basicConfig() 11​logging.basicConfig()
12log = logging.getLogger(__name__) 12​log = logging.getLogger(__name__)
13 13​
14 14​
15class BodhiJobTrigger(fedmsg.consumers.FedmsgConsumer): 15​class BodhiJobTrigger(fedmsg.consumers.FedmsgConsumer):
16 topic = "org.fedoraproject.prod.bodhi.update.request.*" 16​ topic = "org.fedoraproject.prod.bodhi.update.request.*"
17 jsonify = False 17​ jsonify = False
18 config_key = 'taskotron.bodhijobtrigger.enabled' 18​ config_key = 'taskotron.bodhijobtrigger.enabled'
19 19​
20 def __init__(self, hub): 20​ def __init__(self, *args, **kw):
21 self.hub = hub 21​ super(BodhiJobTrigger, self).__init__(*args, **kw)
22 22​ self.trigger = triggers.StreamTrigger()
23 super(BodhiJobTrigger, self).__init__(hub) ​
24 self.trigger = triggers.StreamTrigger(hub.config) ​
25 23​
26 def consume(self, message): 24​ def consume(self, message):
27 usable_message = fedmsg.encoding.loads(message.body) 25​ usable_message = fedmsg.encoding.loads(message.body)
28 26​
29 topic = usable_message['topic'] 27​ topic = usable_message['topic']
30 28​
31 if not topic.endswith('stable') and not topic.endswith('testing'): 29​ if not topic.endswith('stable') and not topic.endswith('testing'):
32 return 30​ return
33 31​
34 updatetitle = usable_message['msg']['update']['title'] 32​ updatetitle = usable_message['msg']['update']['title']
33​ builds = usable_message['msg']['update']['builds']
34​ nvrs = [build['nvr'] for build in builds]
35 35​
36 log.info('forcing update %s', updatetitle) 36​ arches = utils.get_arches(nvr=nvrs[0])
37 ​
38 self.trigger.trigger_job(updatetitle, UNIT_TYPE, 'examplebodhi') ​
39 37​
40 log.info('update forced %s', updatetitle) 38​ for task in settings.bodhi_tasks:
39​ for arch in arches:
40​ log.info('triggering %s for %s on %s', task, updatetitle, arch)
41​ self.trigger.trigger_job(updatetitle, UNIT_TYPE, task, arch)
View Options

jobtriggers/koji_msg.py

1import logging 1​import logging
2 2​
3import fedmsg 3​import fedmsg
4import fedmsg.encoding 4​import fedmsg.encoding
5import fedmsg.consumers 5​import fedmsg.consumers
6 6​
7from . import triggers 7​from . import triggers, utils, settings
8​
8 9​
9UNIT_TYPE = 'koji_build' 10​UNIT_TYPE = 'koji_build'
10 11​
11logging.basicConfig() 12​logging.basicConfig()
12log = logging.getLogger(__name__) 13​log = logging.getLogger(__name__)
13 14​
14 15​
15class KojiJobTrigger(fedmsg.consumers.FedmsgConsumer): 16​class KojiJobTrigger(fedmsg.consumers.FedmsgConsumer):
16 topic = "org.fedoraproject.prod.buildsys.tag" 17​ topic = "org.fedoraproject.prod.buildsys.tag"
17 jsonify = False 18​ jsonify = False
18 config_key = 'taskotron.kojijobtrigger.enabled' 19​ config_key = 'taskotron.kojijobtrigger.enabled'
19 20​
20 def __init__(self, hub): 21​ def __init__(self, *args, **kw):
21 self.hub = hub 22​ super(KojiJobTrigger, self).__init__(*args, **kw)
22 23​ self.trigger = triggers.StreamTrigger()
23 super(KojiJobTrigger, self).__init__(hub) ​
24 self.trigger = triggers.StreamTrigger(hub.config) ​
25 24​
26 def consume(self, message): 25​ def consume(self, message):
27 usable_message = fedmsg.encoding.loads(message.body) 26​ usable_message = fedmsg.encoding.loads(message.body)
28 27​
29 tag = usable_message['msg']['tag'] 28​ tag = usable_message['msg']['tag']
30 29​
31 if not tag.endswith('pending'): 30​ if not tag.endswith('pending'):
32 return 31​ return
33 32​
34 buildname = usable_message['msg']['name'] 33​ buildname = usable_message['msg']['name']
35 buildversion = usable_message['msg']['version'] 34​ buildversion = usable_message['msg']['version']
36 buildrelease = usable_message['msg']['release'] 35​ buildrelease = usable_message['msg']['release']
36​ buildid = usable_message['msg']['build_id']
37 37​
38 envr = "%s-%s-%s" % (buildname, buildversion, buildrelease) 38​ envr = "%s-%s-%s" % (buildname, buildversion, buildrelease)
39 39​
40 log.info('forcing build for %s', envr) 40​ arches = utils.get_arches(buildid=buildid)
41 ​
42 self.trigger.trigger_job(envr, UNIT_TYPE, 'rpmlint') ​
43 41​
44 log.info('build forced for %s', envr) 42​ for task in settings.koji_tasks:
43​ for arch in arches:
44​ log.info('triggering %s for %s on %s', task, envr, arch)
45​ self.trigger.trigger_job(envr, UNIT_TYPE, task, arch)
View Options

jobtriggers/settings.py

  • This file was added.
1​buildbot = {
2​ 'username': "username",
3​ 'password': "password",
4​ 'url': "http://localhost/buildbot",
5​}
6​koji_tasks = ['rpmlint']
7​bodhi_tasks = ['examplebodhi']
8​valid_arches = ['i386', 'i686', 'x86_64', 'noarch']
9​koji_url = "http://koji.fedoraproject.org/kojihub"
View Options

jobtriggers/triggers.py

1import pprint 1​import pprint
2import requests 2​import requests
3 3​
4​from . import settings
4 5​
5class BaseTrigger(object): 6​class BaseTrigger(object):
6 def __init__(self, config): 7​ """
7 self.config = config 8​ Format:
8 9​ {
9 def trigger_job(self, unit, unit_type, job_type): 10​ 'item': <name of item - envr, update id, koji tag, etc.>,
11​ 'item_type': <koji_build, koji_tag, bodhi_update>,
12​ 'taskname': <task to be scheduled>,
13​ 'arch': <arch needed for execution, x86_64, i386 etc.>
14​ }
15​ """
16​ def trigger_job(self, item, item_type, taskname, arch):
10 raise NotImplementedError 17​ raise NotImplementedError
11 18​
12 19​
13class StreamTrigger(BaseTrigger): 20​class StreamTrigger(BaseTrigger):
14 def trigger_job(self, unit, unit_type, job_type): 21​ def trigger_job(self, item, item_type, taskname, arch):
15 job = {'unit': unit, 'unit_type': unit_type, 'job_type': job_type} 22​ job = {'item': item, 'item_type': item_type, 'taskname': taskname, 'arch': arch}
16 pprint.pprint(job) 23​ pprint.pprint(job)
17 print 24​ print
18 25​
19 26​
20class BuildbotTrigger(BaseTrigger): 27​class BuildbotTrigger(BaseTrigger):
21 def __init__(self, config): 28​ def trigger_job(self, item, item_type, taskname, arch):
22 super(BuildbotTrigger, self).__init__(config) ​
23 ​
24 buildbot = self.config.get('taskotron.buildbot', {}) ​
25 self.buildbot_url = buildbot['url'] ​
26 self.username = buildbot['username'] ​
27 self.passwd = buildbot['password'] ​
28 ​
29 def trigger_job(self, unit, unit_type, job_type): ​
30 # TODO the builder is currently only set up for koji jobs ​
31 s = requests.session() 29​ s = requests.session()
32 30​
33 s.post('%s/login' % self.buildbot_url, 31​ s.post('%s/login' % settings.buildbot['url'],
34 data={'username': self.username, 32​ data={'username': settings.buildbot['username'],
35 'passwd': self.passwd}) 33​ 'passwd': settings.buildbot['password']})
36 34​
37 s.post('%s/builders/statictasks/force' % self.buildbot_url, 35​ s.post('%s/builders/statictasks/force' % settings.buildbot['url'],
38 data={'forcescheduler': 'rpmcheck', 36​ data={'forcescheduler': 'rpmcheck',
39 'reason': 'triggered by fedmsg', 37​ 'reason': 'triggered by fedmsg',
40 'checkname': job_type, 38​ 'item': item,
41 'envr': unit_type}) 39​ 'item_type': item_type,
40​ 'taskname': taskname,
41​ 'arch': arch})
42 42​
43 s.get('%s/logout' % self.buildbot_url) 43​ s.get('%s/logout' % settings.buildbot['url'])
View Options

jobtriggers/utils.py

  • This file was added.
1​import koji
2​
3​from . import settings
4​
5​koji_client = koji.ClientSession(settings.koji_url)
6​
7​def get_arches(buildid=None, nvr=None):
8​ if buildid is None:
9​ info = koji_client.getBuild(nvr)
10​ buildid = info['id']
11​
12​ arches = set([rpm['arch'] for rpm in koji_client.listRPMs(buildid) if rpm['arch'] in settings.valid_arches])
13​
14​ if 'i686' in arches:
15​ arches.remove('i686')
16​ arches.add('i386')
17​
18​ return arches
tflinkUnsubmitted
Not Done

What would this do and why? It looks like it would add noarch if there are no valid arches found in a build's rpms?

mkrizekAuthorUnsubmitted
Not Done

Ah, this should be removed since 'noarch' is in valid_arches.

View Options

testing/test_bodhi_trigger.py

1from dingus import Dingus 1​from dingus import Dingus
2from bunch import Bunch 2​from bunch import Bunch
3 3​
4from jobtriggers import bodhi_msg 4​from jobtriggers import bodhi_msg
5 5​
6 6​
7class TestBodhiJobTrigger(): 7​class TestBodhiJobTrigger():
8 def setup_method(self, method): 8​ def setup_method(self, method):
9 self.ref_updatetitle = 'foo' 9​ self.ref_updatetitle = 'foo'
10 self.ref_topic = 'org.fedoraproject.prod.bodhi.update.request.stable' 10​ self.ref_topic = 'org.fedoraproject.prod.bodhi.update.request.stable'
11 11​
12​ self.ref_tasks = ['bodhi_task1', 'bodhi_task2']
13​ self.ref_validarches = ['i386', 'x86_64']
14​
12 self._create_msg(self.ref_updatetitle, self.ref_topic) 15​ self._create_msg(self.ref_updatetitle, self.ref_topic)
13 16​
14 self.ref_config = { 17​ self.helper = bodhi_msg.BodhiJobTrigger(Dingus())
15 'taskotron.buildbot': { ​
16 'username': "username", ​
17 'password': "password", ​
18 'url': "http://localhost/buildbot/", ​
19 }, ​
20 } ​
21 self.stub_hub = Dingus(config__returns = self.ref_config) ​
22 18​
23 def _create_msg(self, ref_updatetitle, ref_topic): 19​ def _create_msg(self, ref_updatetitle, ref_topic):
24 self.ref_message = Bunch(body='{"msg": {"agent": "lmacken",\ 20​ self.ref_message = Bunch(body='{"msg": {"agent": "lmacken",\
25 "update": {"submitter": "lmacken", "title": "%s"}},\ 21​ "update": {"submitter": "lmacken",\
22​ "title": "%s",\
23​ "builds": [{"nvr": "gnome-settings-daemon-3.6.1-1.fc18"}]}},\
26 "topic": "%s"}' % 24​ "topic": "%s"}' %
27 (ref_updatetitle, ref_topic)) 25​ (ref_updatetitle, ref_topic))
28 26​
29 def test_consume(self, monkeypatch): 27​ def test_consume(self, monkeypatch):
30 self.helper = bodhi_msg.BodhiJobTrigger(self.stub_hub) ​
31 ​
32 monkeypatch.setattr(self.helper, 'trigger', Dingus()) 28​ monkeypatch.setattr(self.helper, 'trigger', Dingus())
29​ monkeypatch.setattr(bodhi_msg.settings, 'bodhi_tasks', self.ref_tasks)
30​ monkeypatch.setattr(bodhi_msg.settings, 'valid_arches', self.ref_validarches)
31​ monkeypatch.setattr(bodhi_msg, 'utils', Dingus(get_arches__returns=self.ref_validarches))
33 32​
34 self.helper.consume(self.ref_message) 33​ self.helper.consume(self.ref_message)
35 34​
36 trigger_calls = self.helper.trigger.trigger_job.calls() 35​ trigger_calls = self.helper.trigger.trigger_job.calls()
37 requested_updatetitle = trigger_calls[0][1][0] ​
38 requested_unit_type = trigger_calls[0][1][1] ​
39 36​
40 assert requested_updatetitle == self.ref_updatetitle 37​ assert len(trigger_calls) == 4
41 assert requested_unit_type == bodhi_msg.UNIT_TYPE 38​
39​ assert trigger_calls[0][1] == (self.ref_updatetitle, bodhi_msg.UNIT_TYPE, self.ref_tasks[0], self.ref_validarches[0])
40​ assert trigger_calls[1][1] == (self.ref_updatetitle, bodhi_msg.UNIT_TYPE, self.ref_tasks[0], self.ref_validarches[1])
41​ assert trigger_calls[2][1] == (self.ref_updatetitle, bodhi_msg.UNIT_TYPE, self.ref_tasks[1], self.ref_validarches[0])
42​ assert trigger_calls[3][1] == (self.ref_updatetitle, bodhi_msg.UNIT_TYPE, self.ref_tasks[1], self.ref_validarches[1])
42 43​
43 def test_consume_notwatched_topic(self, monkeypatch): 44​ def test_consume_notwatched_topic(self, monkeypatch):
44 ref_topic = 'org.fedoraproject.pro.bodhi.update.request.randomtopic' 45​ ref_topic = 'org.fedoraproject.pro.bodhi.update.request.randomtopic'
45 self._create_msg(self.ref_updatetitle, ref_topic) 46​ self._create_msg(self.ref_updatetitle, ref_topic)
46 47​
47 self.helper = bodhi_msg.BodhiJobTrigger(self.stub_hub) ​
48 ​
49 monkeypatch.setattr(self.helper, 'trigger', Dingus()) 48​ monkeypatch.setattr(self.helper, 'trigger', Dingus())
50 49​
51 self.helper.consume(self.ref_message) 50​ self.helper.consume(self.ref_message)
52 51​
53 trigger_calls = self.helper.trigger.trigger_job.calls() 52​ trigger_calls = self.helper.trigger.trigger_job.calls()
54 53​
55 assert trigger_calls == [] 54​ assert trigger_calls == []
View Options

testing/test_koji_trigger.py

1from dingus import Dingus 1​from dingus import Dingus
2from bunch import Bunch 2​from bunch import Bunch
3 3​
4from jobtriggers import koji_msg 4​from jobtriggers import koji_msg
5 5​
6 6​
7class TestKojiJobTrigger(): 7​class TestKojiJobTrigger():
8 def setup_method(self, method): 8​ def setup_method(self, method):
9 self.ref_name = 'stage' 9​ self.ref_name = 'stage'
10 self.ref_version = '4.1.1' 10​ self.ref_version = '4.1.1'
11 self.ref_release = '3.fc18' 11​ self.ref_release = '3.fc18'
12 self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release) 12​ self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release)
13 self.ref_tag = 'f18-updates-testing-pending' 13​ self.ref_tag = 'f18-updates-testing-pending'
14 14​
15 self._create_msg(self.ref_name, self.ref_release, self.ref_tag, self.ref_version) 15​ self._create_msg(self.ref_name, self.ref_release, self.ref_tag, self.ref_version)
16 16​
17 self.ref_config = { 17​ self.ref_tasks = ['koji_task1', 'koji_task2']
18 'taskotron.buildbot': { 18​ self.ref_validarches = ['i386', 'x86_64']
19 'username': "username", 19​
20 'password': "password", 20​ self.helper = koji_msg.KojiJobTrigger(Dingus())
21 'url': "http://localhost/buildbot/", ​
22 }, ​
23 } ​
24 self.stub_hub = Dingus(config__returns = self.ref_config) ​
25 21​
26 def _create_msg(self, ref_name, ref_release, ref_tag, ref_version): 22​ def _create_msg(self, ref_name, ref_release, ref_tag, ref_version):
27 self.ref_message = Bunch(body='{"i": 1,\ 23​ self.ref_message = Bunch(body='{"i": 1,\
28 "msg": {"name": "%s",\ 24​ "msg": {"build_id": 221146,\
25​ "name": "%s",\
29 "owner": "ralph",\ 26​ "owner": "ralph",\
30 "release": "%s",\ 27​ "release": "%s",\
31 "tag": "%s",\ 28​ "tag": "%s",\
32 "tag_id": 216,\ 29​ "tag_id": 216,\
33 "user": "bodhi",\ 30​ "user": "bodhi",\
34 "version": "%s"},\ 31​ "version": "%s"},\
35 "timestamp": 1359603469.21164,\ 32​ "timestamp": 1359603469.21164,\
36 "topic": "org.fedoraproject.prod.buildsys.tag",\ 33​ "topic": "org.fedoraproject.prod.buildsys.tag",\
37 "username": "apache"}' % 34​ "username": "apache"}' %
38 (ref_name, ref_release, ref_tag, ref_version)) 35​ (ref_name, ref_release, ref_tag, ref_version))
39 36​
40 def test_consume(self, monkeypatch): 37​ def test_consume(self, monkeypatch):
41 self.helper = koji_msg.KojiJobTrigger(self.stub_hub) ​
42 ​
43 monkeypatch.setattr(self.helper, 'trigger', Dingus()) 38​ monkeypatch.setattr(self.helper, 'trigger', Dingus())
39​ monkeypatch.setattr(koji_msg.settings, 'koji_tasks', self.ref_tasks)
40​ monkeypatch.setattr(koji_msg.settings, 'valid_arches', self.ref_validarches)
41​ monkeypatch.setattr(koji_msg, 'utils', Dingus(get_arches__returns=self.ref_validarches))
44 42​
45 self.helper.consume(self.ref_message) 43​ self.helper.consume(self.ref_message)
46 44​
47 trigger_calls = self.helper.trigger.trigger_job.calls() 45​ trigger_calls = self.helper.trigger.trigger_job.calls()
48 requested_nvr = trigger_calls[0][1][0] ​
49 requested_unit_type = trigger_calls[0][1][1] ​
50 46​
51 assert requested_nvr == self.ref_nvr 47​ assert len(trigger_calls) == 4
52 assert requested_unit_type == koji_msg.UNIT_TYPE 48​
49​ assert trigger_calls[0][1] == (self.ref_nvr, koji_msg.UNIT_TYPE, self.ref_tasks[0], self.ref_validarches[0])
50​ assert trigger_calls[1][1] == (self.ref_nvr, koji_msg.UNIT_TYPE, self.ref_tasks[0], self.ref_validarches[1])
51​ assert trigger_calls[2][1] == (self.ref_nvr, koji_msg.UNIT_TYPE, self.ref_tasks[1], self.ref_validarches[0])
52​ assert trigger_calls[3][1] == (self.ref_nvr, koji_msg.UNIT_TYPE, self.ref_tasks[1], self.ref_validarches[1])
53 53​
54 def test_consume_notwatched_tag(self, monkeypatch): 54​ def test_consume_notwatched_tag(self, monkeypatch):
55 ref_tag = 'f18-randomtag' 55​ ref_tag = 'f18-randomtag'
56 self._create_msg(self.ref_name, self.ref_release, ref_tag, self.ref_version) 56​ self._create_msg(self.ref_name, self.ref_release, ref_tag, self.ref_version)
57 57​
58 self.helper = koji_msg.KojiJobTrigger(self.stub_hub) ​
59 ​
60 monkeypatch.setattr(self.helper, 'trigger', Dingus()) 58​ monkeypatch.setattr(self.helper, 'trigger', Dingus())
61 ​
62 self.helper.consume(self.ref_message) 59​ self.helper.consume(self.ref_message)
63 60​
64 trigger_calls = self.helper.trigger.trigger_job.calls() 61​ trigger_calls = self.helper.trigger.trigger_job.calls()
65 62​
66 assert trigger_calls == [] 63​ assert trigger_calls == []
View Options

testing/test_utils.py

  • This file was added.
1​from dingus import Dingus
2​
3​from jobtriggers import utils
4​
5​
6​class TestBodhiJobTrigger():
7​ def setup_method(self, method):
8​ self.ref_validarches = ['i386', 'x86_64']
9​ self.ref_nvr = 'foo'
10​ self.ref_buildid = 1
11​ self.ref_rpmarches = [{'arch': 'i386'}, {'arch': 'x86_64'}]
12​ self.ref_build = {'id': self.ref_buildid}
13​
14​ def test_get_arches_by_buildid(self, monkeypatch):
15​ monkeypatch.setattr(utils, 'koji_client', Dingus(listRPMs__returns=self.ref_rpmarches))
16​
17​ res = utils.get_arches(buildid=self.ref_buildid)
18​
19​ assert res == set(self.ref_validarches)
20​
21​ def test_get_arches_by_nvr(self, monkeypatch):
22​ monkeypatch.setattr(utils, 'koji_client', Dingus(listRPMs__returns=self.ref_rpmarches, getBuild__returns=self.ref_build))
23​
24​ res = utils.get_arches(nvr=self.ref_nvr)
25​
26​ assert res == set(self.ref_validarches)