Differential D240

Remove querying koji for rpm's arches.
ClosedPublic
Actions

Authored by mkrizek on Sep 30 2014, 2:24 PM.

Details

Reviewers
ralph
• ralph2
tflink
Maniphest Tasks
T341: taskotron-trigger is not scheduling jobs for koji build messages in dev, stg or prod
Commits
rTRGR83f470748b95: Remove querying koji for rpm's arches.
Summary

Since we ignore --arch option in all our tests (rpmlit, depcheck and
ugradepath) we can get rid of get_arches function that caused T341.

Fixes T341

Test Plan

Tests pass. Ran locally, works for me.

Diff Detail

Repository
rTRGR taskotron-trigger
Lint
Lint Skipped
Unit
Unit Tests Skipped
mkrizek retitled this revision from to Remove querying koji for rpm's arches..Sep 30 2014, 2:24 PM
mkrizek updated this object.
mkrizek edited the test plan for this revision. (Show Details)
mkrizek added reviewers: tflink, ralph, • ralph2.
mkrizek added a task: T341: taskotron-trigger is not scheduling jobs for koji build messages in dev, stg or prod.
tflink accepted this revision.Sep 30 2014, 4:36 PM

LGTM

This revision is now accepted and ready to land.Sep 30 2014, 4:36 PM
mkrizek closed this revision.Sep 30 2014, 5:14 PM
mkrizek updated this revision to Diff 667.

Closed by commit rTRGR83f470748b95 (authored by @mkrizek).

Revision Contents

Diff 667

View Options

conf/trigger.cfg.example

1[buildbot] 1​[buildbot]
2url = http://localhost:8080/buildbot/change_hook 2​url = http://localhost:8080/buildbot/change_hook
3 3​
4[trigger] 4​[trigger]
5koji_build_completed_tasks = rpmlint 5​koji_build_completed_tasks = rpmlint
6koji_tag_changed_tasks = upgradepath,depcheck 6​koji_tag_changed_tasks = upgradepath,depcheck
7valid_arches = i386,i686,x86_64,noarch 7​valid_arches = x86_64
8koji_url = http://koji.fedoraproject.org/kojihub 8​koji_url = http://koji.fedoraproject.org/kojihub
9type = StreamTrigger 9​type = StreamTrigger
10blacklist_releases = el5,el6,el7 10​blacklist_releases = el5,el6,el7
11job_logging = False 11​job_logging = False
12joblog_file = /var/log/taskotron-trigger/jobs.csv 12​joblog_file = /var/log/taskotron-trigger/jobs.csv
13fuse_delay = 1000 ; seconds 13​fuse_delay = 1000 ; seconds
View Options

jobtriggers/config.py

1import ConfigParser 1​import ConfigParser
2import io 2​import io
3 3​
4defaults = """ 4​defaults = """
5[buildbot] 5​[buildbot]
6url = http://localhost:8080/buildbot/change_hook 6​url = http://localhost:8080/buildbot/change_hook
7 7​
8[trigger] 8​[trigger]
9koji_build_completed_tasks = rpmlint 9​koji_build_completed_tasks = rpmlint
10koji_tag_changed_tasks = upgradepath,depcheck 10​koji_tag_changed_tasks = upgradepath,depcheck
11valid_arches = i386,i686,x86_64,noarch 11​valid_arches = x86_64
12koji_url = http://koji.fedoraproject.org/kojihub 12​koji_url = http://koji.fedoraproject.org/kojihub
13type = StreamTrigger 13​type = StreamTrigger
14blacklist_releases = el5,el6,el7 14​blacklist_releases = el5,el6,el7
15job_logging = False 15​job_logging = False
16joblog_file = /var/log/taskotron-trigger/jobs.cvs 16​joblog_file = /var/log/taskotron-trigger/jobs.cvs
17fuse_delay = 1000 ; seconds 17​fuse_delay = 1000 ; seconds
18""" 18​"""
19 19​
Show All 16 Lines
View Options

jobtriggers/koji_build_msg.py

Show All 16 Lines10class KojiBuildCompletedJobTrigger(JobTrigger):
17 17​
18 def consume(self, message): 18​ def consume(self, message):
19 usable_message = fedmsg.encoding.loads(message.body) 19​ usable_message = fedmsg.encoding.loads(message.body)
20 20​
21 release = usable_message['msg']['release'] 21​ release = usable_message['msg']['release']
22 state = usable_message['msg']['new'] 22​ state = usable_message['msg']['new']
23 instance = usable_message['msg']['instance'] 23​ instance = usable_message['msg']['instance']
24 24​
25 if not instance == 'primary': ​
26 return ​
27 ​
28 # filter builds ​
29 if state != koji.BUILD_STATES['COMPLETE'] or utils.blacklist_releases(release): ​
30 return ​
31 ​
32 name = usable_message['msg']['name'] 25​ name = usable_message['msg']['name']
33 version = usable_message['msg']['version'] 26​ version = usable_message['msg']['version']
34 buildid = usable_message['msg']['build_id'] 27​ buildid = usable_message['msg']['build_id']
35 28​
36 envr = "%s-%s-%s" % (name, version, release) 29​ envr = "%s-%s-%s" % (name, version, release)
37 30​
38 arches = utils.get_arches(buildid=buildid) 31​ if not instance == 'primary':
32​ self.log.debug('rejecting message not from primary koji instance (%s from %s)' % (envr, instance))
33​ return
34​
35​ if state != koji.BUILD_STATES['COMPLETE']:
36​ self.log.debug('rejecting message for not being a build complete message (%s, status %s)' % (envr, state))
37​ return
38​
39​ if utils.blacklist_releases(release):
40​ self.log.debug('rejecting message from blacklisted release (%s)' % envr)
41​ return
42​
43​ self.log.debug('should be scheduling job for %s' % envr)
39 44​
40 self.trigger_tasks(envr, ITEM_TYPE, config.koji_build_completed_tasks, arches) 45​ self.trigger_tasks(envr, ITEM_TYPE, config.koji_build_completed_tasks, config.valid_arches)
View Options

jobtriggers/koji_tag_msg.py

Show All 27 Lines14class KojiTagChangedJobTrigger(JobTrigger):
28 def delayed_consume(self): 28​ def delayed_consume(self):
29 if self.queued_tags: 29​ if self.queued_tags:
30 try: 30​ try:
31 for tag in self.queued_tags: 31​ for tag in self.queued_tags:
32 # XXX ugly: do not schedule upgradepath for updates-testing-pending 32​ # XXX ugly: do not schedule upgradepath for updates-testing-pending
33 tasks = deepcopy(config.koji_tag_changed_tasks) 33​ tasks = deepcopy(config.koji_tag_changed_tasks)
34 if tag.endswith('testing-pending'): 34​ if tag.endswith('testing-pending'):
35 tasks.remove('upgradepath') 35​ tasks.remove('upgradepath')
36 # it is sufficient to run depcheck and upgradepath only on one arch 36​ self.trigger_tasks(tag, ITEM_TYPE, tasks, config.valid_arches)
37 # XXX hardcoding X86_64 ​
38 self.trigger_tasks(tag, ITEM_TYPE, tasks, ['x86_64']) ​
39 finally: 37​ finally:
40 self.queued_tags = set() 38​ self.queued_tags = set()
41 else: 39​ else:
42 self.log.debug("Woke up, but there were no messages.") 40​ self.log.debug("Woke up, but there were no messages.")
43 41​
44 def consume(self, message): 42​ def consume(self, message):
45 usable_message = fedmsg.encoding.loads(message.body) 43​ usable_message = fedmsg.encoding.loads(message.body)
46 44​
47 koji_tag = usable_message['msg']['tag'] 45​ koji_tag = usable_message['msg']['tag']
48 release = usable_message['msg']['release'] 46​ release = usable_message['msg']['release']
49 instance = usable_message['msg']['instance'] 47​ instance = usable_message['msg']['instance']
50 48​
51 if not instance == 'primary': 49​ if not instance == 'primary':
50​ self.log.debug('rejecting message not from primary koji instance (%s from %s)' % (koji_tag, instance))
52 return 51​ return
53 52​
54 # filter tags 53​ if not koji_tag.endswith('pending'):
55 if not koji_tag.endswith('pending') or utils.blacklist_releases(release): 54​ self.log.debug('rejecting message for not being a pending tag (%s)' % koji_tag)
55​ return
56​
57​ if utils.blacklist_releases(release):
58​ self.log.debug('rejecting message from blacklisted release (%s)' % koji_tag)
56 return 59​ return
57 60​
58 if not len(self.queued_tags): 61​ if not len(self.queued_tags):
59 self.log.info('setting fuse (%d seconds) for koji_tag %s', config.fuse_delay, koji_tag) 62​ self.log.info('setting fuse (%d seconds) for koji_tag %s', config.fuse_delay, koji_tag)
60 elif koji_tag not in self.queued_tags: 63​ elif koji_tag not in self.queued_tags:
61 self.log.info('adding koji_tag %s to the fuse queue', koji_tag) 64​ self.log.info('adding koji_tag %s to the fuse queue', koji_tag)
62 65​
63 self.queued_tags.add(koji_tag) 66​ self.queued_tags.add(koji_tag)
View Options

jobtriggers/utils.py

1import csv 1​import csv
2import datetime 2​import datetime
3import koji ​
4 3​
5from . import config 4​from . import config
6 5​
7koji_client = koji.ClientSession(config.koji_url) ​
8 ​
9 ​
10def get_arches(buildid=None, nvr=None): ​
11 if buildid is None: ​
12 info = koji_client.getBuild(nvr) ​
13 buildid = info['id'] ​
14 ​
15 arches = set([rpm['arch'] for rpm in koji_client.listRPMs(buildid) if rpm['arch'] in config.valid_arches]) ​
16 ​
17 if 'i686' in arches: ​
18 arches.remove('i686') ​
19 arches.add('i386') ​
20 ​
21 return arches ​
22 ​
23 ​
24def log_job(logname, item, item_type, task, arch): 6​def log_job(logname, item, item_type, task, arch):
25 now = datetime.datetime.now() 7​ now = datetime.datetime.now()
26 8​
27 with open(logname, 'a') as csvfile: 9​ with open(logname, 'a') as csvfile:
28 writer = csv.writer(csvfile, delimiter=';') 10​ writer = csv.writer(csvfile, delimiter=';')
29 writer.writerow([now, item, item_type, task, arch]) 11​ writer.writerow([now, item, item_type, task, arch])
30 12​
31def blacklist_releases(release): 13​def blacklist_releases(release):
32 return any(b in release for b in config.blacklist_releases) 14​ return any(b in release for b in config.blacklist_releases)
View Options

testing/test_koji_build_trigger.py

Show All 22 Lines12 def prepare(self, monkeypatch):
23 self.ref_validarches = ['i386', 'x86_64'] 23​ self.ref_validarches = ['i386', 'x86_64']
24 self.ref_blacklist_releases= ['el5', 'el6'] 24​ self.ref_blacklist_releases= ['el5', 'el6']
25 25​
26 stub_hub = Bunch(config=Bunch(get=0)) 26​ stub_hub = Bunch(config=Bunch(get=0))
27 27​
28 self.helper = koji_build_msg.KojiBuildCompletedJobTrigger(stub_hub) 28​ self.helper = koji_build_msg.KojiBuildCompletedJobTrigger(stub_hub)
29 self.helper.trigger = Dingus() 29​ self.helper.trigger = Dingus()
30 30​
31 monkeypatch.setattr(koji_build_msg.utils, 'get_arches', lambda *a, **k: self.ref_validarches) ​
32 ​
33 koji_build_msg.config.koji_build_completed_tasks = self.ref_tasks 31​ koji_build_msg.config.koji_build_completed_tasks = self.ref_tasks
34 koji_build_msg.config.valid_arches = self.ref_validarches 32​ koji_build_msg.config.valid_arches = self.ref_validarches
35 koji_build_msg.config.blacklist_releases = self.ref_blacklist_releases 33​ koji_build_msg.config.blacklist_releases = self.ref_blacklist_releases
36 koji_build_msg.config.job_logging = False 34​ koji_build_msg.config.job_logging = False
37 35​
38 def _create_msg(self, ref_instance, ref_new, ref_name, ref_release, ref_version): 36​ def _create_msg(self, ref_instance, ref_new, ref_name, ref_release, ref_version):
39 self.ref_message = Bunch(body='{"i": 1,\ 37​ self.ref_message = Bunch(body='{"i": 1,\
40 "msg": {"build_id": 221146,\ 38​ "msg": {"build_id": 221146,\
▲ Show 20 Lines • Show All 51 Lines • Show Last 20 Lines
View Options

testing/test_koji_tag_trigger.py

Show All 11 Lines11 def prepare(self, monkeypatch):
12 self.ref_instance = 'primary' 12​ self.ref_instance = 'primary'
13 self.ref_name = 'stage' 13​ self.ref_name = 'stage'
14 self.ref_version = '4.1.1' 14​ self.ref_version = '4.1.1'
15 self.ref_release = '3.fc18' 15​ self.ref_release = '3.fc18'
16 self.ref_tag = 'f18-updates-pending' 16​ self.ref_tag = 'f18-updates-pending'
17 self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release) 17​ self.ref_nvr = '%s-%s-%s' % (self.ref_name, self.ref_version, self.ref_release)
18 18​
19 self.ref_tasks = ['depcheck', 'upgradepath'] 19​ self.ref_tasks = ['depcheck', 'upgradepath']
20 self.ref_arch = 'x86_64' 20​ self.ref_validarches = ['i386', 'x86_64']
21 self.ref_blacklist_releases= ['el5', 'el6'] 21​ self.ref_blacklist_releases= ['el5', 'el6']
22 22​
23 self._create_msg(self.ref_instance, self.ref_name, self.ref_release, self.ref_tag, self.ref_version) 23​ self._create_msg(self.ref_instance, self.ref_name, self.ref_release, self.ref_tag, self.ref_version)
24 24​
25 stub_hub = Bunch(config=Bunch(get=0)) 25​ stub_hub = Bunch(config=Bunch(get=0))
26 self.helper = koji_tag_msg.KojiTagChangedJobTrigger(stub_hub) 26​ self.helper = koji_tag_msg.KojiTagChangedJobTrigger(stub_hub)
27 27​
28 self.helper.trigger = Dingus() 28​ self.helper.trigger = Dingus()
29 29​
30 koji_tag_msg.config.koji_tag_changed_tasks = self.ref_tasks 30​ koji_tag_msg.config.koji_tag_changed_tasks = self.ref_tasks
31​ koji_tag_msg.config.valid_arches = self.ref_validarches
31 koji_tag_msg.config.job_logging = False 32​ koji_tag_msg.config.job_logging = False
32 33​
33 def _create_msg(self, ref_instance, ref_name, ref_release, ref_tag, ref_version): 34​ def _create_msg(self, ref_instance, ref_name, ref_release, ref_tag, ref_version):
34 self.ref_message = Bunch(body='{"i": 1,\ 35​ self.ref_message = Bunch(body='{"i": 1,\
35 "msg": {"build_id": 221146,\ 36​ "msg": {"build_id": 221146,\
36 "instance": "%s",\ 37​ "instance": "%s",\
37 "name": "%s",\ 38​ "name": "%s",\
38 "owner": "ralph",\ 39​ "owner": "ralph",\
Show All 14 Lines
53 54​
54 def test_delayed_consume(self): 55​ def test_delayed_consume(self):
55 self.helper.queued_tags = [self.ref_tag] 56​ self.helper.queued_tags = [self.ref_tag]
56 57​
57 self.helper.delayed_consume() 58​ self.helper.delayed_consume()
58 59​
59 trigger_calls = self.helper.trigger.trigger_job.calls() 60​ trigger_calls = self.helper.trigger.trigger_job.calls()
60 61​
61 assert len(trigger_calls) == 2 62​ assert len(trigger_calls) == 4
62 63​
63 assert trigger_calls[0][1] == (self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_arch) 64​ assert trigger_calls[0][1] == (self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0])
64 assert trigger_calls[1][1] == (self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_arch) 65​ assert trigger_calls[1][1] == (self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1])
66​ assert trigger_calls[2][1] == (self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[0])
67​ assert trigger_calls[3][1] == (self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[1])
65 68​
66 assert self.helper.queued_tags == set() 69​ assert self.helper.queued_tags == set()
67 70​
68 def test_consume_not_primary_instance(self): 71​ def test_consume_not_primary_instance(self):
69 ref_instance = 'ppc' 72​ ref_instance = 'ppc'
70 self._create_msg(ref_instance, self.ref_name, self.ref_release, self.ref_tag, self.ref_version) 73​ self._create_msg(ref_instance, self.ref_name, self.ref_release, self.ref_tag, self.ref_version)
71 74​
72 self.helper.consume(self.ref_message) 75​ self.helper.consume(self.ref_message)
Show All 12 Lines
85 def test_delayed_consume_upgradepath(self): 88​ def test_delayed_consume_upgradepath(self):
86 ref_tag = 'f18-updates-testing-pending' 89​ ref_tag = 'f18-updates-testing-pending'
87 self.helper.queued_tags = [ref_tag] 90​ self.helper.queued_tags = [ref_tag]
88 91​
89 self.helper.delayed_consume() 92​ self.helper.delayed_consume()
90 93​
91 trigger_calls = self.helper.trigger.trigger_job.calls() 94​ trigger_calls = self.helper.trigger.trigger_job.calls()
92 95​
93 assert len(trigger_calls) == 1 96​ assert len(trigger_calls) == 2
94 97​
95 assert trigger_calls[0][1] == (ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_arch) 98​ assert trigger_calls[0][1] == (ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0])
99​ assert trigger_calls[1][1] == (ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1])
96 100​
97 assert self.helper.queued_tags == set() 101​ assert self.helper.queued_tags == set()
98 102​
99 def test_consume_blacklisted_release(self): 103​ def test_consume_blacklisted_release(self):
100 ref_release = '3.el5.1' 104​ ref_release = '3.el5.1'
101 self._create_msg(self.ref_instance, self.ref_name, ref_release, self.ref_tag, self.ref_version) 105​ self._create_msg(self.ref_instance, self.ref_name, ref_release, self.ref_tag, self.ref_version)
102 106​
103 self.helper.consume(self.ref_message) 107​ self.helper.consume(self.ref_message)
104 108​
105 assert self.helper.queued_tags == set([]) 109​ assert self.helper.queued_tags == set([])
View Options

testing/test_utils.py

1from dingus import Dingus 1​from dingus import Dingus
2 2​
3from jobtriggers import utils, config 3​from jobtriggers import utils, config
4 4​
5 5​
6class TestBodhiJobTrigger(): 6​class TestBodhiJobTrigger():
7 def setup_method(self, method): 7​ def setup_method(self, method):
8 self.ref_validarches = ['i386', 'x86_64'] ​
9 self.ref_nvr = 'foo' ​
10 self.ref_buildid = 1 ​
11 self.ref_rpmarches = [{'arch': 'i386'}, {'arch': 'x86_64'}] ​
12 self.ref_build = {'id': self.ref_buildid} ​
13 self.ref_release = '3.el6' 8​ self.ref_release = '3.el6'
14 9​
15 def test_get_arches_by_buildid(self, monkeypatch): ​
16 monkeypatch.setattr(utils, 'koji_client', Dingus(listRPMs__returns=self.ref_rpmarches)) ​
17 ​
18 res = utils.get_arches(buildid=self.ref_buildid) ​
19 ​
20 assert res == set(self.ref_validarches) ​
21 ​
22 def test_get_arches_by_nvr(self, monkeypatch): ​
23 monkeypatch.setattr(utils, 'koji_client', Dingus(listRPMs__returns=self.ref_rpmarches, getBuild__returns=self.ref_build)) ​
24 ​
25 res = utils.get_arches(nvr=self.ref_nvr) ​
26 ​
27 assert res == set(self.ref_validarches) ​
28 ​
29 def test_blacklist_releases_true(self): 10​ def test_blacklist_releases_true(self):
30 config.blacklist_releases = ['el6', 'el7'] 11​ config.blacklist_releases = ['el6', 'el7']
31 res = utils.blacklist_releases(self.ref_release) 12​ res = utils.blacklist_releases(self.ref_release)
32 13​
33 assert res 14​ assert res
34 15​
35 def test_blacklist_releases_false(self): 16​ def test_blacklist_releases_false(self):
36 config.blacklist_releases = ['fc20'] 17​ config.blacklist_releases = ['fc20']
37 res = utils.blacklist_releases(self.ref_release) 18​ res = utils.blacklist_releases(self.ref_release)
38 19​
39 assert not res 20​ assert not res