Differential D1

implemented dynamic directive loading code, moved koji directive to isolate and better support dynamic loading, changed runner code to behave the same as it was, just loading directives on demand. Fixes T45'
ClosedPublic
Actions

Authored by tflink on Jan 21 2014, 12:01 AM.

Details

Reviewers
jdulaney
Maniphest Tasks
T45: dynamically load modules for task directives
Commits
rLTRN01e2700c9ac7: changing printed error to thrown exception, fixing class name for KojiDirective
rLTRNd44783d0e326: changing koji import in koji_directive back to koji, adding on to T45
rLTRNc9c933a21e79: removing extranious 'as koji' from koji import
rLTRN5ff6634537b6: implemented dynamic directive loading code, moved koji directive to isolate and
Summary

implementing dynamic loading for directives, moving some stuff around to better support that and fixing runner to use new code

Test Plan

runner is still very limited, so test with existing rpmlint task yaml

Diff Detail

Lint
Lint Skipped
Unit
Unit Tests Skipped
jdulaney accepted this revision.Jan 21 2014, 2:09 AM

Looked it over fairly well; nothing jumps out.

ack

tflink updated this revision.Jan 21 2014, 2:25 AM
  • implemented dynamic directive loading code, moved koji directive to isolate and better support dynamic loading, changed runner code to behave the same as it was, just loading directives on demand. Fixes T45'
  • changing koji import in koji_directive back to koji, adding on to T45
tflink updated this revision.Jan 21 2014, 2:59 AM
  • removing extranious 'as koji' from koji import
  • changing printed error to thrown exception, fixing class name for KojiDirective
jdulaney added a comment.Jan 21 2014, 3:40 AM

Discussed a few things with tflink over IRC; ack

tflink closed this revision.Jan 21 2014, 3:45 AM

Closed by commit rLTRN5ff6634537b6 (authored by @tflink).

Revision Contents

Diff 4

View Options

libtaskotron/directives/__init__.py

  • This file was added.
1​class BaseDirective(object):
2​
3​ def __init__(self):
4​ pass
5​
6​ def process(self, command, input_data, env_data):
7​ raise NotImplementedError
8​
View Options

libtaskotron/directives/koji_directive.py

  • This file was moved from libtaskotron/koji_helper.py.
1import koji 1​import koji as real_koji
2 2​
3from libtaskotron import file_utils 3​from libtaskotron import file_utils
4​from libtaskotron.directives import BaseDirective
4 5​
5koji_url = 'http://koji.fedoraproject.org/kojihub' 6​koji_url = 'http://koji.fedoraproject.org/kojihub'
6pkg_url = 'http://kojipkgs.fedoraproject.org/packages' 7​pkg_url = 'http://kojipkgs.fedoraproject.org/packages'
7 8​
8class KojiHelper(): 9​class KojiHelper(BaseDirective):
9 def __init__(self, koji_session=None): 10​ def __init__(self, koji_session=None):
10 self.session = koji_session 11​ self.session = koji_session
11 12​
12 def get_koji_session(self): 13​ def get_koji_session(self):
13 if self.session == None: 14​ if self.session == None:
14 self.session = koji.ClientSession(koji_url) 15​ self.session = real_koji.ClientSession(koji_url)
15 16​
16 return self.session 17​ return self.session
17 18​
18 def nvr_to_urls(self, nvr, arches=None, debuginfo=False, src=True): 19​ def nvr_to_urls(self, nvr, arches=None, debuginfo=False, src=True):
19 '''Get list of URLs for RPMs corresponding to a build. 20​ '''Get list of URLs for RPMs corresponding to a build.
20 @param nvr build NVR 21​ @param nvr build NVR
21 @param arches restrict the arches of builds to provide URLs for. If 22​ @param arches restrict the arches of builds to provide URLs for. If
22 basearch i386 is in the list, i686 arch is automatically 23​ basearch i386 is in the list, i686 arch is automatically
Show All 14 Lines37 baseurl = '/'.join((pkg_url, info['package_name'],
37 info['version'], info['release'])) 38​ info['version'], info['release']))
38 39​
39 rpms = ksession.listRPMs(buildID=info['id'], arches=arches) 40​ rpms = ksession.listRPMs(buildID=info['id'], arches=arches)
40 if not debuginfo: 41​ if not debuginfo:
41 rpms = filter(lambda r: not r['name'].endswith('-debuginfo'), rpms) 42​ rpms = filter(lambda r: not r['name'].endswith('-debuginfo'), rpms)
42 if not src: 43​ if not src:
43 rpms = filter(lambda r: not r['arch'] == 'src', rpms) 44​ rpms = filter(lambda r: not r['arch'] == 'src', rpms)
44 45​
45 urls = ['%s/%s' % (baseurl, koji.pathinfo.rpm(r)) for r in rpms] 46​ urls = ['%s/%s' % (baseurl, real_koji.pathinfo.rpm(r)) for r in rpms]
46 if len(urls) == 0: 47​ if len(urls) == 0:
47 print "NO RPMS URLS FOUND!!!" 48​ print "NO RPMS URLS FOUND!!!"
48 return sorted(urls) 49​ return sorted(urls)
49 50​
50 def get_nvr_rpms(self, nvr, rpm_dir, arches=None, debuginfo=False, src=False): 51​ def get_nvr_rpms(self, nvr, rpm_dir, arches=None, debuginfo=False, src=False):
51 '''Retrieve the RPMs associated with a build NVR into the specified 52​ '''Retrieve the RPMs associated with a build NVR into the specified
52 directory. 53​ directory.
53 54​
▲ Show 20 Lines • Show All 50 Lines • Show Last 20 Lines
View Options

libtaskotron/koji_helper.py

  • This file was moved to libtaskotron/directives/koji_directive.py.
View Options

libtaskotron/runner.py

1import tempfile 1​import tempfile
2from libtaskotron import koji_helper, taskyaml 2​from libtaskotron import taskyaml
3import os.path 3​import os.path
4import argparse 4​import argparse
5import imp 5​import imp
6 6​
7class Runner: 7​class Runner:
8 def __init__(self, taskdata, inputdata, workdir=None): 8​ def __init__(self, taskdata, inputdata, workdir=None):
9 if workdir is None: 9​ if workdir is None:
10 self.workdir = tempfile.mkdtemp(dir='/var/tmp') 10​ self.workdir = tempfile.mkdtemp(dir='/var/tmp')
11 else: 11​ else:
12 self.workdir = workdir 12​ self.workdir = workdir
13 13​
14 self.taskdata = taskdata 14​ self.taskdata = taskdata
15 self.inputdata = inputdata 15​ self.inputdata = inputdata
16 self.working_data = {} 16​ self.working_data = {}
17​ self.directives = {}
17 18​
18 def run(self): 19​ def run(self):
19 # this isn't very flexible yet, just a proof of concept 20​ # this isn't very flexible yet, just a proof of concept
20 self.do_prep() 21​ self.do_prep()
21 22​
22 self.do_execution() 23​ self.do_execution()
23 24​
25​ def _load_directive(self, directive_name, directive_dir=None):
26​ # look in default path if nothing is specified
27​ if not directive_dir:
28​ directive_dir = os.path.join(os.path.dirname(__file__), 'directives')
29​
30​ real_name = "%s_directive" % directive_name
31​ directive_file = os.path.join(directive_dir, '%s.py' % real_name)
32​
33​ if not os.path.exists(directive_file):
34​ raise Exception("Directive %s not found in directory %s" % (directive_name, directive_dir))
35​
36​ loaded_directive = imp.load_source(real_name, directive_file)
37​ self.directives[directive_name] = loaded_directive
38​
39​
24 def do_prep(self): 40​ def do_prep(self):
25 commands = self.taskdata['preparation'] 41​ commands = self.taskdata['preparation']
26 42​
27 supported_prep = ['koji'] 43​ supported_prep = ['koji']
28 for command_type in commands.keys(): 44​ for command_type in commands.keys():
29 if command_type not in supported_prep: 45​ if command_type in supported_prep:
46​ self._load_directive(command_type)
47​ else:
30 raise Exception("Prep command %s not currently supported" % command_type) 48​ raise Exception("Prep command %s not currently supported" % command_type)
31 49​
32 # the only supported prep right now is koji, makes for simple code :) 50​ # the only supported prep right now is koji, makes for simple code :)
33 51​
34 helper = koji_helper.KojiHelper() 52​ helper = self.directives['koji'].KojiHelper()
35 53​
36 # instead, merge dict coming out of prep - let prep decide what data is named 54​ # instead, merge dict coming out of prep - let prep decide what data is named
37 prep_output = helper.process(commands['koji'], self.inputdata, {'tmpdir':self.workdir}) 55​ prep_output = helper.process(commands['koji'], self.inputdata, {'tmpdir':self.workdir})
38 self.working_data.update(prep_output) 56​ self.working_data.update(prep_output)
39 57​
40 def do_deps(self, deps): 58​ def do_deps(self, deps):
41 print "fake verifying deps: %s" % str(deps) 59​ print "fake verifying deps: %s" % str(deps)
42 60​
▲ Show 20 Lines • Show All 70 Lines • Show Last 20 Lines
View Options

testing/test_directives/__init__.py

  • This file was added.
This is an empty file.
View Options

testing/test_directives/test_directive.py

  • This file was added.
1​from libtaskotron.directives import BaseDirective
2​
3​class TestDirective(BaseDirective):
4​ def process(self, command, input_data, env_data):
5​ return "this is just a test"
View Options

testing/test_koji.py

1from libtaskotron import koji_helper 1​from libtaskotron.directives import koji_directive
2from dingus import Dingus 2​from dingus import Dingus
3 3​
4class TestKojiDownloads(): 4​class TestKojiDownloads():
5 def setup_method(self, method): 5​ def setup_method(self, method):
6 self.ref_nvr = 'foo-1.2-3.fc99' 6​ self.ref_nvr = 'foo-1.2-3.fc99'
7 self.ref_arch = 'noarch' 7​ self.ref_arch = 'noarch'
8 self.ref_name = 'foo' 8​ self.ref_name = 'foo'
9 self.ref_version = '1.2' 9​ self.ref_version = '1.2'
10 self.ref_release = '3.fc99' 10​ self.ref_release = '3.fc99'
11 self.ref_buildid = 123456 11​ self.ref_buildid = 123456
12 12​
13 self.ref_build = {'package_name':self.ref_name, 'version':self.ref_version, 13​ self.ref_build = {'package_name':self.ref_name, 'version':self.ref_version,
14 'release':self.ref_release, 'id': self.ref_buildid} 14​ 'release':self.ref_release, 'id': self.ref_buildid}
15 15​
16 self.ref_rpms = [{'name':self.ref_name, 'version':self.ref_version, 'release':self.ref_release, 'nvr':self.ref_nvr, 'arch':self.ref_arch}, 16​ self.ref_rpms = [{'name':self.ref_name, 'version':self.ref_version, 'release':self.ref_release, 'nvr':self.ref_nvr, 'arch':self.ref_arch},
17 {'name':self.ref_name, 'version':self.ref_version, 'release':self.ref_release, 'nvr':self.ref_nvr, 'arch':'src'}] 17​ {'name':self.ref_name, 'version':self.ref_version, 'release':self.ref_release, 'nvr':self.ref_nvr, 'arch':'src'}]
18 18​
19 self.ref_url = 'http://downloads.example.com/builds/%s.noarch.rpm' % self.ref_nvr 19​ self.ref_url = 'http://downloads.example.com/builds/%s.noarch.rpm' % self.ref_nvr
20 self.helper = koji_helper.KojiHelper() 20​ self.helper = koji_directive.KojiHelper()
21 21​
22 def test_get_noarch_rpmurls_from_nvr(self): 22​ def test_get_noarch_rpmurls_from_nvr(self):
23 stub_koji = Dingus(getBuild__returns = self.ref_build, listRPMs__returns = self.ref_rpms) 23​ stub_koji = Dingus(getBuild__returns = self.ref_build, listRPMs__returns = self.ref_rpms)
24 24​
25 test_koji = koji_helper.KojiHelper(stub_koji) 25​ test_koji = koji_directive.KojiHelper(stub_koji)
26 26​
27 test_urls = test_koji.nvr_to_urls(self.ref_nvr) 27​ test_urls = test_koji.nvr_to_urls(self.ref_nvr)
28 28​
29 koji_baseurl = koji_helper.pkg_url 29​ koji_baseurl = koji_directive.pkg_url
30 ref_urls = ['%s/%s/%s/%s/%s/%s.%s.rpm' % (koji_baseurl, self.ref_name, self.ref_version, self.ref_release, self.ref_arch, self.ref_nvr, self.ref_arch), 30​ ref_urls = ['%s/%s/%s/%s/%s/%s.%s.rpm' % (koji_baseurl, self.ref_name, self.ref_version, self.ref_release, self.ref_arch, self.ref_nvr, self.ref_arch),
31 '%s/%s/%s/%s/src/%s.src.rpm' % (koji_baseurl, self.ref_name, self.ref_version, self.ref_release, self.ref_nvr), 31​ '%s/%s/%s/%s/src/%s.src.rpm' % (koji_baseurl, self.ref_name, self.ref_version, self.ref_release, self.ref_nvr),
32 ] 32​ ]
33 33​
34 assert test_urls == ref_urls 34​ assert test_urls == ref_urls
35 35​
36 def should_i386_rpmurls_query_i686(self): 36​ def should_i386_rpmurls_query_i686(self):
37 self.ref_arch = 'i386' 37​ self.ref_arch = 'i386'
38 38​
39 stub_koji = Dingus(getBuild__returns = self.ref_build) 39​ stub_koji = Dingus(getBuild__returns = self.ref_build)
40 40​
41 test_koji = koji_helper.KojiHelper(stub_koji) 41​ test_koji = koji_directive.KojiHelper(stub_koji)
42 42​
43 test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch]) 43​ test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch])
44 44​
45 listrpm_calls = stub_koji.calls('listRPMs') 45​ listrpm_calls = stub_koji.calls('listRPMs')
46 requested_arches = listrpm_calls[0][2]['arches'] 46​ requested_arches = listrpm_calls[0][2]['arches']
47 47​
48 assert 'i686' in requested_arches 48​ assert 'i686' in requested_arches
49 assert 'i386' in requested_arches 49​ assert 'i386' in requested_arches
50 50​
51 def test_parse_download_command(self, monkeypatch): 51​ def test_parse_download_command(self, monkeypatch):
52 ref_command = 'download envr' 52​ ref_command = 'download envr'
53 ref_input = {'arch':self.ref_arch, 'envr':self.ref_nvr} 53​ ref_input = {'arch':self.ref_arch, 'envr':self.ref_nvr}
54 ref_envdata = {'tmpdir':'/var/tmp/foo'} 54​ ref_envdata = {'tmpdir':'/var/tmp/foo'}
55 55​
56 stub_koji = Dingus(getBuild__returns = self.ref_build, listRPMs__returns = self.ref_rpms) 56​ stub_koji = Dingus(getBuild__returns = self.ref_build, listRPMs__returns = self.ref_rpms)
57 ref_rpms = ['/var/tmp/fake/%s.%s.rpm' % (self.ref_nvr, self.ref_arch)] 57​ ref_rpms = ['/var/tmp/fake/%s.%s.rpm' % (self.ref_nvr, self.ref_arch)]
58 58​
59 stub_get_rpms = Dingus(get_nvr_rpms__returns=ref_rpms) 59​ stub_get_rpms = Dingus(get_nvr_rpms__returns=ref_rpms)
60 test_helper = koji_helper.KojiHelper(stub_koji) 60​ test_helper = koji_directive.KojiHelper(stub_koji)
61 61​
62 monkeypatch.setattr(test_helper, 'get_nvr_rpms', stub_get_rpms) 62​ monkeypatch.setattr(test_helper, 'get_nvr_rpms', stub_get_rpms)
63 test_helper.process(ref_command, ref_input, ref_envdata) 63​ test_helper.process(ref_command, ref_input, ref_envdata)
64 64​
65 getrpm_calls = stub_get_rpms.calls() 65​ getrpm_calls = stub_get_rpms.calls()
66 print getrpm_calls 66​ print getrpm_calls
67 requested_nvr = getrpm_calls[0][1][0] 67​ requested_nvr = getrpm_calls[0][1][0]
68 68​
69 assert len(getrpm_calls) == 1 69​ assert len(getrpm_calls) == 1
70 assert requested_nvr == self.ref_nvr 70​ assert requested_nvr == self.ref_nvr
View Options

testing/test_yamlrunner.py

1import pytest 1​import pytest
2​import os
3​import sys
4​
2from libtaskotron import runner 5​from libtaskotron import runner
3 6​
4 7​
5 8​
6class TestRunnerInputVerify(): 9​class TestRunnerInputVerify():
7 def test_yamlrunner_valid_input(self): 10​ def test_yamlrunner_valid_input(self):
8 ref_cmd = '-e foo-1.2-3.fc99 -a x86_64 footask.yml' 11​ ref_cmd = '-e foo-1.2-3.fc99 -a x86_64 footask.yml'
9 ref_inputspec = {'args':'envr,arch'} 12​ ref_inputspec = {'args':'envr,arch'}
Show All 32 Lines44class TestRunnerPrep():
42 def test_prep_invalid_commands(self, tmpdir): 45​ def test_prep_invalid_commands(self, tmpdir):
43 ref_taskdata = {'preparation': {'turboencabulator':'reduce sinusoidal repleneration'}} 46​ ref_taskdata = {'preparation': {'turboencabulator':'reduce sinusoidal repleneration'}}
44 ref_inputdata = {} 47​ ref_inputdata = {}
45 48​
46 test_runner = runner.Runner(ref_taskdata, ref_inputdata, tmpdir) 49​ test_runner = runner.Runner(ref_taskdata, ref_inputdata, tmpdir)
47 50​
48 with pytest.raises(Exception): 51​ with pytest.raises(Exception):
49 test_runner.do_prep() 52​ test_runner.do_prep()
53​
54​class TestRunnerDirectiveLoader():
55​
56​ def setup_method(self, method):
57​ self.ref_dir = os.path.abspath('testing/test_directives/')
58​ self.ref_taskdata = {}
59​ self.ref_inputdata = {}
60​ self.ref_directivename = 'test'
61​
62​ def test_runner_load_directive(self, tmpdir):
63​ test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata, tmpdir)
64​
65​ test_runner._load_directive(self.ref_directivename, self.ref_dir)
66​
67​ assert '%s_directive' % self.ref_directivename in sys.modules.keys()
68​
69​ def test_runner_load_directive_indict(self, tmpdir):
70​ test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata, tmpdir)
71​
72​ test_runner._load_directive(self.ref_directivename, self.ref_dir)
73​
74​ assert self.ref_directivename in test_runner.directives.keys()
75​
76​ def test_runner_load_directive_notexist(self, tmpdir):
77​ self.ref_directivename = 'noexist'
78​
79​ test_runner = runner.Runner(self.ref_taskdata, self.ref_inputdata, tmpdir)
80​
81​ with pytest.raises(Exception):
82​ test_runner._load_directive(self.ref_directivename, self.ref_dir)
83​