Differential D1170

Update logic to parse compose metadata instead of having a hardcoded value.
ClosedPublic
Actions

Authored by roshi on Mar 14 2017, 8:28 PM.

Details

Reviewers
tflink
mkrizek
Commits
rTRGR64231a152451: Fix hardcoded paths in cloud_compose_completed_msg.py. Fixes T928
Summary
Fix issue: T928

Fixed more lint issues.

Test Plan
Run Py.Test

Diff Detail

Repository
rTRGR taskotron-trigger
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.
roshi created this revision.Mar 14 2017, 8:28 PM
tflink added a comment.Mar 14 2017, 9:02 PM

at first glance, it looks good to me but I'll leave the review decision up to @mkrizek

tflink accepted this revision.Mar 16 2017, 3:27 AM
This revision is now accepted and ready to land.Mar 16 2017, 3:27 AM
Closed by commit rTRGR64231a152451: Fix hardcoded paths in cloud_compose_completed_msg.py. Fixes T928 (authored by roshi). · Explain WhyMar 16 2017, 4:50 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

Diff 2976

View Options

jobtriggers/cloud_compose_complete_msg.py

1import fedmsg 1​import fedmsg
2import fedmsg.encoding 2​import fedmsg.encoding
3import fedmsg.consumers 3​import fedmsg.consumers
4 4​
5​import json
6​import requests
7​
5from . import config 8​from . import config
6from .jobtrigger import JobTrigger 9​from .jobtrigger import JobTrigger
7from . import exceptions as exc 10​from . import exceptions as exc
8 11​
9 12​
10def _pull_version_release(compose_id): 13​def _pull_version_release(compose_id):
11 """Splits out the relevant bits of the compose ID. Input requires the form 14​ """Splits out the relevant bits of the compose ID. Input requires the form
12 "Fedora-TYPE-VERSION-Date.Serial" (ie, Fedora-Cloud-24-20170224.0) that fedmsg 15​ "Fedora-TYPE-VERSION-Date.Serial" (ie, Fedora-Cloud-24-20170224.0) that fedmsg
13 spits out in compose_id.""" 16​ spits out in compose_id."""
14 17​
15 parts = compose_id.split('-') 18​ parts = compose_id.split('-')
16 if len(parts) < 4: 19​ if len(parts) < 4:
17 raise exc.TriggerMsgError("Provided compose_id did not contain" 20​ raise exc.TriggerMsgError("Provided compose_id did not contain"
18 "all parts.") 21​ "all parts.")
19 else: 22​ else:
20 name = '-'.join(parts[:2]) 23​ name = '-'.join(parts[:2])
21 version = parts[1] 24​ version = parts[1]
22 release = parts[2] 25​ release = parts[2]
23 26​
24 return name, version, release 27​ return name, version, release
25 28​
26 29​
30​def _fetch_metadata(compose_url):
31​ """This method simply isolates the one network call for the metadata out
32​ to make it easier to test the other logic. We'll presume that the 'requests'
33​ library does what it's supposed to.
34​
35​ Input is the 'location' from fedmsg and it returns a json object from the
36​ metadata file."""
37​
38​ metadata_path = "/metadata/images.json"
39​ metadata = requests.get(compose_url + metadata_path)
40​ print metadata.status_code
41​ if metadata.status_code is not 200:
42​ raise exc.TriggerMsgError("Couldn't pull metadata from Koji.")
43​
44​ metadata = json.loads(metadata.text)
45​
46​ return metadata
47​
48​
49​def _find_image(compose_url):
50​ """Parse out the images from the compose metadata to determine which test to
51​ to run. This does not include Rawhide composes, as those include more than
52​ one qcow2 image to test."""
53​
54​ metadata = _fetch_metadata(compose_url)
55​
56​ x86_images = metadata['payload']['images']['CloudImages']['x86_64']
57​
58​ for image in x86_images:
59​ if image['format'] == 'qcow2':
60​ # Have to add the / here to make a valid URL because the compose in
61​ # the fedmsg doesn't have a trailing slash and the path in the
62​ # metadata doesn't have a leading slash.
63​ return compose_url + "/" + image['path']
64​
65​
27def _process(msg): 66​def _process(msg):
28 """This method preps the incoming fedmsg data so the trigger can set 67​ """This method preps the incoming fedmsg data so the trigger can set
29 off the task. Expects fedmsg.body data as input and returns a dict 68​ off the task. Expects fedmsg.body data as input and returns a dict
30 for JobTrigger.do_trigger.""" 69​ for JobTrigger.do_trigger."""
31 70​
32 MESSAGE_TYPE = "CloudCompose" 71​ MESSAGE_TYPE = "CloudCompose"
33 72​
34 if msg['msg']['status'] != "FINISHED": 73​ if msg['msg']['status'] != "FINISHED":
35 raise exc.TriggerMsgError("Rejecting message because status not " 74​ raise exc.TriggerMsgError("Rejecting message because status not "
36 "'FINISHED'.") 75​ "'FINISHED'.")
37 76​
38 # Split this up to make the lines shorter 77​ item = _find_image(msg['msg']['location'])
39 # FIXME: We shouldn't hard code these paths here ​
40 item = ("{0}/CloudImages/x86_64/" ​
41 "images/{1}.x86_64.qcow2").format(msg['msg']['location'], ​
42 msg['msg']['compose_id']) ​
43 78​
44 name, version, release = _pull_version_release(msg['msg']['compose_id']) 79​ name, version, release = _pull_version_release(msg['msg']['compose_id'])
45 80​
46 if version == "Atomic": 81​ if version == "Atomic":
47 MESSAGE_TYPE = "AtomicCompose" 82​ MESSAGE_TYPE = "AtomicCompose"
48 83​
49 data = { 84​ data = {
50 "_msg": msg, 85​ "_msg": msg,
Show All 34 Lines
View Options

testing/test_cloud_compose_trigger.py

1import pytest 1​import pytest
2import json 2​import json
3 3​
4# Only going to import the interesting bits 4​# Only going to import the interesting bits
5from jobtriggers import cloud_compose_complete_msg 5​from jobtriggers import cloud_compose_complete_msg
6from jobtriggers import exceptions as exc 6​from jobtriggers import exceptions as exc
7 7​
8# Set up some dummy data to work with that's in various states of correct 8​# Set up some dummy data to work with that's in various states of correct
9# These comprise just the interesting bits from the fedmsg that we want to 9​# These comprise just the interesting bits from the fedmsg that we want to
10# consume with our trigger. 10​# consume with our trigger.
11 11​
12location = "http://kojipkgs.fedoraproject.org/compose/Fedora-Cloud-24-20170224.0/compose" ​
13 ​
14template_msg = ('{"msg": { "status": "FINISHED",' 12​template_msg = ('{"msg": { "status": "FINISHED",'
15 '"location":' 13​ '"location":'
16 '"http://kojipkgs.fedoraproject.org/compose/Fedora-Cloud-24-20170224.0/compose",' 14​ '"https://kojipkgs.fedoraproject.org/compose/Fedora-Cloud-25-20170313.0/compose",'
17 '"compose_id": ""}}') 15​ '"compose_id": ""}}')
18 16​
19msg_good = json.loads(template_msg) 17​msg_good = json.loads(template_msg)
20msg_good['msg']['compose_id'] = "Fedora-Cloud-24-20170302.0" 18​msg_good['msg']['compose_id'] = "Fedora-Cloud-24-20170302.0"
21 19​
22msg_bad_compose_id = json.loads(template_msg) 20​msg_bad_compose_id = json.loads(template_msg)
23msg_bad_compose_id['msg']['compose_id'] = "Fedora-Cloud-23" 21​msg_bad_compose_id['msg']['compose_id'] = "Fedora-Cloud-23"
24 22​
25msg_atomic = json.loads(template_msg) 23​msg_atomic = json.loads(template_msg)
26msg_atomic['msg']['compose_id'] = "Fedora-Atomic-25-20170302.0" 24​msg_atomic['msg']['compose_id'] = "Fedora-Atomic-25-20170302.0"
27 25​
28 26​
27​@pytest.fixture(autouse=True)
28​def _no_requests(monkeypatch):
29​ monkeypatch.setattr(cloud_compose_complete_msg,
30​ "_fetch_metadata",
31​ _fetch_metadata)
32​
33​
34​@pytest.fixture(scope="module")
35​def _fetch_metadata(compose_url):
36​ """This is a stub to provide mocked requests objects so we don't have
37​ to hit the network in our tests."""
38​
39​ # This is really ugly, but it's as sparse a dataset as we need to
40​ # complete the test.
41​ dummy_data = ('{"payload":'
42​ '{"images":'
43​ '{"CloudImages":'
44​ '{"x86_64": ['
45​ '{"format": "qcow2",'
46​ '"path": "/test/path/image.qcow2"'
47​ '}]'
48​ '}'
49​ '}'
50​ '}'
51​ '}')
52​
53​ return json.loads(dummy_data)
54​
55​
29def test_process(): 56​def test_process():
30 """Test that nothing fails with known good data.""" 57​ """Test that nothing fails with known good data."""
58​
31 data = cloud_compose_complete_msg._process(msg_good) 59​ data = cloud_compose_complete_msg._process(msg_good)
32 assert data['message_type'] == 'CloudCompose' 60​ assert data['message_type'] == 'CloudCompose'
33 assert data['name'] == 'Fedora-Cloud' 61​ assert data['name'] == 'Fedora-Cloud'
34 assert data['version'] == 'Cloud' 62​ assert data['version'] == 'Cloud'
35 assert data['release'] == '24' 63​ assert data['release'] == '24'
36 64​
37 65​
38def test_process_atomic(): 66​def test_process_atomic():
39 """Test that the detection of Atomic composes sets the correct 67​ """Test that the detection of Atomic composes sets the correct
40 message_type, which would be "AtomicCompose".""" 68​ message_type, which would be "AtomicCompose"."""
41 data = cloud_compose_complete_msg._process(msg_atomic) ​
42 69​
43 print data 70​ data = cloud_compose_complete_msg._process(msg_atomic)
44 71​
45 assert data['name'] == "Fedora-Atomic" 72​ assert data['name'] == "Fedora-Atomic"
46 assert data['version'] == "Atomic" 73​ assert data['version'] == "Atomic"
47 assert data['message_type'] == "AtomicCompose" 74​ assert data['message_type'] == "AtomicCompose"
48 75​
49 76​
50def test_pull_version_release(): 77​def test_pull_version_release():
51 """Ensure that execution failes if the 'compose_id' is malformed.""" 78​ """Ensure that execution failes if the 'compose_id' is malformed."""
52 with pytest.raises(exc.TriggerMsgError): 79​ with pytest.raises(exc.TriggerMsgError):
53 cloud_compose_complete_msg._process(msg_bad_compose_id) 80​ cloud_compose_complete_msg._process(msg_bad_compose_id)