Differential D44

T110, T116 - Export keyvals, Import TAP
ClosedPublic
Actions

Authored by jskladan on Apr 3 2014, 3:22 PM.

Details

Reviewers
kparal
tflink
mkrizek
pschindl
Maniphest Tasks
T110: Export keyvals from CheckDetail to TAP
T116: Write a method for converting TAP to CheckDetail
Commits
rLTRNf39c5e57c53c: T110, T116 - Export keyvals, Import TAP
Summary

Added functionality for exporting keyvals from TAP.
Added functionality for converting the TAP output into a list
CheckDetails.

Test Plan

Unit-tests appended

Diff Detail

Branch
feature/T110-check-keyvals
Lint
No Linters Available
Unit
No Unit Test Coverage
tflink accepted this revision.Apr 3 2014, 4:50 PM

Looks good to me

jskladan closed this revision.Apr 4 2014, 9:15 AM
kparal added inline comments.Apr 4 2014, 9:41 AM
libtaskotron/check.py
18

Can you add a comment here explaining the purpose?

315

Can you document thrown exceptions as well? And clarify that source is a string?

321

Is e.message the same as str(e)? I wouldn't want to lose some information here.

Btw, you can always do this:

log.exception('Failed parsing of the TAP output')
raise exc.TaskotronValueError('Failed parsing of the TAP output: %s' % e)

That will log the whole error stacktrace, message, everything. But I'm not sure we need it in this case, we probably need just the parser error describing what line was incorrect.

Revision Contents

Path
Mlibtaskotron/check.py (50 lines)
Mrequirements.txt (1 line)
Mtesting/test_check.py (110 lines)

Diff 115

View Options

libtaskotron/check.py

1# -*- coding: utf-8 -*- 1​# -*- coding: utf-8 -*-
2# Copyright 2014, Red Hat, Inc. 2​# Copyright 2014, Red Hat, Inc.
3# License: GNU General Public License version 2 or later 3​# License: GNU General Public License version 2 or later
4 4​
5'''Helper tools for managing check status, outcome, and output.''' 5​'''Helper tools for managing check status, outcome, and output.'''
6 6​
7import pprint 7​import pprint
8​import collections
8 9​
9import tap 10​import tap
10 11​
11from . import python_utils 12​from . import python_utils
12from . import exceptions as exc 13​from . import exceptions as exc
14​from .logger import log
13 15​
16​from pytap13 import TAP13
17​
18​RESERVED_KEYS = ('item', 'type', 'outcome', 'summary', 'details')
kparalUnsubmitted
Not Done

Can you add a comment here explaining the purpose?

14 19​
15class CheckDetail(object): 20​class CheckDetail(object):
16 '''Class encompassing everything related to the outcome of a check run. 21​ '''Class encompassing everything related to the outcome of a check run.
17 It contains convenience methods for tracking, evaluating and reporting 22​ It contains convenience methods for tracking, evaluating and reporting
18 check results. 23​ check results.
19 24​
20 For some checks, it's trivial to parse its output at the end of its 25​ For some checks, it's trivial to parse its output at the end of its
21 execution and evaluate the results. For some, however, it's much easier 26​ execution and evaluate the results. For some, however, it's much easier
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Line(s)68 @attr keyvals all key-value pairs in this dictionary are stored in ResultsDB
66 querying the results. Keep it as short as possible. 71​ querying the results. Keep it as short as possible.
67 ''' 72​ '''
68 73​
69 outcome_priority = ('PASSED', 'INFO', 'FAILED', 'NEEDS_INSPECTION', 74​ outcome_priority = ('PASSED', 'INFO', 'FAILED', 'NEEDS_INSPECTION',
70 'ABORTED', 'CRASHED') 75​ 'ABORTED', 'CRASHED')
71 76​
72 77​
73 def __init__(self, item, report_type=None, outcome=None, summary='', 78​ def __init__(self, item, report_type=None, outcome=None, summary='',
74 output=None): 79​ output=None, keyvals=None):
75 '''Create new CheckDetail. 80​ '''Create new CheckDetail.
76 For param descriptions see class docstring.''' 81​ For param descriptions see class docstring.'''
77 82​
78 # validate input 83​ # validate input
79 if output is not None and not python_utils.listlike(output): 84​ if output is not None and not python_utils.listlike(output):
80 raise exc.TaskotronValueError("'output' parameter must be a list. " 85​ raise exc.TaskotronValueError("'output' parameter must be a list. "
81 "Yours was: %s" % type(output)) 86​ "Yours was: %s" % type(output))
87​ if keyvals is not None and not isinstance(keyvals, collections.Mapping):
88​ raise exc.TaskotronValueError("'keyvals' parameter must behave like"
89​ " a dictionary. Yours was: %s" % type(keyvals))
82 90​
83 self.item = item 91​ self.item = item
84 self.report_type = report_type 92​ self.report_type = report_type
85 self._outcome = None 93​ self._outcome = None
86 if outcome: 94​ if outcome:
87 self.outcome = outcome 95​ self.outcome = outcome
88 self.summary = summary 96​ self.summary = summary
89 self.output = output or [] 97​ self.output = output or []
90 self.keyvals = {} 98​ self.keyvals = keyvals or {}
91 99​
92 100​
93 @property 101​ @property
94 def outcome(self): 102​ def outcome(self):
95 '''See class documentation.''' 103​ '''See class documentation.'''
96 return self._outcome or 'NEEDS_INSPECTION' 104​ return self._outcome or 'NEEDS_INSPECTION'
97 105​
98 106​
▲ Show 20 Lines • Show All 186 Lines • ▼ Show 20 Line(s)275 for detail in check_details:
285 if detail.summary: 293​ if detail.summary:
286 data['summary'] = detail.summary 294​ data['summary'] = detail.summary
287 data['details'] = {} 295​ data['details'] = {}
288 if detail.output: 296​ if detail.output:
289 data['details']['output'] = '\n'.join(detail.output) 297​ data['details']['output'] = '\n'.join(detail.output)
290 if not data['details']: 298​ if not data['details']:
291 del data['details'] 299​ del data['details']
292 300​
301​ for key, value in detail.keyvals.iteritems():
302​ if key in RESERVED_KEYS:
303​ log.warn("Reserved key %r in keyvals. Ignoring for export purposes.", key)
304​ continue
305​
306​ data[key] = value
307​
293 # generate 308​ # generate
294 output = tapgen.format_TAP_msg(result=tap_outcome, name=name, data=data) 309​ output = tapgen.format_TAP_msg(result=tap_outcome, name=name, data=data)
295 tapout.append(output) 310​ tapout.append(output)
296 311​
297 return '\n'.join(tapout) 312​ return '\n'.join(tapout)
298 313​
314​def import_TAP(source):
315​ '''Parses the source, and returns list of CheckDetails'''
kparalUnsubmitted
Not Done

Can you document thrown exceptions as well? And clarify that source is a string?

316​
317​ tap = TAP13()
318​ try:
319​ tap.parse(source)
320​ except ValueError as e:
321​ raise exc.TaskotronValueError('Failed parsing of the TAP output: %s' % e.message)
kparalUnsubmitted
Not Done

Is e.message the same as str(e)? I wouldn't want to lose some information here.

Btw, you can always do this:

log.exception('Failed parsing of the TAP output')
raise exc.TaskotronValueError('Failed parsing of the TAP output: %s' % e)

That will log the whole error stacktrace, message, everything. But I'm not sure we need it in this case, we probably need just the parser error describing what line was incorrect.

322​
323​ check_details = []
324​ for test in tap.tests:
325​ y = test.yaml
326​ item = y.get('item', None)
327​ report_type = y.get('type', None)
328​ outcome = 'PASSED' if test.result == 'ok' else 'FAILED'
329​ outcome = y.get('outcome', outcome)
330​ summary = y.get('summary', '')
331​ details = y.get('details', {})
332​ output = details.get('output', None)
333​
334​ # TODO: is there a better way to do this?
335​ other_keys = set(y.keys()) - set(RESERVED_KEYS)
336​ keyvals = dict([(k,y[k]) for k in other_keys])
337​
338​ cd = CheckDetail(item, report_type, outcome, summary, [output])
339​
340​ cd.keyvals = keyvals
341​ check_details.append(cd)
342​
343​ return check_details
344​
View Options

requirements.txt

1PyYAML==3.10 1​PyYAML==3.10
2dingus==0.3.4 2​dingus==0.3.4
3py>=1.4.18 3​py>=1.4.18
4pyaml==13.07.1 4​pyaml==13.07.1
5pytest>=2.4.2 5​pytest>=2.4.2
6pytest-cov>=1.6 6​pytest-cov>=1.6
7jinja2>=2.7.2 7​jinja2>=2.7.2
8bayeux>=0.8 8​bayeux>=0.8
9yamlish>=0.11 9​yamlish>=0.11
10​pytap13>=0.0.2
View Options

testing/test_check.py

1# -*- coding: utf-8 -*- 1​# -*- coding: utf-8 -*-
2# Copyright 2014, Red Hat, Inc. 2​# Copyright 2014, Red Hat, Inc.
3# License: GNU General Public License version 2 or later 3​# License: GNU General Public License version 2 or later
4 4​
5'''Unit tests for libtaskotron/check.py''' 5​'''Unit tests for libtaskotron/check.py'''
6 6​
7import pytest 7​import pytest
8import yaml 8​import yaml
9​from copy import deepcopy
9 10​
10from libtaskotron import check 11​from libtaskotron import check
11from libtaskotron import exceptions as exc 12​from libtaskotron import exceptions as exc
12 13​
13class TestCheckDetail(object): 14​class TestCheckDetail(object):
14 15​
15 def setup_method(self, method): 16​ def setup_method(self, method):
16 '''Run this before every test invocation''' 17​ '''Run this before every test invocation'''
▲ Show 20 Lines • Show All 165 Lines • ▼ Show 20 Line(s)181class TestExportTAP(object):
182 item = 'xchat-0.5-1.fc20' 183​ item = 'xchat-0.5-1.fc20'
183 outcome = 'PASSED' 184​ outcome = 'PASSED'
184 summary='5 ERRORS, 10 WARNINGS' 185​ summary='5 ERRORS, 10 WARNINGS'
185 report_type=check.ReportType.KOJI_BUILD 186​ report_type=check.ReportType.KOJI_BUILD
186 output = '''\ 187​ output = '''\
187xchat.x86_64: W: file-not-utf8 /usr/share/doc/xchat/ChangeLog 188​xchat.x86_64: W: file-not-utf8 /usr/share/doc/xchat/ChangeLog
188xchat.x86_64: W: non-conffile-in-etc /etc/gconf/schemas/apps_xchat_url_handler.schemas 189​xchat.x86_64: W: non-conffile-in-etc /etc/gconf/schemas/apps_xchat_url_handler.schemas
189xchat.x86_64: W: no-manual-page-for-binary xchat''' 190​xchat.x86_64: W: no-manual-page-for-binary xchat'''
191​ keyvals = {"foo": "bar", "moo": 11}
190 192​
191 def setup_method(self, method): 193​ def setup_method(self, method):
192 self.cd = check.CheckDetail(item=self.item, outcome=self.outcome, 194​ self.cd = check.CheckDetail(item=self.item, outcome=self.outcome,
193 summary=self.summary, 195​ summary=self.summary,
194 report_type=self.report_type) 196​ report_type=self.report_type,
197​ keyvals = self.keyvals)
195 self.cd.output.append(self.output) 198​ self.cd.output.append(self.output)
196 199​
197 def _tap_to_yaml(self, tap): 200​ def _tap_to_yaml(self, tap):
198 '''Remove all TAP-related contents and return only its YAML block(s)''' 201​ '''Remove all TAP-related contents and return only its YAML block(s)'''
199 lines = tap.splitlines() 202​ lines = tap.splitlines()
200 203​
201 # split the output to TAP result lines and YAML blocks 204​ # split the output to TAP result lines and YAML blocks
202 tap_lines = [line for line in lines if line.startswith('ok') or 205​ tap_lines = [line for line in lines if line.startswith('ok') or
Show All 22 Lines222 def test_single_yaml(self):
225 yaml_obj = yaml.load(yaml_block) 228​ yaml_obj = yaml.load(yaml_block)
226 229​
227 assert yaml_obj['item'] == self.item 230​ assert yaml_obj['item'] == self.item
228 assert yaml_obj['outcome'] == self.outcome 231​ assert yaml_obj['outcome'] == self.outcome
229 assert yaml_obj['summary'] == self.summary 232​ assert yaml_obj['summary'] == self.summary
230 assert yaml_obj['type'] == self.report_type 233​ assert yaml_obj['type'] == self.report_type
231 assert yaml_obj['details'] 234​ assert yaml_obj['details']
232 assert yaml_obj['details']['output'] == self.output 235​ assert yaml_obj['details']['output'] == self.output
236​ assert yaml_obj['foo'] == self.keyvals['foo']
237​ assert yaml_obj['moo'] == self.keyvals['moo']
238​
239​ def test_invalid_keyvals(self):
240​ '''Test export with keyvals containing reserved keys.'''
241​ cd = deepcopy(self.cd)
242​
243​ for key in check.RESERVED_KEYS:
244​ cd.keyvals[key] = 'foo'
245​
246​ tap = check.export_TAP(cd)
247​
248​ # specific lines must exist
249​ yaml_block = self._tap_to_yaml(tap)
250​ yaml_obj = yaml.load(yaml_block)
251​
252​ assert yaml_obj['item'] == self.item
253​ assert yaml_obj['outcome'] == self.outcome
254​ assert yaml_obj['summary'] == self.summary
255​ assert yaml_obj['type'] == self.report_type
256​ assert yaml_obj['details']
257​ assert yaml_obj['details']['output'] == self.output
258​ assert yaml_obj['foo'] == self.keyvals['foo']
259​ assert yaml_obj['moo'] == self.keyvals['moo']
233 260​
234 def test_multi(self): 261​ def test_multi(self):
235 '''Test export with multiple item sections''' 262​ '''Test export with multiple item sections'''
236 cd2 = check.CheckDetail(item='foobar-1.2-3.fc20', outcome='FAILED', 263​ cd2 = check.CheckDetail(item='foobar-1.2-3.fc20', outcome='FAILED',
237 summary='dependency error', 264​ summary='dependency error',
238 report_type=check.ReportType.BODHI_UPDATE) 265​ report_type=check.ReportType.BODHI_UPDATE)
239 cd3 = check.CheckDetail(item='f20-updates', outcome='INFO', 266​ cd3 = check.CheckDetail(item='f20-updates', outcome='INFO',
240 summary='2 stale updates', 267​ summary='2 stale updates',
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Line(s)
302 def test_custom_report_type(self): 329​ def test_custom_report_type(self):
303 '''Test that user can specify a custom CheckDetail.report_type''' 330​ '''Test that user can specify a custom CheckDetail.report_type'''
304 self.cd.report_type = 'My Custom Type' 331​ self.cd.report_type = 'My Custom Type'
305 tap = check.export_TAP(self.cd) 332​ tap = check.export_TAP(self.cd)
306 yaml_block = self._tap_to_yaml(tap) 333​ yaml_block = self._tap_to_yaml(tap)
307 yaml_obj = yaml.load(yaml_block) 334​ yaml_obj = yaml.load(yaml_block)
308 335​
309 assert yaml_obj['type'] == self.cd.report_type 336​ assert yaml_obj['type'] == self.cd.report_type
337​
338​
339​class TestImportTAP(object):
340​ '''Test @method import_TAP'''
341​ item = 'xchat-0.5-1.fc20'
342​ outcome = 'PASSED'
343​ summary='5 ERRORS, 10 WARNINGS'
344​ report_type=check.ReportType.KOJI_BUILD
345​ output = '''\
346​xchat.x86_64: W: file-not-utf8 /usr/share/doc/xchat/ChangeLog
347​xchat.x86_64: W: non-conffile-in-etc /etc/gconf/schemas/apps_xchat_url_handler.schemas
348​xchat.x86_64: W: no-manual-page-for-binary xchat'''
349​ keyvals = {"foo": "bar", "moo": 11}
350​
351​ tap = """
352​TAP version 13
353​1..1
354​ok - $CHECKNAME for Koji build xchat-0.5-1.fc20
355​ ---
356​ details:
357​ output: |-
358​ xchat.x86_64: W: file-not-utf8 /usr/share/doc/xchat/ChangeLog
359​ xchat.x86_64: W: non-conffile-in-etc /etc/gconf/schemas/apps_xchat_url_handler.schemas
360​ xchat.x86_64: W: no-manual-page-for-binary xchat
361​ item: xchat-0.5-1.fc20
362​ outcome: PASSED
363​ summary: 5 ERRORS, 10 WARNINGS
364​ type: koji_build
365​ foo: bar
366​ moo: 11
367​ ..."""
368​
369​ def setup_method(self, method):
370​ self.cd = check.CheckDetail(item=self.item, outcome=self.outcome,
371​ summary=self.summary,
372​ report_type=self.report_type,
373​ keyvals=self.keyvals)
374​ self.cd.output.append(self.output)
375​
376​ def test_import(self):
377​ cds = check.import_TAP(self.tap)
378​
379​ assert len(cds) == 1
380​ assert isinstance(cds[0], check.CheckDetail)
381​
382​ cd = cds[0]
383​ assert cd.item == self.cd.item
384​ assert cd.outcome == self.cd.outcome
385​ assert cd.summary == self.cd.summary
386​ assert cd.report_type == self.cd.report_type
387​ assert cd.output == self.cd.output
388​ assert cd.keyvals == self.keyvals
389​
390​ def test_import_exported(self):
391​ tap = check.export_TAP(self.cd)
392​
393​ #TODO: remove this ugly hack - it is here because of the bug in bayeux
394​ tap = "TAP version 13\n1..1\n%s" % tap
395​
396​ cds = check.import_TAP(self.tap)
397​
398​ assert len(cds) == 1
399​ assert isinstance(cds[0], check.CheckDetail)
400​
401​ cd = cds[0]
402​ assert cd.item == self.cd.item
403​ assert cd.outcome == self.cd.outcome
404​ assert cd.summary == self.cd.summary
405​ assert cd.report_type == self.cd.report_type
406​ assert cd.output == self.cd.output
407​ assert cd.keyvals == self.keyvals
408​
409​
410​ def test_invalid_tap(self):
411​ tap = """
412​TAP version 13
413​ok 2 foo
414​ok 1 bar"""
415​ with pytest.raises(exc.TaskotronValueError):
416​ check.import_TAP(tap)
417​