Index: libtaskotron/bodhi_comment_utils.py =================================================================== --- /dev/null +++ libtaskotron/bodhi_comment_utils.py @@ -0,0 +1,391 @@ +# Copyright 2011, Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: +# Tim Flink +# Josef Skladanka + + +import datetime +import peak.util.symbols + +import libtaskotron.bodhi_utils as bodhi_utils +import libtaskotron.config as config +from libtaskotron.rpm_utils import get_basearch + + +# symbols used for consistant declaration of state +PASS = peak.util.symbols.Symbol('PASS', __name__) +FAIL = peak.util.symbols.Symbol('FAIL', __name__) +INCOMPLETE = peak.util.symbols.Symbol('INCOMPLETE', __name__) +UNKNOWN = peak.util.symbols.Symbol('UNKNOWN', __name__) + +#TODO: go through the BodhiUpdateState and try to remove the references +# to depcheck & upgradepath +class BodhiUpdateState: + + def __init__(self, name, kojitag='updates'): + self.test_states = { 'depcheck_32':'NORUN', + 'depcheck_64':'NORUN', + 'upgradepath':'NORUN' + } + + # since upgradepath is only run on updates-pending, + # set it to passed if we're working on updates-testing + if kojitag == 'updates-testing': + self.test_states['upgradepath'] = 'PASSED' + + self.result_history = [] + self.update_name = name + self.test_change = False + + def num_results(self): + ''' Retrieve the number of results currently held ''' + return len(self.result_history) + + def add_result(self, new_result): + ''' Update the current state with a new result. + + Args: + new_result: a dictionary containing the following keys: + time (datetime object), testname, result, arch + ''' + + self.result_history.append(new_result) + + # copy the current test state so that we can compare later + old_state = self.test_states.copy() + + # for depcheck, we only care about i386 and x86_64 arches + # anything else will be ignored + if new_result['testname'] == 'depcheck': + arch = get_basearch(new_result['arch']) + + if arch == 'i386': + self.test_states['depcheck_32'] = new_result['result'] + + elif arch == 'x86_64': + self.test_states['depcheck_64'] = new_result['result'] + + else: + print "Arch %s is not valid for depcheck and will be ignored." \ + % new_result['arch'] + + # don't care about the arch for upgrade path. It should be noarch + # but the results aren't changed if it isn't + elif new_result['testname'] == 'upgradepath': + self.test_states['upgradepath'] = new_result['result'] + + # detect if any tests changed from the last result + if old_state != self.test_states: + self.test_change = True + else: + self.test_change = False + + def get_state(self): + ''' Determine state based on currently known results ''' + + # print out the current state for debugging purposes + state_str = "Current State of update %s :" % self.update_name + state_str += '\n %s' % self.test_states + for result in self.result_history: + state_str += '\n %s' % result + print state_str + '\n' + + test_values = set(self.test_states.values()) + if 'NORUN' in test_values: + return INCOMPLETE + if 'FAILED' in test_values: + return FAIL + if 'PASSED' in test_values and len(test_values) == 1: + return PASS + + return UNKNOWN + + def did_test_change(self): + ''' Determine whether the state of any tests changed with the addition + of the last result''' + return self.test_change + + +def _parse_result_from_comment(comment): + ''' + Parses timestamp and results from bodhi comments + + Args: + comment -- the string containing comment + ''' + + comment_time = datetime.strptime(comment['timestamp'], '%Y-%m-%d %H:%M:%S') + + # comment form to match: + # 'AutoQA: %s test %s on %s. Result log: %s (results are informative only)' \ + # note that it isn't looking for the autoqa user right now, that needs to + # be done in any code that calls this + comment_match = re.match(r'Taskotron: (?P\w+) test (?P\w+)'\ + r' on (?P\w+)\. Result log:[\t \n]+'\ + r'(?P[/:\w]+).*', comment['text']) + + test_name = '' + result = '' + arch = '' + #result_url = '' + + if comment_match: + test_name = comment_match.group('test_name') + result = comment_match.group('result') + arch = comment_match.group('arch') + #result_url = comment_match.group('result_url') + else: + print >> sys.stderr, 'Failed to parse bodhi comment: %s' % comment['text'] + + return {'time':comment_time, 'testname':test_name, + 'result':result, 'arch':arch} + + +def bodhi_already_commented(update, testname, arch): + '''Check if Taskotron comment is already posted. + + Args: + update -- Bodhi update object --or-- update title --or-- update ID --or-- package NVR + testname -- the name of the test + arch -- tested architecture + + Note: Only NVR allowed, not ENVR. See https://fedorahosted.org/bodhi/ticket/592. + + Returns: + Tuple containing old result and time when the last comment was posted. + If no comment is posted yet, or it is, but the update + has been modified since, tuple will contain two empty strings. + + Throws: + ValueError -- if no such update can be found + ''' + + Config = config.get_config() + + # if we received update title or ID, let's convert it to update object first + if isinstance(update, unicode) or isinstance(update, str): + u = bodhi_utils.query_update(update) + if u: + update = u + else: + raise ValueError("No such update: %s" % update) + + comment_re = r'Taskotron:[\s]+%s[\s]+test[\s]+(\w+)[\s]+on[\s]+%s' % (testname, arch) + old_result = '' + comment_time = '' + + taskotron_comments = [comment for comment in update['comments'] + if comment['author'] == Config.fas_username] + for comment in taskotron_comments: + m = re.match(comment_re, comment['text']) + if m is None: + continue + old_result = m.group(1) + comment_time = comment['timestamp'] + # check whether update was modified after the last posted comment + if update['date_modified'] > comment_time: + return ('','') + return (old_result, comment_time) + +def _is_bodhi_testresult_needed(old_result, comment_time, result, time_span): + '''Check if the comment is meant to be posted. + + Args: + old_result -- the result of the last test + comment_time -- the comment time of the last test + result -- the result of the test + time_span -- waiting period before posting the same comment + + Returns: + True if the comment will be posted, False otherwise. + ''' + # the first comment or a comment with different result, post it + if not old_result or old_result != result: + return True + + # If we got here, it means that the comment with the same result has been + # already posted, we now need to determine whether we can post the + # comment again or not. + # If the previous result is *not* 'FAILED', we won't post it in order not to + # spam developers. + # If the previous result *is* 'FAILED', we will need to check whether given + # time span expired, if so, we will post the same comment again to remind + # a developer about the issue. + + if result != 'FAILED': + return False + + posted_datetime = datetime.strptime(comment_time, '%Y-%m-%d %H:%M:%S') + delta = (datetime.utcnow() - posted_datetime) + # total_seconds() is introduced in python 2.7, until 2.7 is everywhere... + total_seconds = (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6 + minutes = total_seconds/60.0 + if minutes < time_span: + return False + + return True + +def _is_bodhi_comment_email_needed(update_name, parsed_comments, new_result, + kojitag): + ''' + Determines whether or not to send an email with the comment posted to bodhi. + Uses previous comments on update in order to determine current state + + Args: + update_name -- name of the update to be tested + parsed_comments -- already existing AutoQA comments for the update + new_result -- the new result to be posted + kojitag -- the koji tag being tested (affects the expected tests) + ''' + + Config = config.get_config() + + print "checking email needed for %s (%s)" % (update_name, kojitag) + # compute current state + update_state = BodhiUpdateState(update_name, kojitag=kojitag) + for result in parsed_comments: + update_state.add_result(result) + + current_state = update_state.get_state() + update_state.add_result(new_result) + new_state = update_state.get_state() + update_state_change = current_state != new_state + + # time to figure out whether or not to send an email + send_email = False + + print 'update state : %s to %s' % (str(current_state), str(new_state)) + + # if we always send an email, just return true + if Config.email_always_send: + return True + + # by default, we want to send email on state change, so start from there + if update_state_change: + send_email = True + + # if configured, don't send an email if all tests are passed unless the + # state change is from fail to pass, then we want to leave the current + # state alone. + if not Config.email_all_passed: + if new_state is PASS \ + and current_state is not FAIL: + + print 'not sending email due to configuration for email_all_passed' + send_email = False + + # send emails on test state changes if configured + if Config.email_test_state_change: + if update_state.did_test_change(): + print 'sending email due to configuration for email_test_state_change' + send_email = True + + # don't send results from updates_testing if not configured + if not Config.email_updates_testing: + if kojitag.endswith('updates-testing'): + print 'not sending email due to configuration for email_updates_testing' + send_email = False + + + return send_email + +def bodhi_post_testresult(update, testname, result, url, arch = 'noarch', + karma = 0, doreport='onchange'): + '''Post comment and karma to bodhi + + Args: + update -- the *title* of the update comment on + testname -- the name of the test + result -- the result of the test + url -- url of the result of the test + arch -- tested architecture (default 'noarch') + karma -- karma points (default 0) + doreport -- set to 'all' to "force" posting bodhi comment + + Returns: + True if comment was posted successfully or comment wasn't meant to be + posted (either posting is turned off or comment was already posted), + False otherwise. + ''' + + # TODO when new bodhi releases, add update identification by UPDATEID support + + Config = config.get_config() + + err_msg = 'Could not post a comment to bodhi' + + if not update or not testname or not result or url == None: + sys.stderr.write('Incomplete arguments!\n%s\n' % err_msg) + return False + + if not Config.send_bodhi_comments: + print 'Sending bodhi comments is turned off. Test result will NOT be sent.' + # do not send comments (but return True since it's intentional, not an error) + return True + + if not Config.fas_username or not Config.fas_password: + sys.stderr.write('Conf file containing FAS credentials is incomplete!'\ + '\n%s\n' % err_msg) + return False + + comment = 'Taskotron: %s test %s on %s. Result log: %s ' \ + '(results are informative only)' % (testname, result, arch, url) + try: + (old_result, comment_time) = bodhi_already_commented(update, testname, arch) + time_span = Config.bodhi_posting_comments_span + + if doreport != 'all' and not _is_bodhi_testresult_needed(old_result, comment_time, result, time_span): + print 'The test result already posted to bodhi.' + return True + + bodhi_update = bodhi_utils.query_update(update) + parsed_results = [] + for found_comment in bodhi_update['comments']: + if found_comment['author'] == Config.fas_username: + parsed_results.append(_parse_result_from_comment(found_comment)) + + kojitag = 'updates-testing' + if bodhi_update['request'] == 'stable' or bodhi_update['status'] == 'stable': + kojitag = 'updates' + + new_result = {'time':datetime.utcnow(), 'testname':testname, + 'result':result, 'arch':arch} + send_email = _is_bodhi_comment_email_needed(update, parsed_results, + new_result, kojitag=kojitag) + + # log email decision + if send_email: + print "Bodhi email will be sent" + else: + print "Bodhi email will NOT be sent" + + if not bodhi_utils.bodhi.comment(update, comment, karma, send_email): + sys.stderr.write('%s\n' % err_msg) + return False + + print 'The test result was sent to bodhi successfully.' + except Exception, e: + sys.stderr.write('An error occured: %s\n' % e) + sys.stderr.write('Could not connect to bodhi!\n%s\n' % err_msg) + import traceback + traceback.print_exc(e) + return False + + return True + + Index: libtaskotron/check.py =================================================================== --- libtaskotron/check.py +++ libtaskotron/check.py @@ -9,6 +9,7 @@ from . import python_utils from . import exceptions as exc from . import tap +from pytap13 import TAP13 class CheckDetail(object): @@ -295,3 +296,28 @@ return '\n'.join(tapout) +def import_TAP(source): + '''Parses the source, and returns list of CheckDetails''' + tap = TAP13() + try: + tap.parse(source) + except ValueError: + raise exc.TaskotronValueError('Failed parsing of the TAP output') + + check_details = [] + for test in tap.tests: + y = test.yaml + item = y.get('item', None) + report_type = y.get('type', None) + outcome = 'PASSED' if test.result == 'ok' else 'FAILED' + outcome = y.get('outcome', outcome) + summary = y.get('summary', '') + details = y.get('details', {}) + output = details.get('output', None) + if output: + output = output.split('\n') + + check_details.append(CheckDetail(item, report_type, outcome, summary, output)) + + return check_details + Index: libtaskotron/directives/bodhi_directive.py =================================================================== --- /dev/null +++ libtaskotron/directives/bodhi_directive.py @@ -0,0 +1,79 @@ +# Copyright 2014, Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: +# Josef Skladanka + +from libtaskotron.directives import BaseDirective +from libtaskotron.directives.koji_directive import KojiDirective + +import libtaskotron.check as check +import libtaskotron.config as config + +from libtaskotron.exceptions import TaskotronError, TaskotronValueError +from libtaskotron.logger import log +from libtaskotron.bodhi_comment_utils import bodhi_post_testresult + +directive_class = 'BodhiDirective' + +class BodhiDirective(BaseDirective): + + def process(self, input_data, env_data): + valid_actions = ['comment'] + + output_data = {} + + action = input_data['action'] + + if action not in valid_actions: + raise TaskotronError('%s is not a valid command for bodhi directive' % action) + + if action == 'comment': + + Config = config.get_config() + if not (Config.reporting_enabled and Config.report_to_bodhi): + log.info("Reporting to Bodhi is disabled.") + return + + + if not ('doreport' in input_data and 'results' in input_data): + detected_args = ', '.join(input_data.keys()) + raise TaskotronError("The bodhi directive 'comment' requires 'doreport' and "\ + "'results' arguments. Detected arguments: %s" % detected_args) + if input_data['doreport'] not in ['all', 'onchange']: + raise Taskotron("The argument 'doreport' is set to invalid value '%s'. "\ + "Valid values are: 'all', 'onchange'.") + + try: + check_details = check.import_TAP(input_data['results']) + except TaskotronValueError: + raise TaskotronError("Failed to load the 'results'") + + # Filter the results of "BODHI_UPDATE" type + check_details = [r for r in check_details if r.report_type == check.ReportType.BODHI_UPDATE] + + # ? Log when no results of the type are found ? + + for ch_d in check_details: + + #TODO: replace url with proper URL + r = bodhi_post_testresult(ch_d.item, env_data['checkname'], ch_d.outcome, + url = "http://example.com", doreport = input_data['doreport']) + + if not r: + log.info("Failed to post Bodhi Comment: `%s` `%s` `%s`", + ch_d.item, env_data['checkname'], ch_d.outcome) + Index: libtaskotron/rpm_utils.py =================================================================== --- /dev/null +++ libtaskotron/rpm_utils.py @@ -0,0 +1,38 @@ +# Copyright 2014, Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: +# Tim Flink +# Josef Skladanka + + +def get_basearch(arch = None): + ''' + If 'arch' parameter is not given, this behaves like get_arch, + but returns the basearch (as used by yum etc.) + + If 'arch' parameter is set, this function returns appropriate + 'base' arch (i.e. i386 for i[456]86, ppc for ppc64) if applicable. + ''' + if arch == None: + arch = os.uname()[4] + if arch in ('i486', 'i586', 'i686'): + arch = 'i386' + elif arch == 'ppc64': + arch = 'ppc' + return arch + + Index: libtaskotron/runner.py =================================================================== --- libtaskotron/runner.py +++ libtaskotron/runner.py @@ -27,6 +27,7 @@ def run(self): self._validate_input() self.envdata['workdir'] = self.workdir + self.envdata['checkname'] = self.taskdata['name'] self.do_actions() Index: requirements.txt =================================================================== --- requirements.txt +++ requirements.txt @@ -5,6 +5,7 @@ pytest>=2.4.2 pytest-cov>=1.6 jinja2>=2.7.2 +pytap13 >= 0.0.1 # these are bundled in libtaskotron for now #bayeux==0.7