#!/usr/bin/python import os import sys import locale import optparse import re import copy from getpass import getpass try: import bugzilla except ImportError: print "Unable to import bugzilla. Is python-bugzilla installed?" sys.exit(1) try: from simplemediawiki import MediaWiki except ImportError: print "Unable to import simplemediawiki. Is python-simplemediawiki installed?" sys.exit(1) BUG_STATUS = ['NEW', 'ASSIGNED', 'ON_DEV', 'MODIFIED', 'POST', 'ON_QA', 'FAILS_QA', 'PASSES_QA', 'REOPENED', 'VERIFIED', 'RELEASE_PENDING'] # Blocker query values BLOCKER_ACCEPTED = {'status_whiteboard': 'AcceptedBlocker', 'status_whiteboard_type': 'anywords'} BLOCKER_PROPOSED = {'status_whiteboard': 'AcceptedBlocker RejectedBlocker', 'status_whiteboard_type': 'nowords'} # NTH query values NTH_ACCEPTED = {'status_whiteboard': 'AcceptedNTH', 'status_whiteboard_type': 'anywords'} NTH_PROPOSED = {'status_whiteboard': 'AcceptedNTH RejectedNTH', 'status_whiteboard_type': 'nowords'} COOKIE_FILE = os.path.join(os.environ.get('HOME','/tmp'), '.fedora_cookiefile') # UTF8 helper - "borrored" from python-bugzilla def to_encoding(ustring): if isinstance(ustring, basestring): if isinstance(ustring, unicode): return ustring.encode(locale.getpreferredencoding(), 'replace') return ustring return u'' # Helper method to flatten a list of lists def join_lists(l): ''' Takes a list of lists, and joins them into a single list. For example: [[1,2,3],[4],[5,6,7]] will become [1,2,3,4,5,6,7] ''' return [item for sublist in l for item in sublist] # Display list of bugs, organized by components def wikilist_bugs(bugs_by_component, bugs_by_id): buf = '' components = sorted(bugs_by_component.keys()) for component in components: buf += '\n=== %s ===\n' % component # sorted list bugs = sorted(bugs_by_component.get(component,[])) for b in bugs: b = bugs_by_id[b] buf += '* [https://bugzilla.redhat.com/show_bug.cgi?id=%s %s] (%s) - %s\n' % (b.bug_id, b.bug_id, b.bug_status, b.short_desc) return buf # Display sortable table of bugs def wikitable_bugs(bugs_by_component, bugs_by_id): buf = '' components = sorted(bugs_by_component.keys()) if len(components) > 0: buf = '''{| class="wikitable sortable" ! Bug !! Component !! Status !! Description !! class="unsortable"|Links |- \n''' for component in components: # sorted list bugs = sorted(bugs_by_component.get(component,[])) for b in bugs: b = bugs_by_id[b] buf += '| [http://bugzilla.redhat.com/show_bug.cgi?id=%s %s] || %s ' % (b.bug_id, b.bug_id, component) buf += '|| %s || %s || ' % (b.bug_status, b.short_desc,) buf += '[[Image:Infra-icon_koji.png|link=http://koji.fedoraproject.org/koji/search?type=package&match=glob&terms=%s|alt=Build Status]] ' % (component) buf += '[[Image:Infra-icon_bodhi.png|link=http://admin.fedoraproject.org/updates/%s|alt=Update Status]] ' % (component) buf += '[[Image:Infra-icon_cvssource.png|link=http://pkgs.fedoraproject.org/gitweb/?p=%s.git|alt=Package Source]] ' % (component) buf += '[[Image:Infra-icon_bugzilla.png|link=http://admin.fedoraproject.org/pkgdb/acls/bugs/%s|alt=Bug Reports]] ' % (component) buf += '\n|-\n' # End table row buf += '|}\n' return buf def parse_args(): '''Set up the option parser''' parser = optparse.OptionParser(usage="%prog [options]") parser.add_option('-v', '--verbose', action='store_true', default=False, help='Enable verbosity') parser.add_option('--hotdog', action='store_true', default=False, help='Enable hot dog (default: %default)') parser.add_option('--cookiefile', action='store', default=COOKIE_FILE, help='Cookiejar path to store wiki login cookies (default: %default)') optgrp = optparse.OptionGroup(parser, "Required options") optgrp.add_option('-n', '--name', action='store', default=None, help='Wiki page name to save results (required)') optgrp.add_option('--blocker', action='store', default='auto', help='Blocker tracking bug number (default: %default)') optgrp.add_option('--nth', action='store', default='auto', help='Nice-to-Have tracking bug number (default: %default)') optgrp.add_option('-m', '--milestone', type='choice', action='store', default=None, choices=['Alpha', 'Beta', 'Final',], help='Release milestone used when --blocker or --nth are set to \'auto\' (e.g. Alpha, Beta or Final)') optgrp.add_option('-u', '--user', action='store', default=None, help='Mediawiki username') optgrp.add_option('-p', '--passwd', action='store', default=None, help='Mediawiki password') parser.add_option_group(optgrp) (opts, args) = parser.parse_args() # sanitize helper def sanitize_input(parser, value, label, ispass=False): if value is None and sys.stdin.isatty(): prompt = 'Enter %s: ' % label if ispass: value = getpass(prompt) else: value = raw_input(prompt) if value is None: parser.error('Must provide a valid %s' % label) return value def valid_cookiefile(parser, cookiefile): # if it doesn't exist, it can't be valid if not os.path.exists(cookiefile): return False # Check if it contains valid data fd = open(cookiefile, 'r') buf = fd.read() fd.close() valid = False for line in buf.split('\n'): # Skip empty lines or lines with only comments if re.match(r'^(\s*#|\s*$)', line): continue return True # Otherwise, no valid content found return False if not valid_cookiefile(parser, opts.cookiefile): opts.user = sanitize_input(parser, opts.user, "username") opts.passwd = sanitize_input(parser, opts.passwd, "password", ispass=True) opts.blocker = sanitize_input(parser, opts.blocker, "Blocker bug number") opts.nth = sanitize_input(parser, opts.nth, "Nice-to-have bug number") opts.name = sanitize_input(parser, opts.name, "Wiki page name") opts.milestone = sanitize_input(parser, opts.milestone, "Release milestone") return opts def flatten_bug_list(query_args, max_depth=4): buglist = bz.query(query_args) # avoid endless recursion if max_depth <= 0: return buglist # find any additional dependencies new_deps = list() for b in buglist: if len(b.dependson) > 0: new_deps.append(b.bug_id) # Remove any duplicates existing_deps = {int(bug):True for bug in query_args['value0-0-0'].split()}.keys() new_deps = {bug:True for bug in new_deps}.keys() # Ignore anything already being tracked copy_new_deps = copy.deepcopy(new_deps) for bug in copy_new_deps: if bug in existing_deps: new_deps.remove(bug) if len(new_deps) > 0: query_args['value0-0-0'] = ' '.join([str(bug) for bug in existing_deps]) + ' ' + ' '.join([str(bug) for bug in new_deps]) if opts.verbose: print ' additional dependencies: %s' % ' '.join([str(bug) for bug in new_deps]) return flatten_bug_list(query_args, max_depth-1) return buglist if __name__ == '__main__': opts = parse_args() # Create mediawiki handle if opts.verbose: print 'Connecting to mediawiki ...' wiki = MediaWiki('https://fedoraproject.org/w/api.php', cookie_file=opts.cookiefile) # Get current Fedora version number if opts.blocker == 'auto' or opts.nth == 'auto': if opts.verbose: sys.stdout.write('Detecting upcoming Fedora release ... ') q = dict(action='expandtemplates', text='{{Template:FedoraVersionNumber|next}}',) response = wiki.call(q) release = response.get('expandtemplates',{}).get('*','') if opts.verbose: print release if release.isdigit(): if opts.blocker == 'auto': # HACK - the blocker bug format is inconsistently named opts.blocker = 'F{release}{milestone}'.format(release=release, milestone=re.sub('Final', 'Blocker', opts.milestone)) if opts.nth == 'auto': # HACK - the blocker bug format is inconsistently named opts.nth = 'F{release}{milestone}-accepted'.format(release=release, milestone=re.sub('Final', '', opts.milestone)) # Connect to bugzilla bz = bugzilla.RHBugzilla3(url='https://bugzilla.redhat.com/xmlrpc.cgi') # If needed, Convert bug alias to number if not opts.blocker.isdigit(): opts.blocker = bz.getbug(opts.blocker).id if not opts.nth.isdigit(): opts.nth = bz.getbug(opts.nth).id # Track all bug lists in a single dictionary bugs_by_tracker = dict() # Get a list of accepted blocker bugs if opts.verbose: sys.stdout.write('Querying accepted blocker bugs ... ') q = {'bug_status': BUG_STATUS, 'value0-0-0': str(opts.blocker), 'type0-0-0': 'anywords', 'field0-0-0': 'blocked'} q.update(BLOCKER_ACCEPTED) bugs_by_tracker['accepted_blockers'] = flatten_bug_list(q) if opts.verbose: print len(bugs_by_tracker['accepted_blockers']) # Get a list of proposed blocker bugs if opts.verbose: sys.stdout.write('Querying proposed blocker bugs ... ') q = {'bug_status': BUG_STATUS, 'value0-0-0': str(opts.blocker), 'type0-0-0': 'anywords', 'field0-0-0': 'blocked'} q.update(BLOCKER_PROPOSED) bugs_by_tracker['proposed_blockers'] = flatten_bug_list(q) if opts.verbose: print len(bugs_by_tracker['proposed_blockers']) # Get a list of accepted NTH bugs if opts.verbose: sys.stdout.write('Querying accepted nice-to-have bugs ... ') q = {'bug_status': BUG_STATUS, 'value0-0-0': str(opts.nth), 'type0-0-0': 'anywords', 'field0-0-0': 'blocked'} q.update(NTH_ACCEPTED) bugs_by_tracker['accepted_nths'] = flatten_bug_list(q) if opts.verbose: print len(bugs_by_tracker['accepted_nths']) # Get a list of proposed NTH bugs if opts.verbose: sys.stdout.write('Querying proposed nice-to-have bugs ... ') q = {'bug_status': BUG_STATUS, 'value0-0-0': str(opts.nth), 'type0-0-0': 'anywords', 'field0-0-0': 'blocked'} q.update(NTH_PROPOSED) bugs_by_tracker['proposed_nths'] = flatten_bug_list(q) if opts.verbose: print len(bugs_by_tracker['proposed_nths']) # Organize bugs for later reference if opts.verbose: print 'Organizing bugs ...' bugs_by_id = dict() bugs_by_component = dict() for tracker, blocker_bugs in bugs_by_tracker.items(): bugs_by_component[tracker] = dict() for b in blocker_bugs: # Skip Tracking bugs if 'Tracking' in b.keywords: continue # Convert component(s) from list to string if isinstance(b.component, list): b.component = ','.join(b.component) # Organize by component if not bugs_by_component[tracker].has_key(b.component): bugs_by_component[tracker][b.component] = list() bugs_by_component[tracker][b.component].append(b.bug_id) # Remember this bug id in the master list of bugs bugs_by_id[b.bug_id] = b # Generate page content page_content = ''' ''This page generated automatically using [http://fedorapeople.org/gitweb?p=jlaska/public_git/scripts.git %s].'' ''' % (os.path.basename(sys.argv[0]),) # Display hotdog if requested, or if no blockers (approved or proposed) exist if opts.hotdog or \ (not bugs_by_component['accepted_blockers'] and not bugs_by_component['proposed_blockers']): page_content += '[[File:Hotdog.gif|right]]' # Otherwise, display release logo else: page_content += '''{{#ifexist: File:F{{FedoraVersionNumber|next}}_anaconda_center.png | [[File:F{{FedoraVersionNumber|next}}_anaconda_center.png|right]] | [[File:FedoraLogo_infinity.svg|right|150px]] }} ''' # Display approved blockers page_content += ''' == Approved Blockers == The following list of bugs are approved blockers that must be resolved. There {{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}} affecting %(component_count)s component{{plural:%(component_count)s||s}}. ''' % dict (bug_count=len(join_lists(bugs_by_component['accepted_blockers'].values())), component_count=len(bugs_by_component['accepted_blockers'])) # Sorted list of approved bugs page_content += wikitable_bugs(bugs_by_component['accepted_blockers'], bugs_by_id) # Display proposed blockers page_content += ''' == Proposed Blockers == The following list of bugs are not yet approved to block the release. There {{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}} affecting %(component_count)s component{{plural:%(component_count)s||s}}. For guidance on reviewing the following bugs, refer to [[QA:SOP_blocker_bug_process]]. ''' % dict (bug_count=len(join_lists(bugs_by_component['proposed_blockers'].values())), component_count=len(bugs_by_component['proposed_blockers'])) # Sorted list of proposed bugs page_content += wikitable_bugs(bugs_by_component['proposed_blockers'], bugs_by_id) # Display approved nths page_content += ''' == Approved NICE-TO-HAVE == The following list of of bugs are approved nice-to-have. Fixes for nice-to-have bugs will be accepted during the freeze. There {{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}} affecting %(component_count)s component{{plural:%(component_count)s||s}}. ''' % dict (bug_count=len(join_lists(bugs_by_component['accepted_nths'].values())), component_count=len(bugs_by_component['accepted_nths'])) # Sorted list of approved bugs page_content += wikitable_bugs(bugs_by_component['accepted_nths'], bugs_by_id) # Display proposed nths page_content += ''' == Proposed NICE-TO-HAVE == The following list of bugs are not yet approved nice-to-have issues. Only fixes for approved nice-to-have bugs will be accepted during the freeze. There {{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}} affecting %(component_count)s component{{plural:%(component_count)s||s}}. For guidance on reviewing the following bugs, refer to [[QA:SOP_nth_bug_process]]. ''' % dict (bug_count=len(join_lists(bugs_by_component['proposed_nths'].values())), component_count=len(bugs_by_component['proposed_nths'])) # Sorted list of proposed bugs page_content += wikitable_bugs(bugs_by_component['proposed_nths'], bugs_by_id) # Add a category so this page isn't lost on the wiki page_content += '\n[[Category:QA]]' # Login to the wiki if opts.user: if opts.verbose: print 'Logging into mediawiki ...' if not wiki.login(opts.user, opts.passwd): print "Error: invalid mediawiki username or password" sys.exit(1) else: if opts.verbose: print 'Using cookies in %s for authentication' % opts.cookiefile # Get an edit token q = dict(action='query', prop='info', intoken='edit', titles=opts.name) response = wiki.call(q) pages = [v for k,v in response.get('query', {}).get('pages',{}).items()] edit_token = list() if pages: edit_token = pages[0] # Make the wiki edit q = dict(action='edit', token=edit_token.get('edittoken',''), starttimestamp=edit_token.get('starttimestamp',''), summary='This scripted update brought to you by python-simplemediawiki and python-bugzilla.', text=to_encoding(u'%s' % page_content), title=opts.name) response = wiki.call(q) url = "https://fedoraproject.org/wiki/%s" % q.get('title') # Failure - Captcha if response.get('edit',{}).get('result','').lower() == 'failure': # Is a captcha required? captcha = response.get('edit', {}).get('captcha', False) if captcha: q.update(dict(captchaid=captcha.get('id'), captchaword=eval(captcha.get('question')))) response = wiki.call(q) # Error - Permission denied if response.has_key('error'): print "Failed to update '%s'\n%s" % (q.get('title'), response.get('error',{}).get('info','')) sys.exit(1) if response.get('result','').lower() == 'failure': print "Failed to update '%s'\n%s" % (q.get('title'), response) sys.exit(1) # Info - nochange elif response.get('edit',{}).has_key('nochange'): print "No changes, %s not updated" % (q.get('title'),) # Success elif response.get('edit',{}).has_key('newrevid'): print "Updated %s (revision: %s)" % (url, response.get('edit',{}).get('newrevid')) # Unknown else: print "Unknown result while updating %s\n%s" % (q.get('title'), response) sys.exit(0)