path: root/update-blocker-wiki

#!/usr/bin/python

import os
import sys
import locale
import optparse
import re
import copy
from getpass import getpass

try:
    import bugzilla
except ImportError:
    print "Unable to import bugzilla.  Is python-bugzilla installed?"
    sys.exit(1)

try:
    from simplemediawiki import MediaWiki
except ImportError:
    print "Unable to import simplemediawiki.  Is python-simplemediawiki installed?"
    sys.exit(1)

BUG_STATUS = ['NEW', 'ASSIGNED', 'ON_DEV', 'MODIFIED', 'POST', 'ON_QA', 'FAILS_QA',
              'PASSES_QA', 'REOPENED', 'VERIFIED', 'RELEASE_PENDING']

# Blocker query values
BLOCKER_ACCEPTED = {'status_whiteboard': 'AcceptedBlocker',
                    'status_whiteboard_type': 'anywords'}
BLOCKER_PROPOSED = {'status_whiteboard': 'AcceptedBlocker RejectedBlocker',
                    'status_whiteboard_type': 'nowords'}

# NTH query values
NTH_ACCEPTED = {'status_whiteboard': 'AcceptedNTH',
                'status_whiteboard_type': 'anywords'}
NTH_PROPOSED = {'status_whiteboard': 'AcceptedNTH RejectedNTH',
                'status_whiteboard_type': 'nowords'}

COOKIE_FILE = os.path.join(os.environ.get('HOME','/tmp'), '.fedora_cookiefile')

# UTF8 helper - "borrored" from python-bugzilla
def to_encoding(ustring):
    if isinstance(ustring, basestring):
        if isinstance(ustring, unicode):
            return ustring.encode(locale.getpreferredencoding(), 'replace')
        return ustring
    return u''

# Helper method to flatten a list of lists
def join_lists(l):
    '''
    Takes a list of lists, and joins them into a single list. For example:
    [[1,2,3],[4],[5,6,7]] will become [1,2,3,4,5,6,7]
    '''
    return [item for sublist in l for item in sublist]

# Display list of bugs, organized by components
def wikilist_bugs(bugs_by_component, bugs_by_id):
    buf = ''
    components = sorted(bugs_by_component.keys())
    for component in components:
        buf += '\n=== %s ===\n' % component
        # sorted list
        bugs = sorted(bugs_by_component.get(component,[]))
        for b in bugs:
            b = bugs_by_id[b]
            buf += '* [https://bugzilla.redhat.com/show_bug.cgi?id=%s %s] (%s) - %s\n' % (b.bug_id, b.bug_id, b.bug_status, b.short_desc)
    return buf

# Display sortable table of bugs
def wikitable_bugs(bugs_by_component, bugs_by_id):
    buf = ''

    components = sorted(bugs_by_component.keys())
    if len(components) > 0:
        buf = '''{| class="wikitable sortable"
! Bug !! Component !! Status !! Description !! class="unsortable"|Links
|- \n'''

        for component in components:
            # sorted list
            bugs = sorted(bugs_by_component.get(component,[]))
            for b in bugs:
                b = bugs_by_id[b]
                buf += '| [http://bugzilla.redhat.com/show_bug.cgi?id=%s %s] || %s ' % (b.bug_id, b.bug_id, component)
                buf += '|| %s || %s || ' % (b.bug_status, b.short_desc,)
                buf += '[[Image:Infra-icon_koji.png|link=http://koji.fedoraproject.org/koji/search?type=package&match=glob&terms=%s|alt=Build Status]] ' % (component)
                buf += '[[Image:Infra-icon_bodhi.png|link=http://admin.fedoraproject.org/updates/%s|alt=Update Status]] ' % (component)
                buf += '[[Image:Infra-icon_cvssource.png|link=http://pkgs.fedoraproject.org/gitweb/?p=%s.git|alt=Package Source]] ' % (component)
                buf += '[[Image:Infra-icon_bugzilla.png|link=http://admin.fedoraproject.org/pkgdb/acls/bugs/%s|alt=Bug Reports]] ' % (component)
                buf += '\n|-\n' # End table row
        buf += '|}\n'
    return buf

def parse_args():
    '''Set up the option parser'''
    parser = optparse.OptionParser(usage="%prog [options]")
    parser.add_option('-v', '--verbose', action='store_true', default=False,
        help='Enable verbosity')
    parser.add_option('--hotdog', action='store_true', default=False,
        help='Enable hot dog (default: %default)')
    parser.add_option('--cookiefile', action='store', default=COOKIE_FILE,
        help='Cookiejar path to store wiki login cookies (default: %default)')

    optgrp = optparse.OptionGroup(parser, "Required options")
    optgrp.add_option('-n', '--name', action='store', default=None,
        help='Wiki page name to save results (required)')
    optgrp.add_option('--blocker', action='store', default='auto',
        help='Blocker tracking bug number (default: %default)')
    optgrp.add_option('--nth', action='store', default='auto',
        help='Nice-to-Have tracking bug number (default: %default)')
    optgrp.add_option('-m', '--milestone', type='choice', action='store',
        default=None, choices=['Alpha', 'Beta', 'Final',],
        help='Release milestone used when --blocker or --nth are set to \'auto\' (e.g. Alpha, Beta or Final)')
    optgrp.add_option('-u', '--user', action='store', default=None,
        help='Mediawiki username')
    optgrp.add_option('-p', '--passwd', action='store', default=None,
        help='Mediawiki password')
    parser.add_option_group(optgrp)

    (opts, args) = parser.parse_args()

    # sanitize helper
    def sanitize_input(parser, value, label, ispass=False):
        if value is None and sys.stdin.isatty():
            prompt = 'Enter %s: ' % label
            if ispass:
                value = getpass(prompt)
            else:
                value = raw_input(prompt)
        if value is None:
            parser.error('Must provide a valid %s' % label)
        return value

    def valid_cookiefile(parser, cookiefile):
        # if it doesn't exist, it can't be valid
        if not os.path.exists(cookiefile):
            return False

        # Check if it contains valid data
        fd = open(cookiefile, 'r')
        buf = fd.read()
        fd.close()
        valid = False
        for line in buf.split('\n'):
            # Skip empty lines or lines with only comments
            if re.match(r'^(\s*#|\s*$)', line):
                continue
            return True

        # Otherwise, no valid content found
        return False

    if not valid_cookiefile(parser, opts.cookiefile):
        opts.user = sanitize_input(parser, opts.user, "username")
        opts.passwd = sanitize_input(parser, opts.passwd, "password", ispass=True)

    opts.blocker = sanitize_input(parser, opts.blocker, "Blocker bug number")
    opts.nth = sanitize_input(parser, opts.nth, "Nice-to-have bug number")
    opts.name = sanitize_input(parser, opts.name, "Wiki page name")
    opts.milestone = sanitize_input(parser, opts.milestone, "Release milestone")

    return opts

def flatten_bug_list(query_args, max_depth=4):
    buglist = bz.query(query_args)

    # avoid endless recursion
    if max_depth <= 0:
        return buglist

    # find any additional dependencies
    new_deps = list()
    for b in buglist:
        if len(b.dependson) > 0:
            new_deps.append(b.bug_id)

    # Remove any duplicates
    existing_deps = {int(bug):True for bug in query_args['value0-0-0'].split()}.keys()
    new_deps = {bug:True for bug in new_deps}.keys()

    # Ignore anything already being tracked
    copy_new_deps = copy.deepcopy(new_deps)
    for bug in copy_new_deps:
        if bug in existing_deps:
            new_deps.remove(bug)

    if len(new_deps) > 0:
        query_args['value0-0-0'] = ' '.join([str(bug) for bug in existing_deps]) + ' ' + ' '.join([str(bug) for bug in new_deps])
        if opts.verbose: print '   additional dependencies: %s' % ' '.join([str(bug) for bug in new_deps])
        return flatten_bug_list(query_args, max_depth-1)

    return buglist

if __name__ == '__main__':

    opts = parse_args()

    # Create mediawiki handle
    if opts.verbose: print 'Connecting to mediawiki ...'
    wiki = MediaWiki('https://fedoraproject.org/w/api.php', cookie_file=opts.cookiefile)

    # Get current Fedora version number
    if opts.blocker == 'auto' or opts.nth == 'auto':
        if opts.verbose: sys.stdout.write('Detecting upcoming Fedora release ... ')
        q = dict(action='expandtemplates',
                 text='{{Template:FedoraVersionNumber|next}}',)
        response = wiki.call(q)
        release = response.get('expandtemplates',{}).get('*','')
        if opts.verbose: print release
        if release.isdigit():
            if opts.blocker == 'auto':
                # HACK - the blocker bug format is inconsistently named
                opts.blocker = 'F{release}{milestone}'.format(release=release,
                    milestone=re.sub('Final', 'Blocker', opts.milestone))
            if opts.nth == 'auto':
                # HACK - the blocker bug format is inconsistently named
                opts.nth = 'F{release}{milestone}-accepted'.format(release=release,
                    milestone=re.sub('Final', '', opts.milestone))

    # Connect to bugzilla
    bz = bugzilla.RHBugzilla3(url='https://bugzilla.redhat.com/xmlrpc.cgi')

    # If needed, Convert bug alias to number
    if not opts.blocker.isdigit():
        opts.blocker = bz.getbug(opts.blocker).id
    if not opts.nth.isdigit():
        opts.nth = bz.getbug(opts.nth).id

    # Track all bug lists in a single dictionary
    bugs_by_tracker = dict()

    # Get a list of accepted blocker bugs
    if opts.verbose: sys.stdout.write('Querying accepted blocker bugs ... ')
    q = {'bug_status': BUG_STATUS,
         'value0-0-0': str(opts.blocker),
         'type0-0-0': 'anywords',
         'field0-0-0': 'blocked'}
    q.update(BLOCKER_ACCEPTED)
    bugs_by_tracker['accepted_blockers'] = flatten_bug_list(q)
    if opts.verbose: print len(bugs_by_tracker['accepted_blockers'])

    # Get a list of proposed blocker bugs
    if opts.verbose: sys.stdout.write('Querying proposed blocker bugs ... ')
    q = {'bug_status': BUG_STATUS,
         'value0-0-0': str(opts.blocker),
         'type0-0-0': 'anywords',
         'field0-0-0': 'blocked'}
    q.update(BLOCKER_PROPOSED)
    bugs_by_tracker['proposed_blockers'] = flatten_bug_list(q)
    if opts.verbose: print len(bugs_by_tracker['proposed_blockers'])

    # Get a list of accepted NTH bugs
    if opts.verbose: sys.stdout.write('Querying accepted nice-to-have bugs ... ')
    q = {'bug_status': BUG_STATUS,
         'value0-0-0': str(opts.nth),
         'type0-0-0': 'anywords',
         'field0-0-0': 'blocked'}
    q.update(NTH_ACCEPTED)
    bugs_by_tracker['accepted_nths'] = flatten_bug_list(q)
    if opts.verbose: print len(bugs_by_tracker['accepted_nths'])

    # Get a list of proposed NTH bugs
    if opts.verbose: sys.stdout.write('Querying proposed nice-to-have bugs ... ')
    q = {'bug_status': BUG_STATUS,
         'value0-0-0': str(opts.nth),
         'type0-0-0': 'anywords',
         'field0-0-0': 'blocked'}
    q.update(NTH_PROPOSED)
    bugs_by_tracker['proposed_nths'] = flatten_bug_list(q)
    if opts.verbose: print len(bugs_by_tracker['proposed_nths'])

    # Organize bugs for later reference
    if opts.verbose: print 'Organizing bugs ...'
    bugs_by_id = dict()

    bugs_by_component = dict()
    for tracker, blocker_bugs in bugs_by_tracker.items():
        bugs_by_component[tracker] = dict()
        for b in blocker_bugs:
            # Skip Tracking bugs
            if 'Tracking' in b.keywords:
                continue
            # Convert component(s) from list to string
            if isinstance(b.component, list):
                b.component = ','.join(b.component)
            # Organize by component
            if not bugs_by_component[tracker].has_key(b.component):
                bugs_by_component[tracker][b.component] = list()
            bugs_by_component[tracker][b.component].append(b.bug_id)
            # Remember this bug id in the master list of bugs
            bugs_by_id[b.bug_id] = b

    # Generate page content
    page_content = '''

''This page generated automatically using [http://fedorapeople.org/gitweb?p=jlaska/public_git/scripts.git %s].''

''' % (os.path.basename(sys.argv[0]),)

    # Display hotdog if requested, or if no blockers (approved or proposed) exist
    if opts.hotdog or \
       (not bugs_by_component['accepted_blockers'] and not bugs_by_component['proposed_blockers']):
        page_content += '[[File:Hotdog.gif|right]]'
    # Otherwise, display release logo
    else:
        page_content += '''{{#ifexist: File:F{{FedoraVersionNumber|next}}_anaconda_center.png
| [[File:F{{FedoraVersionNumber|next}}_anaconda_center.png|right]]
| [[File:FedoraLogo_infinity.svg|right|150px]] }}
'''

    # Display approved blockers
    page_content += '''
== Approved Blockers ==
The following list of bugs are approved blockers that must be resolved.  There
{{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}}
affecting %(component_count)s component{{plural:%(component_count)s||s}}.
''' % dict (bug_count=len(join_lists(bugs_by_component['accepted_blockers'].values())),
            component_count=len(bugs_by_component['accepted_blockers']))
    # Sorted list of approved bugs
    page_content += wikitable_bugs(bugs_by_component['accepted_blockers'], bugs_by_id)

    # Display proposed blockers
    page_content += '''
== Proposed Blockers ==
The following list of bugs are not yet approved to block the release.  There
{{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}}
affecting %(component_count)s component{{plural:%(component_count)s||s}}.  For
guidance on reviewing the following bugs, refer to
[[QA:SOP_blocker_bug_process]].
''' % dict (bug_count=len(join_lists(bugs_by_component['proposed_blockers'].values())),
            component_count=len(bugs_by_component['proposed_blockers']))
    # Sorted list of proposed bugs
    page_content += wikitable_bugs(bugs_by_component['proposed_blockers'], bugs_by_id)

    # Display approved nths
    page_content += '''
== Approved NICE-TO-HAVE ==
The following list of of bugs are approved nice-to-have.  Fixes for
nice-to-have bugs will be accepted during the freeze.  There
{{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}}
affecting %(component_count)s component{{plural:%(component_count)s||s}}.
''' % dict (bug_count=len(join_lists(bugs_by_component['accepted_nths'].values())),
            component_count=len(bugs_by_component['accepted_nths']))

    # Sorted list of approved bugs
    page_content += wikitable_bugs(bugs_by_component['accepted_nths'], bugs_by_id)

    # Display proposed nths
    page_content += '''
== Proposed NICE-TO-HAVE ==
The following list of bugs are not yet approved nice-to-have issues.  Only
fixes for approved nice-to-have bugs will be accepted during the freeze.  There
{{plural:%(bug_count)s|is|are}} %(bug_count)s bug{{plural:%(bug_count)s||s}}
affecting %(component_count)s component{{plural:%(component_count)s||s}}.  For
guidance on reviewing the following bugs, refer to [[QA:SOP_nth_bug_process]].
''' % dict (bug_count=len(join_lists(bugs_by_component['proposed_nths'].values())),
            component_count=len(bugs_by_component['proposed_nths']))

    # Sorted list of proposed bugs
    page_content += wikitable_bugs(bugs_by_component['proposed_nths'], bugs_by_id)

    # Add a category so this page isn't lost on the wiki
    page_content += '\n[[Category:QA]]'

    # Login to the wiki
    if opts.user:
        if opts.verbose: print 'Logging into mediawiki ...'
        if not wiki.login(opts.user, opts.passwd):
            print "Error: invalid mediawiki username or password"
            sys.exit(1)
    else:
        if opts.verbose: print 'Using cookies in %s for authentication' % opts.cookiefile

    # Get an edit token
    q = dict(action='query',
             prop='info',
             intoken='edit',
             titles=opts.name)
    response = wiki.call(q)
    pages = [v for k,v in response.get('query', {}).get('pages',{}).items()]
    edit_token = list()
    if pages:
        edit_token = pages[0]

    # Make the wiki edit
    q = dict(action='edit',
             token=edit_token.get('edittoken',''),
             starttimestamp=edit_token.get('starttimestamp',''),
             summary='This scripted update brought to you by python-simplemediawiki and python-bugzilla.',
             text=to_encoding(u'%s' % page_content),
             title=opts.name)
    response = wiki.call(q)

    url = "https://fedoraproject.org/wiki/%s" % q.get('title')
    # Failure - Captcha
    if response.get('edit',{}).get('result','').lower() == 'failure':
        # Is a captcha required?
        captcha = response.get('edit', {}).get('captcha', False)
        if captcha:
            q.update(dict(captchaid=captcha.get('id'),
                          captchaword=eval(captcha.get('question'))))
            response = wiki.call(q)

    # Error - Permission denied
    if response.has_key('error'):
        print "Failed to update '%s'\n%s" % (q.get('title'), response.get('error',{}).get('info',''))
        sys.exit(1)
    if response.get('result','').lower() == 'failure':
        print "Failed to update '%s'\n%s" % (q.get('title'), response)
        sys.exit(1)

    # Info - nochange
    elif response.get('edit',{}).has_key('nochange'):
        print "No changes, %s not updated" % (q.get('title'),)
    # Success
    elif response.get('edit',{}).has_key('newrevid'):
        print "Updated %s (revision: %s)" % (url,
            response.get('edit',{}).get('newrevid'))
    # Unknown
    else:
        print "Unknown result while updating %s\n%s" % (q.get('title'), response)

    sys.exit(0)