Add repoclosure-bz script that auto-files repoclosure bugs

1 files changed, 407 insertions, 0 deletions

diff --git a/repoclosure-bz b/repoclosure-bz
new file mode 100755
index 0000000..2b50441
--- /dev/null
+++ b/repoclosure-bz
@@ -0,0 +1,407 @@
+#!/usr/bin/python
+
+import os
+import sys
+import re
+import subprocess
+import logging
+import hashlib
+import optparse
+import urlparse
+import urlgrabber
+import rpmUtils.miscutils
+import bugzilla
+
+# Initial simple logging stuff
+logging.basicConfig()
+log = logging.getLogger("bugzilla")
+if '--debug' in sys.argv:
+    log.setLevel(logging.DEBUG)
+elif '--verbose' in sys.argv:
+    log.setLevel(logging.INFO)
+
+default_bz = 'https://bugzilla.redhat.com/xmlrpc.cgi'
+bzclass = bugzilla.RHBugzilla3
+#bzclass = bugzilla.Bugzilla3
+bz = bzclass(url=default_bz)
+
+# Extract and return a package name when supplied a string in the format of
+# %{name}-%{version}-%{release}
+def get_package_name(envra):
+    (name, version, release, epoch, arch) = rpmUtils.miscutils.splitFilename(envra)
+    return name
+
+# Given a list of strings, return a list of hashes corresponding to the strings
+def hash_data(buffer):
+    # Sort the buffer
+    buffer.sort()
+
+    # convert to string
+    buffer = ' '.join(buffer)
+
+    # Hash and save for later
+    return "repoclosure_hash:%s" % hashlib.sha256(buffer).hexdigest()
+
+# Given a string, return a list of hashes that characterize the repoclosure
+# failures
+def generate_hash_list(lines):
+    buffer = list()
+    hashes = dict() # use a dictionary to avoid duplicates
+    name = ""
+
+    for line in lines:
+        # If this is a 'package:' line, ignore it
+        if line.startswith("package:"):
+            # Determine binary package name for use with hashing
+            envra = re.search("^package: ([^ ]*) from.*", line).group(1)
+            assert isinstance(envra, str) and envra != ""
+            name = get_package_name(envra)
+            continue
+        # If this is a 'unresolved deps:' line, process it
+        elif "unresolved deps:" in line:
+            # If existing information exists, hash it
+            if len(buffer) > 0:
+                hashes[hash_data(buffer)] = True
+            # Clear the buffer, using binary package name
+            assert isinstance(name, str) and name != ""
+            buffer = [name]
+        # Otherwise, continue updating buffer
+        else:
+            # Remove leading+trailing whitespace
+            line = line.strip()
+
+            # Remove any arch-specific data: "()(64bit)"
+            line = line.replace("()(64bit)","")
+
+            # Update the buffer with data
+            assert isinstance(buffer, list)
+            buffer.append(line)
+
+    # If any buffered data remains, hash it
+    if len(buffer) > 0:
+        hashes[hash_data(buffer)] = True
+        buffer = list()
+
+    return hashes.keys()
+
+# Using a supplied product, version and list of hashes return a list of bugs
+# matching any of the supplied hashes
+def bz_find_matching_hashes(bz_product, bz_version, hashes):
+    # Force a list
+    if isinstance(hashes, str):
+        hashes = [hashes]
+
+    # Build the query parameters
+    q = dict()
+    q['product'] = bz_product
+    q['version'] = bz_version
+    q['status_whiteboard'] = ' '.join(hashes)
+    q['status_whiteboard_type'] = 'anywords'
+
+    q['column_list'] = [ 'bug_id', 'bug_status', 'assigned_to', 'component',
+        'short_desc', 'keywords', 'blockedby' ]
+    log.debug("bz.query: %s", q)
+    buglist = bz.query(q)
+
+    return buglist
+
+# File a bug in bugzilla using the provided information
+def bz_file_bug(product, version, component, summary, hashes, description, blocked=[]):
+    data = dict(product=product, version=version, component=component,
+            short_desc=summary, comment=description,
+            rep_platform='All', bug_severity='medium', priority='medium',
+            op_sys='Linux', bug_file_loc='',
+            status_whiteboard = ' '.join(hashes))
+    if len(blocked) > 0:
+        data['blocked'] = map(lambda b: int(b), blocked) # force as a list of int's
+    b = bz.createbug(**data)
+    b.refresh()
+    return b
+
+# Add the supplied 'blocks' information to the given bug_id
+def bz_add_to_blocker(bug, blocks):
+    # Ensure arguments are of the expected type
+    assert isinstance(bug, bugzilla.base._Bug)
+    # Convert to a list
+    if isinstance(blocks, int):
+        blocks = [blocks]
+    # If no data was provided, simply return
+    if len(blocks) == 0:
+        return
+
+    # Get list of old blocks
+    old_blocks = bug.blocked
+    log.debug("Updating bug#%s blocks - from '%s'" % (bug.bug_id, old_blocks))
+
+    # Create a dict of the current list of blocks
+    blocks_dict = {int(b): True for b in old_blocks}
+
+    # Update blocks_dict with new hashes
+    for b in blocks:
+        if not isinstance(b, int):
+            b = int(b)
+        if not blocks_dict.has_key(b):
+            print "\t- adding to blocks: %s" % b
+            blocks_dict[b] = True
+
+    # Convert back to a list
+    new_blocks = blocks_dict.keys()
+
+    # Update the bug
+    log.debug("Updating bug#%s blocks - to '%s'" % (bug.bug_id, new_blocks))
+    bz._updatedeps(bug.bug_id, new_blocks, [], 'add')
+
+# Update the 'status_whiteboard' with new hashes
+def bz_update_whiteboard(bug, hashes):
+    # Ensure arguments are of the expected type
+    assert isinstance(bug, bugzilla.base._Bug)
+    # Force hash list
+    if isinstance(hashes, str):
+        hashes = [hashes]
+    # If no data was provided, simply return
+    if len(hashes) == 0:
+        return
+
+    old_whiteboard = bug.getwhiteboard()
+    # Strip leading+trailing whitespace
+    old_whiteboard = old_whiteboard.strip()
+    log.debug("Updating bug#%s whiteboard - from '%s'" % (bug.bug_id, old_whiteboard))
+
+    # Create a dict of current whiteboard
+    whiteboard_dict = {k: True for k in old_whiteboard.split(' ')}
+
+    # Update whiteboard_dict with new hashes
+    for hash in hashes:
+        if not whiteboard_dict.has_key(hash):
+            print "\t- adding to whiteboard: %s" % hash
+            whiteboard_dict[hash] = True
+
+    # flatten to a string
+    new_whiteboard = ' '.join(whiteboard_dict.keys())
+    # Remove leading+trailing whitespace
+    new_whiteboard = new_whiteboard.strip()
+
+    # update whiteboard to include any new hashes
+    log.debug("Updating bug#%s whiteboard - to '%s'" % (bug.bug_id, new_whiteboard))
+    bug.setwhiteboard(new_whiteboard)
+
+def get_basearch():
+    '''like get_arch, but returns the basearch (as used by yum etc.)'''
+    arch = os.uname()[4]
+    if arch in ('i486', 'i586', 'i686'):
+        arch = 'i386'
+    elif arch == 'ppc64':
+        arch = 'ppc'
+    return arch
+
+# Parse arguments and return options
+def parse_args():
+    parser = optparse.OptionParser()
+    parser.add_option('--verbose', action='store_true',
+        help="give more info about what's going on")
+    parser.add_option('--debug', action='store_true',
+        help="output bunches of debugging info")
+    parser.add_option("-y", "--assumeyes", dest="assumeyes", default=False,
+        action="store_true", help="Answer yes for all questions")
+
+    # Repoclosure options
+    optgrp = optparse.OptionGroup(parser, "Repoclosure options")
+    archlist = ['i386', 'x86_64', 'ppc']
+    optgrp.add_option("-a", "--arch", action="store", type="choice",
+        choices=archlist, default=get_basearch(),
+        help="target architecture (%default)")
+    optgrp.add_option("-l", "--logurl", action="store", default="",
+        help="URL for existing repoclosure results.  When provided, it will" + \
+             "just process existing results and not re-run repoclosure (useful" + \
+             "when running multiple times)")
+    optgrp.add_option('-r', '--repo', action='append', default=[], dest="repos",
+        help="Repository URL; can be used multiple times (default: use system " + \
+             "repos)")
+    parser.add_option_group(optgrp)
+
+    # Bugzilla options
+    optgrp = optparse.OptionGroup(parser, "Bugzilla options")
+    optgrp.add_option('-b', '--blocks', action='append',
+        default=[], help="Set blocks field when filing bugs")
+    optgrp.add_option('--product', dest='bz_product', action='store',
+        default='Fedora', help='Bugzilla product (default: %default)')
+    optgrp.add_option('--version', dest='bz_version', action='store',
+        default='rawhide', help='Bugzilla version (default: %default)')
+    parser.add_option_group(optgrp)
+
+    (opts, args) = parser.parse_args()
+
+    if len(opts.repos) == 0 and opts.logurl == "":
+        parser.error("Must provide either one -r|--repo or -l|--logurl")
+        sys.exit(1)
+
+    return (opts, args)
+
+# Validate that specific applications are available on this system
+def system_sanity():
+    '''Check the system to be sure the required binaries/services are there'''
+    try:
+        cmds = [ "repoclosure --help", \
+                 "repoquery --help", \
+               ]
+        for cmd in cmds:
+            msg = "'%s' failed. Be sure %s is installed." % (cmd, cmd.split(' ')[0])
+            assert subprocess.call(cmd.split(' '), stdout=subprocess.PIPE) == 0
+        return True
+    except (IOError, OSError, AssertionError):
+        print msg
+        return False
+
+if __name__ == "__main__":
+
+    # Parse arguments
+    (opts, args) = parse_args()
+
+    # Validate environment
+    if not system_sanity():
+        sys.exit(1)
+
+    # Attempt to authenticate to bugzilla
+    if os.path.exists(bz.cookiefile):
+        log.info('Using cookies in %s for bugzilla authentication', bz.cookiefile)
+    else:
+        # FIXME check to see if .bugzillarc is in use
+        log.info("No authentication info provided.  Have you run `bugzilla login`?")
+
+    # Build shared repoclosure and repoquery argument string
+    repo_opts = "--tempcache "
+    count = 1
+    for url in opts.repos:
+        repo_opts += " --repofrompath=repo-%s,%s --repoid=repo-%s" % (count,
+            url, count)
+        count += 1
+
+    # Has repoclosure already been run?
+    if opts.logurl:
+        try:
+            data = urlgrabber.urlread(opts.logurl)
+        except Exception, e:
+            log.error("Failed to read '%s'" % opts.logurl)
+            log.debug(e)
+            sys.exit(1)
+
+    else:
+        # Run repoclosure
+        print "Running repoclosure (this may take a few minutes) ..."
+        cmd = 'repoclosure --newest %s' % (repo_opts,)
+        logging.debug(cmd)
+        data = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT).communicate()[0].strip()
+        print data
+    # Output the results for debugging
+    log.debug(data)
+
+    # Parse results
+    print "Parsing results ..."
+    problems = dict()
+    package = ""
+    src = ""
+    for line in data.split('\n'):
+
+        log.debug("Parsing line '%s'" % line)
+        if line.startswith("package:"):
+            package = re.search("^package: ([^ ]*) from.*", line).group(1)
+
+            # Keep track of problems by src-rpm name (not binary)
+            cmd = "repoquery -q --qf %%{sourcerpm},%%{PACKAGER} %s %s" % (repo_opts, package)
+            result = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE,
+                stderr=open('/dev/null', 'w')).communicate()[0].strip()
+            (src, packager) = result.split(',',1)
+
+            # Ignore anything that isn't packaged by Fedora (e.g. rpmfusion)
+            if packager != "Fedora Project":
+                log.warn("Unsupported packager (%s), skipping '%s'" % (packager, package))
+                src = ""
+                continue
+
+            # Strip off '.src.rpm'
+            if src.endswith(".src.rpm"):
+                src = src[:-8]
+
+            # Log result
+            if src == "":
+                log.warn("Unable to locate src.rpm for '%s'" % package)
+                continue
+            else:
+                log.info("Identified failure with package '%s', src '%s'" % \
+                    (package, src))
+
+            # Update dictionary of src names
+            if not problems.has_key(src):
+                problems[src] = dict(output=list(), packages=[package])
+            else:
+                problems[src]['packages'].append(package)
+
+        # always save the output
+        if src != "":
+            problems[src]['output'].append(line)
+
+    # Print summary
+    print "%d dependency problems affecting %s source packages" % \
+        (data.count("unresolved deps:"), len(problems.keys()))
+
+    # Hash the dependency data
+    for (src,problem) in problems.items():
+
+        output = problem['output']
+        # Generate a list of unique hashes for each problem in the output
+        # string
+        hashes = generate_hash_list(output)
+        problem['hashes'] = hashes
+
+        # Strip off the %{version}-%{release}, leaving just %{name}
+        component = get_package_name(src)
+
+        # Create a description that consists of the error, and stdout leading up to
+        # it (not full repoclosure stdout)
+        description = data[:data.index("package:")] + "\n".join(output)
+
+        # Attempt to make a meaningful summary
+        # XXX - this relies on pattern matching repoclosure output ... likely
+        # to break
+        match = re.search('package: ([^ ]*) from [^ ]*\n[ ]*unresolved deps:[ ]*\n(.*)', '\n'.join(output), re.MULTILINE)
+        if match:
+            summary = "Broken dependency: %s requires %s" % (match.group(1).strip(), match.group(2).strip())
+        else:
+            summary = "Broken dependencies for %s" % src
+
+        # Print summary
+        print "\n== %s ==" % src
+        print "Summary: %s" % summary
+        print "Product: %s" % opts.bz_product
+        print "Version: %s" % opts.bz_version
+        print "Component: %s" % component
+        print "Whiteboard:\n\t%s" % "\n\t".join(hashes)
+        print "Binaries affected:\n\t%s" % "\n\t".join(problem['packages'])
+        print "Description:\n%s\n" % description
+
+        # Attempt to find an existing bug matching any of the hashes
+        matching_bugs = bz_find_matching_hashes(opts.bz_product, opts.bz_version, hashes)
+        if len(matching_bugs) > 0:
+            for b in matching_bugs:
+                print "Found existing bug: http://%s/%s (%s)" % (urlparse.urlparse(default_bz).netloc, str(b.bug_id), b.bug_status)
+                # If --blocks were provided, add them now
+                if opts.blocks:
+                    bz_add_to_blocker(b, opts.blocks)
+                # Update whiteboard with any new hashes
+                bz_update_whiteboard(b, hashes)
+
+        # No matching bug was found, let's create one
+        else:
+            yesno = opts.assumeyes
+            # If we are at a terminal, and weren't told to assume 'yes'
+            if not yesno and sys.stdin.isatty():
+                yesno = raw_input("File a new bug (y|n): ").lower() == 'y'
+            if yesno:
+                # File a new bug
+                b = bz_file_bug(product=opts.product, version=opts.version,
+                    component=component, summary=summary, hashes=hashes,
+                    description=description, blocked=opts.blocks)
+                print "Filed new bug: http://%s/%s" % (urlparse.urlparse(default_bz).netloc, str(b.bug_id))
+            else:
+                log.warn("Not filing a bug at user request")