From 23d5631ae8a82fc6c474fc27afac1c4c428f3d4f Mon Sep 17 00:00:00 2001 From: Karel Klic Date: Thu, 14 Oct 2010 15:38:12 +0200 Subject: btparser initial integration --- scripts/abrt-bz-downloader | 82 ------------ scripts/abrt-bz-dupchecker | 281 -------------------------------------- scripts/abrt-bz-hashchecker | 59 -------- scripts/abrt-bz-ratingfixer | 210 ----------------------------- scripts/abrt-bz-stats | 313 ------------------------------------------- scripts/check-bt-parsability | 20 --- 6 files changed, 965 deletions(-) delete mode 100755 scripts/abrt-bz-downloader delete mode 100755 scripts/abrt-bz-dupchecker delete mode 100755 scripts/abrt-bz-hashchecker delete mode 100755 scripts/abrt-bz-ratingfixer delete mode 100755 scripts/abrt-bz-stats delete mode 100755 scripts/check-bt-parsability (limited to 'scripts') diff --git a/scripts/abrt-bz-downloader b/scripts/abrt-bz-downloader deleted file mode 100755 index d7f5a719..00000000 --- a/scripts/abrt-bz-downloader +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python -# -*- mode:python -*- -# ABRT Bugzilla Backtrace Downloader -# Downloads all backtraces reported by ABRT from Bugzilla. -# -# Please do not run this script unless it's neccessary to do so. -# It forces Bugzilla to send data related to thousands of bug reports. - -from bugzilla import RHBugzilla -from optparse import OptionParser -import sys -import os.path - -parser = OptionParser(version="%prog 1.0") -parser.add_option("-u", "--user", dest="user", - help="Bugzilla user name (REQUIRED)", metavar="USERNAME") -parser.add_option("-p", "--password", dest="password", - help="Bugzilla password (REQUIRED)", metavar="PASSWORD") -parser.add_option("-b", "--bugzilla", dest="bugzilla", - help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") -parser.add_option("-f", "--fields", - action="store_true", dest="fields", default=False, - help="Print possible bug fields and exit.") - -(options, args) = parser.parse_args() - -if not options.user or len(options.user) == 0: - parser.error("User name is required.\nTry {0} --help".format(sys.argv[0])) - -if not options.password or len(options.password) == 0: - parser.error("Password is required.\nTry {0} --help".format(sys.argv[0])) - -if not options.bugzilla or len(options.bugzilla) == 0: - options.bugzilla = "https://bugzilla.redhat.com/xmlrpc.cgi" - -bz = RHBugzilla() -bz.connect(options.bugzilla) -bz.login(options.user, options.password) - -if options.fields: - print bz.bugfields - exit(0) - -buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash'}) - -print "{0} bugs found.".format(len(buginfos)) - -for buginfo in buginfos: - # Skip bugs with already downloaded backtraces. - filename = "{0}.bt".format(buginfo.bug_id) - if os.path.isfile(filename): - print "Skipping {0} (already exists).".format(filename) - continue - - # Skip bugs with broken or Python backtraces - broken_backtrace_bugs = [ 517116, # binary file :) - 518516, # not a backtrace, GDB fail - 524259, # multiple backtraces in single file - 524427, # multiple backtraces in single file - 528529, # just [New Thread xx] lines - #528915, 10000 frames, out of memory, to be fixed - #529422, 10000 frames, out of memory, to be fixed - #530239, 10000 frames, out of memory, to be fixed - 532264, # no header - 533475, # no backtrace - #537819, 50000 frames, out of memory, to be fixed - #539699, to be fixed, parser bug - 539992] # completely broken backtrace - if buginfo.bug_id in broken_backtrace_bugs: - continue - - # Get backtrace from bug and store it as a file. - bug = bz.getbug(buginfo.bug_id) - for attachment in bug.attachments: - if attachment['filename'] == 'backtrace': - data = bz.openattachment(attachment['id']) - f = open(filename, 'w') - f.write(data.read()) - f.close() - print "Attachment {0} downloaded.".format(filename) - -bz.logout() diff --git a/scripts/abrt-bz-dupchecker b/scripts/abrt-bz-dupchecker deleted file mode 100755 index 65e11531..00000000 --- a/scripts/abrt-bz-dupchecker +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/python -# -*- mode:python -*- -# ABRT Bugzilla Duplication Checker -# Downloads all backtraces reported by ABRT from Bugzilla, -# and search for duplicates using the newest ABRT duplication -# checker. -# -# Some bugs in Bugzilla were reported by older ABRT -# versions, which had poor duplication detection. -# -# Please do not run this script unless it's neccessary to do so. -# It forces Bugzilla to send data related to thousands of bug reports. -# -# -# Useful text to be pasted to Bugzilla: -""" -This bug appears to have been filled using a buggy version of ABRT, because -it contains unusable backtrace. Sorry for the inconvenience. -Closing as INSUFFICIENT_DATA. -""" - -from bugzilla import RHBugzilla -from optparse import OptionParser -import sys -import os.path -import subprocess -import cPickle -import urllib -import json - -parser = OptionParser(version="%prog 1.0") -parser.add_option("-u", "--user", dest="user", - help="Bugzilla user name (REQUIRED)", metavar="USERNAME") -parser.add_option("-p", "--password", dest="password", - help="Bugzilla password (REQUIRED)", metavar="PASSWORD") -parser.add_option("-b", "--bugzilla", dest="bugzilla", default="https://bugzilla.redhat.com/xmlrpc.cgi", - help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") -parser.add_option("-v", "--verbose", dest="verbose", - help="Detailed output") -parser.add_option("-c", "--close", help="Close some of the bugs in Bugzilla (DANGEROUS)", - action="store_true", default=False, dest="close") -parser.add_option("-i", "--wiki", help="Generate output in wiki syntax", - action="store_true", default=False, dest="wiki") - -(options, args) = parser.parse_args() - -if not options.user or len(options.user) == 0: - parser.error("User name is required.\nTry {0} --help".format(sys.argv[0])) - -if not options.password or len(options.password) == 0: - parser.error("Password is required.\nTry {0} --help".format(sys.argv[0])) - -bz = RHBugzilla() -bz.connect(options.bugzilla) -bz.login(options.user, options.password) - -buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash', 'product':'Fedora'}) - -print "{0} bugs found.".format(len(buginfos)) - -# -# Load cache from previous run. Speeds up the case Bugzilla closes connection. -# The cache should be manually removed after a day or so, because the data in it -# are no longer valid. -# -database = {} -ids = {} -CACHE_FILE = "abrt-bz-dupchecker-cache.tmp" -if os.path.isfile(CACHE_FILE): - f = open(CACHE_FILE, 'r') - database = cPickle.load(f) - ids = cPickle.load(f) - f.close() - -def save_to_cache(): - global database - f = open(CACHE_FILE, 'w') - cPickle.dump(database, f, 2) - cPickle.dump(ids, f, 2) - f.close() - -count = 0 -for buginfo in buginfos: - count += 1 - print "{0}/{1}".format(count, len(buginfos)) - if count % 100 == 0: - save_to_cache() - - if ids.has_key(buginfo.bug_id): - continue - - ids[buginfo.bug_id] = True - - if not buginfo.bug_status in ["NEW", "ASSIGNED", "MODIFIED", "VERIFIED"]: - if options.verbose: - print "Bug {0} has status {1}, skipping.".format(buginfo.bug_id, buginfo.bug_status) - continue - - bug = bz.getbug(buginfo.bug_id) - - # Skip bugs with already downloaded backtraces. - filename = "{0}.bt".format(buginfo.bug_id) - if os.path.isfile(filename): - if options.verbose: - print "Skipping {0} (already exists).".format(filename) - else: - # Get backtrace from bug and store it as a file. - downloaded = False - for attachment in bug.attachments: - if attachment['filename'] == 'backtrace': - data = bz.openattachment(attachment['id']) - f = open(filename, 'w') - f.write(data.read()) - f.close() - downloaded = True - if options.verbose: - print "Attachment {0} downloaded.".format(filename) - - # Silently skip bugs without backtrace. - # Those are usually duplicates of bugs; the duplication copies - # abrt_hash, but it does not copy the attachment. - if not downloaded: - continue - - command = ["abrt-backtrace"] - command.append(filename) - command.append("--single-thread") - command.append("--frame-depth=5") - command.append("--remove-exit-handlers") - command.append("--remove-noncrash-frames") - - helper = subprocess.Popen(command, stdout=subprocess.PIPE) - backtrace, err = helper.communicate() - helper.wait() - - if helper.returncode != 0: - print "Problems parsing {0}".format(filename) - continue - - # Empty backtrace is provided by Python apps. - if len(backtrace) == 0: - continue - - bugitem = {'id':buginfo.bug_id, 'comments':len(bug.longdescs)} - if backtrace in database: - components = database[backtrace] - if buginfo.component in components: - components[buginfo.component].append(bugitem) - if options.verbose: - print "Duplicate found: {0}".format(database[out]['id']) - print "Backtrace: {0}".format(out) - else: - components[buginfo.component] = [ bugitem ] - else: - database[backtrace] = { buginfo.component: [ bugitem ] } - -# The number of duplicates. -dupcount = 0 -# The number of duplicates that can be closed. -dupclosecount = 0 -for backtrace, components in database.items(): - for component, bugitems in components.items(): - dupcount += len(bugitems) - 1 - dupclosecount += min(len(filter(lambda x: x <= 2, - map(lambda x: x["comments"], - bugitems))), - len(bugitems) - 1) - -# Get the component owner. -# Sort the duplicates by the component owner, and -# filter out those which should not be printed. -dups = [] -for backtrace, components in database.items(): - for component, bugitems in components.items(): - if len(bugitems) <= 1: - continue - - # Get the component owner - owner = "Failed to get component owner" - try: - component_info = json.load(urllib.urlopen("https://admin.fedoraproject.org/pkgdb/acls/name/{0}?tg_format=json".format(component))) - component_packages = component_info['packageListings'] - component_f12 = filter(lambda x:x["collection"]["version"]=="12", component_packages) - if len(component_f12) == 1: - owner = component_f12[0]["owner"] - except KeyError: - pass - - dups.append((component, owner, bugitems, backtrace)) - print "." - -# Close all bugs where it is appropriate. -if options.close: - LIMIT = 10000 # infinite - counter = 0 - for (component, owner, bugitems, backtrace) in dups: - # Find the master bug item - # Its the one with the most comments. - - # Sort function sorting by comment count. - def commentCmp(x, y): - if x['comments'] < y['comments']: - return 1 - elif x['comments'] == y['comments']: - # Sort by bug id, older bugs should became the master bug - if x['id'] > y['id']: - return 1 - elif x['id'] == y['id']: - return 0 - else: - return -1 - else: - return -1 - - # Sort the duplicates by the number of comments. - # Select the bug with the highest number of comments as the master bug. - # All other bugs without user comments will be closed as a duplicate of - # the master bug. - sorteditems = sorted(bugitems, commentCmp) - master = sorteditems[0] - - # Check the master bug status AGAIN to make sure the bug is still opened. - bug = bz.getbug(int(master['id'])) - if not bug.bug_status in ["NEW", "ASSIGNED"]: - continue - - for item in sorteditems[1:]: - if item['comments'] > 2: - continue - - # Check the bug status AGAIN to make sure the bug is still opened. - bug = bz.getbug(int(item['id'])) - if not bug.bug_status in ["NEW", "ASSIGNED"]: - continue - - print "Closing bug #{0} with {1} comments as a duplicate of #{2}.".format(item['id'], item['comments'], master['id']) - bug.close("DUPLICATE", int(master['id']), "", - ("This bug appears to have been filled using a buggy version of ABRT, because\n" + - "it contains a backtrace which is a duplicate of backtrace from bug #{0}.\n\n" + - "Sorry for the inconvenience.").format(master['id'])) - - counter += 1 - if counter > LIMIT: - sys.exit(0) - -bz.logout() - -print -print "SUMMARY" -print "==========================================================================" -print "Total number of duplicate bugs detected: {0}".format(dupcount) -print "Number of duplicate bugs that will be closed : {0}".format(dupclosecount) -print "------------------------------" - -# Print the duplicates sorted by package owner. -def cmp(x, y): - if x[1] < y[1]: - return -1 - elif x[1] == y[1]: - return 0 - else: - return 1 - -for (component, owner, bugitems, backtrace) in sorted(dups, cmp): - if options.wiki: - print "----" - print "* component: '''{0}''' ({1})".format(component, owner) - print "* duplicates: {0}".format( - reduce(lambda x,y: x+", "+y, - map(lambda x: "#[https://bugzilla.redhat.com/show_bug.cgi?id={0} {0}] ({1} comments)".format(x['id'],x['comments']), - bugitems))) - print "* backtrace:" - for line in backtrace.replace("Thread\n", "").splitlines(): - print "*# {0}".format(line) - else: - print "Component: {0} ({1})".format(component, owner) - print "Duplicates: {0}".format( - reduce(lambda x,y: x+", "+y, - map(lambda x: "{0} ({1})".format(x['id'],x['comments']), - bugitems))) - print "Backtrace: {0}".format(backtrace) diff --git a/scripts/abrt-bz-hashchecker b/scripts/abrt-bz-hashchecker deleted file mode 100755 index bb66c7d4..00000000 --- a/scripts/abrt-bz-hashchecker +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python -# -*- mode:python -*- -# Checks how many bugs in Bugzilla have the same hash. -# -# Please do not run this script unless it's neccessary to do so. -# It forces Bugzilla to send data related to thousands of bug reports. - -from bugzilla import RHBugzilla -from optparse import OptionParser -import sys -import os.path -import subprocess -import re - -parser = OptionParser(version="%prog 1.0") -parser.add_option("-u", "--user", dest="user", - help="Bugzilla user name (REQUIRED)", metavar="USERNAME") -parser.add_option("-p", "--password", dest="password", - help="Bugzilla password (REQUIRED)", metavar="PASSWORD") -parser.add_option("-b", "--bugzilla", dest="bugzilla", - help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") - -(options, args) = parser.parse_args() - -if not options.user or len(options.user) == 0: - parser.error("User name is required.\nTry {0} --help".format(sys.argv[0])) - -if not options.password or len(options.password) == 0: - parser.error("Password is required.\nTry {0} --help".format(sys.argv[0])) - -if not options.bugzilla or len(options.bugzilla) == 0: - options.bugzilla = "https://bugzilla.redhat.com/xmlrpc.cgi" - -bz = RHBugzilla() -bz.connect(options.bugzilla) -bz.login(options.user, options.password) - -buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash'}) - -print "{0} bugs found.".format(len(buginfos)) - -hashes = {} -for buginfo in buginfos: - match = re.search("abrt_hash:([^ ]+)", buginfo.status_whiteboard) - if not match: - continue - hash = match.group(1) - if not hash: - continue - if hash in hashes: - hashes[hash].append(buginfo.bug_id) - else: - hashes[hash] = [ buginfo.bug_id ] - print hash -bz.logout() - -for hash, ids in hashes.items(): - if len(ids) > 1: - print "Duplicates found: ", reduce(lambda x,y: str(x)+", "+str(y), ids) diff --git a/scripts/abrt-bz-ratingfixer b/scripts/abrt-bz-ratingfixer deleted file mode 100755 index d7881978..00000000 --- a/scripts/abrt-bz-ratingfixer +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/python -# -*- mode:python -*- -# -# Finds bugs with incomplete backtraces in Buzilla. -# Incomplete backtraces are caused by missing debuginfo. -# -# Please do not run this script unless it's neccessary to do so. -# It forces Bugzilla to send data related to thousands of bug reports. - -from bugzilla import RHBugzilla -from optparse import OptionParser -import sys -import os.path -import subprocess -import cPickle -import urllib -import json - -# -# Parse command line options. -# Exit if mandatory options are missing. -# -parser = OptionParser(version="%prog 1.0") -parser.add_option("-u", "--user", dest="user", - help="Bugzilla user name (REQUIRED)", metavar="USERNAME") -parser.add_option("-p", "--password", dest="password", - help="Bugzilla password (REQUIRED)", metavar="PASSWORD") -parser.add_option("-b", "--bugzilla", dest="bugzilla", default="https://bugzilla.redhat.com/xmlrpc.cgi", - help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") -parser.add_option("-i", "--wiki", help="Generate output in wiki syntax", - action="store_true", default=False, dest="wiki") -parser.add_option("-c", "--close", help="Close some of the bugs in Bugzilla (DANGEROUS)", - action="store_true", default=False, dest="close") -(options, args) = parser.parse_args() - -if not options.user or len(options.user) == 0: - parser.error("User name is required.\nTry {0} --help".format(sys.argv[0])) - -if not options.password or len(options.password) == 0: - parser.error("Password is required.\nTry {0} --help".format(sys.argv[0])) - -# -# Connect to the Bugzilla and get all bugs reported by ABRT -# -bz = RHBugzilla() -bz.connect(options.bugzilla) -bz.login(options.user, options.password) - -buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash'}) -print "{0} bugs found.".format(len(buginfos)) - -# -# Load cache from previous run. It speeds up the case Bugzilla closes connection. -# The cache should be manually removed after a day or so, because the data in it -# are no longer valid. -# -ids = {} -CACHE_FILE = "abrt-bz-ratingfixer-cache.tmp" -if os.path.isfile(CACHE_FILE): - f = open(CACHE_FILE, 'r') - ids = cPickle.load(f) - f.close() - -def save_to_cache(): - global database - f = open(CACHE_FILE, 'w') - cPickle.dump(ids, f, 2) - f.close() - -# -# Go through all bugs, and get the rating for their backtraces. -# The result is stored into ids map. -# -count = 0 -for buginfo in buginfos: - # The progress indicator. - count += 1 - print "{0}/{1}".format(count, len(buginfos)) - - # Save to cache for the case the connection will be closed by the Bugzilla. - # This happens pretty often. - if count % 100 == 0: - save_to_cache() - - # Skip the bugs already loaded from cache. - if ids.has_key(buginfo.bug_id): - continue - - # We handle only unprocessed bugs. - if not buginfo.bug_status in ["NEW", "ASSIGNED"]: - continue - - # By default: rating 4, no comments. Do not touch strange bugs. - ids[buginfo.bug_id] = { - 'rating': 4, - 'comment_count': 0, - 'component': buginfo.component - } - - # Skip bugs with already downloaded backtraces. - filename = "{0}.bt".format(buginfo.bug_id) - if not os.path.isfile(filename): - # Get backtrace from bug and store it as a file. - downloaded = False - bug = bz.getbug(buginfo.bug_id) - for attachment in bug.attachments: - if attachment['filename'] == 'backtrace': - data = bz.openattachment(attachment['id']) - f = open(filename, 'w') - f.write(data.read()) - f.close() - downloaded = True - - # Silently skip bugs without backtrace. - # Those are usually duplicates of bugs; the duplication copies - # abrt_hash, but it does not copy the attachment. - if not downloaded: - continue - - # Rate the backtrace using external program. - command = ["abrt-backtrace", "--rate"] - command.append(filename) - helper = subprocess.Popen(command, stdout=subprocess.PIPE) - rating, err = helper.communicate() - helper.wait() - if helper.returncode != 0: - print "Problems with rating {0}".format(filename) - continue - - # Get the comment count. We do not want to close bugs which - # are in the middle of a discussion. - bug = bz.getbug(buginfo.bug_id) - comment_count = 0 - for comment in bug.longdescs: - # Do not count "rawhide" comments from Bug Zappers - if comment["body"].find("against 'rawhide' during") > 0: - continue - comment_count += 1 - - # Put the result to the database. - ids[buginfo.bug_id] = { - 'rating': int(rating), - 'comment_count': comment_count, - 'component': buginfo.component - } - - # Close the bug if it's appropriate. - if options.close and comment_count <= 2 and int(rating) < 3: - print "Closing bug #{0} with {1} comments and rating {2}/4.".format(buginfo.bug_id, comment_count, int(rating)) - bug.close("INSUFFICIENT_DATA", 0, "", - "This bug appears to have been filled using a buggy version of ABRT, because\n" + - "it contains unusable backtrace. Sorry for the inconvenience.\n\n" + - "Closing as INSUFFICIENT_DATA.") - -bz.logout() - -# -# Get the component owners -# -bugids = [] -for id, bug in ids.items(): - # Skip bugs with good rating. - if bug['rating'] >= 3: - continue - - # Get the component owner - owner = "Failed to get component owner" - try: - component_info = json.load(urllib.urlopen("https://admin.fedoraproject.org/pkgdb/acls/name/{0}?tg_format=json".format(bug['component']))) - component_packages = component_info['packageListings'] - component_f12 = filter(lambda x:x["collection"]["version"]=="12", component_packages) - if len(component_f12) == 1: - owner = component_f12[0]["owner"] - except KeyError: - pass - - bug['id'] = id - bug['owner'] = owner - bugids.append(bug) - -def ownerCmp(a, b): - if a['owner'] < b['owner']: - return -1 - elif a['owner'] == b['owner']: - return 0 - else: - return 1 - -print "============= SUMMARY" -closedcount = 0 -if options.wiki: - print "{|" - print " ! Bug !! Backtrace rating !! Comment count !! Component !! Owner" - print " |-" -for bug in sorted(bugids, ownerCmp): - count += 1 - if bug['comment_count'] <= 2: - closedcount += 1 - - if options.wiki: - print " | #[https://bugzilla.redhat.com/show_bug.cgi?id={0} {0}] || {1}/4 || {2} || {3} || {4}".format(bug['id'], bug['rating'], bug['comment_count'], bug['component'], bug['owner']) - print " |-" - else: - print "#{0} has a backtrace with rating {1}/4 and {2} comments, component {3}, owner {4}".format(bug['id'], bug['rating'], bug['comment_count'], bug['component'], bug['owner']) - -if options.wiki: - print " |}" - -print "{0} bugs are included.".format(len(bugids)) -print "{0} bugs should be closed.".format(closedcount) diff --git a/scripts/abrt-bz-stats b/scripts/abrt-bz-stats deleted file mode 100755 index ea44f5bd..00000000 --- a/scripts/abrt-bz-stats +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/python -# -*- mode:python -*- -# ABRT Bugzilla Statistics script -# -# Please do not run this script unless it's neccessary to do so. -# It forces Bugzilla to send info about thousands of bug reports. - -from bugzilla import RHBugzilla -from optparse import OptionParser -import sys -import os.path -import subprocess -from datetime import datetime -import pickle - -# -# Parse the command line input -# -parser = OptionParser(version="%prog 1.0") -parser.add_option("-u", "--user", dest="user", - help="Bugzilla user name (REQUIRED)", metavar="USERNAME") -parser.add_option("-p", "--password", dest="password", - help="Bugzilla password (REQUIRED)", metavar="PASSWORD") -parser.add_option("-b", "--bugzilla", dest="bugzilla", - help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") -# Weekly stats shows the impact of changes in ABRT early. -parser.add_option("-w", "--weekly", help="Generate weekly report instead of monthly", - action="store_true", default=False, dest="weekly") -# HTML output for blogs etc. -parser.add_option("-t", "--html", help="Generate HTML output", - action="store_true", default=False, dest="html") -parser.add_option("-i", "--wiki", help="Generate output in wiki syntax", - action="store_true", default=False, dest="wiki") -# Newest stats first -parser.add_option("-r", "--reversed", help="Display the newest stats first", - action="store_true", default=False, dest="reversed") -(options, args) = parser.parse_args() -if not options.user or len(options.user) == 0: - parser.error("User name is required.\nTry {0} --help".format(sys.argv[0])) -if not options.password or len(options.password) == 0: - parser.error("Password is required.\nTry {0} --help".format(sys.argv[0])) -if not options.bugzilla or len(options.bugzilla) == 0: - options.bugzilla = "https://bugzilla.redhat.com/xmlrpc.cgi" - -# -# Connect to Bugzilla and get the list of all bugs reported by ABRT -# -bz = RHBugzilla() -bz.connect(options.bugzilla) -bz.login(options.user, options.password) - -buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash'}) -total = len(buginfos) -print "{0} bugs found.".format(total) - -# -# Load cache from previous run. Speeds up the case Bugzilla closes connection. -# -buginfos_loaded = {} -CACHE_FILE = "abrt-bz-stats-cache.tmp" -if os.path.isfile(CACHE_FILE): - f = open(CACHE_FILE, 'r') - buginfos_loaded = pickle.load(f) - f.close() - -def save_to_cache(): - global buginfos_loaded - f = open(CACHE_FILE, 'w') - pickle.dump(buginfos_loaded, f, 2) - f.close() - -# -# Load data from Bugzilla -# -count = 0 -for buginfo in buginfos: - count += 1 - print "{0}/{1}".format(count, total) - - if count % 100 == 0: - save_to_cache() - - if buginfos_loaded.has_key(buginfo.bug_id): - continue - - # creation date, format YEAR-MONTH-DAY - created = buginfo.creation_ts[0:10].replace(".", "-") - # last change to bug, format YEAR-MONTH-DAY - lastchange = buginfo.delta_ts[0:10].replace(".", "-") - status = buginfo.bug_status # status during the last change - if buginfo.resolution != "": - status += "_" + buginfo.resolution - buginfos_loaded[buginfo.bug_id] = { - 'created':created, - 'lastchange':lastchange, - 'status':status, - 'component':buginfo.component} - -bz.logout() -save_to_cache() - -# -# Interpret data from Bugzilla -# -# Bugs reported this month/week by ABRT -# Bugs closed as useful this month/week by ABRT -# Bugs closed as waste this month/week by ABRT -# Top crashers this month/week. -# -class TimeSpan: - """ - It's either a week or month. - """ - def __init__(self): - # Number of bugs reported to certain component this month. - self.components = {} - - self.closed_as_useful = 0 - self.closed_as_waste = 0 - self.closed_as_other = 0 - - def bugs_reported(self): - result = 0 - for component in self.components.values(): - result += component - return result - - def top_crashers(self, n = 10): - """ - Top n components causing crash this month. - Returns list of tuples (component, number of crashes) - """ - result = sorted(self.components.items(), key=lambda x: x[1]) - result.reverse() - return result[0:n] - - def closed_as_useful_percentage(self): - return int(100 * self.closed_as_useful / self.closed()) - - def closed_as_waste_percentage(self): - return int(100 * self.closed_as_waste / self.closed()) - - def closed_as_other_percentage(self): - return 100 - self.closed_as_useful_percentage() \ - - self.closed_as_waste_percentage() - - def closed(self): - return self.closed_as_useful + self.closed_as_waste + self.closed_as_other - - def add_component_crash(self, component): - if component in self.components: - self.components[component] += 1 - else: - self.components[component] = 1 - - def add_resolution(self, resolution): - # Catches only resolutions starting with "CLOSED_" - if resolution in ["CLOSED_CURRENTRELEASE", "CLOSED_RAWHIDE", "CLOSED_ERRATA", - "CLOSED_UPSTREAM", "CLOSED_NEXTRELEASE"]: - self.closed_as_useful += 1 - elif resolution in ["CLOSED_DUPLICATE", "CLOSED_CANTFIX", - "CLOSED_INSUFFICIENT_DATA"]: - self.closed_as_waste += 1 - elif resolution in ["CLOSED_NOTABUG", "CLOSED_WONTFIX", - "CLOSED_DEFERRED", "CLOSED_WORKSFORME"]: - self.closed_as_other += 1 - - def __str__(self): - def bug(count): - if count == 1: - return "%d bug" % count - else: - return "%d bugs" % count - - def crash(count): - if count == 1: - return "%d crash" % count - else: - return "%d crashes" % count - - start = "" - bugs_reported = " - %s reported\n" - bugs_closed = " - %s closed\n" - bugs_cl_useful = " - %s (%d%%) as fixed, so ABRT was useful\n" - bugs_cl_notuseful = " - %s (%d%%) as duplicate, can't fix, insuf. data, so ABRT was not useful\n" - bugs_cl_other = " - %s (%d%%) as notabug, wontfix, worksforme\n" - bugs_closed_end = "" - top_crashers = " - top crashers:\n" - top_crasher_item = " # %s: %s\n" - top_crashers_end = "" - end = "" - if options.html: - start = "\n" - elif options.wiki: - start = "" - bugs_reported = "* %s reported\n" - bugs_closed = "* %s closed\n" - bugs_cl_useful = "** %s (%d%%) as fixed, so ABRT was useful\n" - bugs_cl_notuseful = "** %s (%d%%) as duplicate, can't fix, insuf. data, so ABRT was not useful\n" - bugs_cl_other = "** %s (%d%%) as notabug, wontfix, worksforme\n" - bugs_closed_end = "" - top_crashers = "* top crashers:\n" - top_crasher_item = "*# %s: %s\n" - top_crashers_end = "" - end = "" - - - str = start - str += bugs_reported % bug(self.bugs_reported()) - if self.closed() > 0: - str += bugs_closed % bug(self.closed()) - if self.closed_as_useful > 0: - str += bugs_cl_useful % (bug(self.closed_as_useful), self.closed_as_useful_percentage()) - if self.closed_as_waste > 0: - str += bugs_cl_notuseful % (bug(self.closed_as_waste), self.closed_as_waste_percentage()) - if self.closed_as_other > 0: - str += bugs_cl_other % (bug(self.closed_as_other), self.closed_as_other_percentage()) - str += bugs_closed_end - if len(self.top_crashers()) > 0: - str += top_crashers - for (component, num_crashes) in self.top_crashers(): - str += top_crasher_item % (component, crash(num_crashes)) - str += top_crashers_end - str += end - return str - -monthly_stats = {} # key == YEAR-MONTH, value == Month() -weekly_stats = {} # key == YEAR-WEEK, value == Month() - -def get_month(month): - global monthly_stats - if month in monthly_stats: - return monthly_stats[month] - else: - monthly_stats[month] = TimeSpan() - return monthly_stats[month] - -def get_week(week): - global weekly_stats - if week in weekly_stats: - return weekly_stats[week] - else: - weekly_stats[week] = TimeSpan() - return weekly_stats[week] - -for buginfo in buginfos_loaded.values(): - # Bugs reported this month by ABRT - # Top crashers this month - month_key = buginfo['created'][0:7] - month = get_month(month_key) - month.add_component_crash(buginfo['component']) - - # Bugs reported this week by ABRT - # Top crashers this week - week_key = datetime.strptime(buginfo['created'], "%Y-%m-%d").strftime("%Y-%W") - week = get_week(week_key) - week.add_component_crash(buginfo['component']) - - # Bugs closed as useful this month by ABRT - # Bugs closed as waste this month by ABRT - month_key = buginfo['lastchange'][0:7] - month = get_month(month_key) - month.add_resolution(buginfo['status']) - - # Bugs closed as useful this week by ABRT - # Bugs closed as waste this week by ABRT - week_key = datetime.strptime(buginfo['lastchange'], "%Y-%m-%d").strftime("%Y-%W") - week = get_week(week_key) - week.add_resolution(buginfo['status']) - -# -# Print interpreted data -# -print "STATS" -print "==========================================================================" -if not options.weekly: - months = monthly_stats.keys() - months.sort() - if options.reversed: - months.reverse() - for month in months: - m = monthly_stats[month] - if options.html: - print "

Month %s

" % month - elif options.wiki: - print "==Month %s==" % month - else: - print "MONTH %s" % month - print m -else: - weeks = weekly_stats.keys() - weeks.sort() - if options.reversed: - weeks.reverse() - for week in weeks: - w = weekly_stats[week] - if options.html: - print "

Week %s

" % week - elif options.wiki: - print "==Week %s==" % week - else: - print "WEEK %s" % week - print w diff --git a/scripts/check-bt-parsability b/scripts/check-bt-parsability deleted file mode 100755 index b154ad88..00000000 --- a/scripts/check-bt-parsability +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -*- mode: bash -*- - -PASS=0 -FAIL=0 -for file in *.bt -do - #echo "$file" - ./abrt-backtrace $file 1> /dev/null - if [ "$?" -eq "0" ] - then - echo -n "." - PASS=$(($PASS+1)) - else - echo "-$file" - FAIL=$(($FAIL+1)) - fi -done -echo "" -echo "Passed $PASS and failed $FAIL." -- cgit