diff options
| -rwxr-xr-x | scripts/abrt-bz-ratingfixer | 142 | ||||
| -rw-r--r-- | scripts/abrt-rate-backtrace.c | 183 | ||||
| -rwxr-xr-x | src/Backtrace/abrt-bz-dupchecker | 69 |
3 files changed, 384 insertions, 10 deletions
diff --git a/scripts/abrt-bz-ratingfixer b/scripts/abrt-bz-ratingfixer new file mode 100755 index 00000000..8f45d821 --- /dev/null +++ b/scripts/abrt-bz-ratingfixer @@ -0,0 +1,142 @@ +#!/usr/bin/python +# -*- mode:python -*- +# +# Please do not run this script unless it's neccessary to do so. +# It forces Bugzilla to send data related to thousands of bug reports. + +from bugzilla import RHBugzilla +from optparse import OptionParser +import sys +import os.path +import subprocess +import cPickle + +parser = OptionParser(version="%prog 1.0") +parser.add_option("-u", "--user", dest="user", + help="Bugzilla user name (REQUIRED)", metavar="USERNAME") +parser.add_option("-p", "--password", dest="password", + help="Bugzilla password (REQUIRED)", metavar="PASSWORD") +parser.add_option("-b", "--bugzilla", dest="bugzilla", + help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") +parser.add_option("-i", "--wiki", help="Generate output in wiki syntax", + action="store_true", default=False, dest="wiki") + +(options, args) = parser.parse_args() + +if not options.user or len(options.user) == 0: + parser.error("User name is required.\nTry {0} --help".format(sys.argv[0])) + +if not options.password or len(options.password) == 0: + parser.error("Password is required.\nTry {0} --help".format(sys.argv[0])) + +if not options.bugzilla or len(options.bugzilla) == 0: + options.bugzilla = "https://bugzilla.redhat.com/xmlrpc.cgi" + +bz = RHBugzilla() +bz.connect(options.bugzilla) +bz.login(options.user, options.password) + +buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash'}) + +print "{0} bugs found.".format(len(buginfos)) + +# +# Load cache from previous run. Speeds up the case Bugzilla closes connection. +# The cache should be manually removed after a day or so, because the data in it +# are no longer valid. +# +ids = {} +CACHE_FILE = "abrt-bz-ratingfixer-cache.tmp" +if os.path.isfile(CACHE_FILE): + f = open(CACHE_FILE, 'r') + ids = cPickle.load(f) + f.close() + +def save_to_cache(): + global database + f = open(CACHE_FILE, 'w') + cPickle.dump(ids, f, 2) + f.close() + +count = 0 +for buginfo in buginfos: + count += 1 + print "{0}/{1}".format(count, len(buginfos)) + if count % 100 == 0: + save_to_cache() + + if ids.has_key(buginfo.bug_id): + continue + + if not buginfo.bug_status in ["NEW", "ASSIGNED"]: + continue + + # By default: rating 4, no comments. Do not touch strange bugs. + ids[buginfo.bug_id] = ( 4, 0 ) + + # Skip bugs with already downloaded backtraces. + filename = "{0}.bt".format(buginfo.bug_id) + if not os.path.isfile(filename): + # Get backtrace from bug and store it as a file. + downloaded = False + bug = bz.getbug(buginfo.bug_id) + for attachment in bug.attachments: + if attachment['filename'] == 'backtrace': + data = bz.openattachment(attachment['id']) + f = open(filename, 'w') + f.write(data.read()) + f.close() + downloaded = True + + # Silently skip bugs without backtrace. + # Those are usually duplicates of bugs; the duplication copies + # abrt_hash, but it does not copy the attachment. + if not downloaded: + continue + + command = ["./abrt-rate-backtrace"] + command.append(filename) + + helper = subprocess.Popen(command, stdout=subprocess.PIPE) + rating, err = helper.communicate() + helper.wait() + + if helper.returncode != 0: + print "Problems with rating {0}".format(filename) + continue + + bug = bz.getbug(buginfo.bug_id) + comments = 0 + for comment in bug.longdescs: + # Do not count "rawhide" comments from Bug Zappers + if comment["body"].find("against 'rawhide' during") > 0: + continue + comments += 1 + + ids[buginfo.bug_id] = ( int(rating), comments ) + +bz.logout() + +print "============= SUMMARY" +count = 0 +bugids = ids.keys() +bugids.sort() +if options.wiki: + print "{|" + print " ! Bug !! Backtrace rating !! Comment count" + print " |-" +for bugid in bugids: + rating = ids[bugid] + if rating[0] < 3 and rating[1] <= 2: + count += 1 + if options.wiki: + print " | #[https://bugzilla.redhat.com/show_bug.cgi?id={0} {0}] || {1}/4 || {2}".format(bugid, rating[0], rating[1]) + print " |-" + else: + print "#{0} has a backtrace with rating {1}/4 and {2} comments".format(bugid, rating[0], rating[1]) + +if options.wiki: + print " |}" + + +print "{0} bugs should be closed.".format(count) diff --git a/scripts/abrt-rate-backtrace.c b/scripts/abrt-rate-backtrace.c new file mode 100644 index 00000000..07248233 --- /dev/null +++ b/scripts/abrt-rate-backtrace.c @@ -0,0 +1,183 @@ +/* -*-mode:c++;c-file-style:"bsd";c-basic-offset:4;indent-tabs-mode:nil-*- + * Returns rating 0-4 of backtrace file on stdout. + * 4 - backtrace with complete or almost-complete debuginfos + * 0 - useless backtrace with no debuginfos + * Compile: + * gcc abrt-rate-backtrace.c -std=c99 -o abrt-rate-backtrace + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdbool.h> + +// Die if we can't allocate n+1 bytes (space for the null terminator) and copy +// the (possibly truncated to length n) string into it. +static char* xstrndup(const char *s, int n) +{ + int m; + char *t; + + /* We can just xmalloc(n+1) and strncpy into it, */ + /* but think about xstrndup("abc", 10000) wastage! */ + m = n; + t = (char*) s; + while (m) { + if (!*t) break; + m--; + t++; + } + n -= m; + t = (char*) malloc(n + 1); + t[n] = '\0'; + + return (char*) memcpy(t, s, n); +} + +enum LineRating +{ + // RATING EXAMPLE + MissingEverything = 0, // #0 0x0000dead in ?? () + MissingFunction = 1, // #0 0x0000dead in ?? () from /usr/lib/libfoobar.so.4 + MissingLibrary = 2, // #0 0x0000dead in foobar() + MissingSourceFile = 3, // #0 0x0000dead in FooBar::FooBar () from /usr/lib/libfoobar.so.4 + Good = 4, // #0 0x0000dead in FooBar::crash (this=0x0) at /home/user/foobar.cpp:204 + BestRating = Good, +}; + +static enum LineRating rate_line(const char *line) +{ +#define FOUND(x) (strstr(line, x) != NULL) + /* see the "enum LineRating" comments for possible combinations */ + if (FOUND(" at ")) + return Good; + const char *function = strstr(line, " in "); + if (function) + { + if (function[4] == '?') /* " in ??" does not count */ + { + function = NULL; + } + } + bool library = FOUND(" from "); + if (function && library) + return MissingSourceFile; + if (function) + return MissingLibrary; + if (library) + return MissingFunction; + + return MissingEverything; +#undef FOUND +} + +/* returns number of "stars" to show */ +static int rate_backtrace(const char *backtrace) +{ + int i, len; + int multiplier = 0; + int rating = 0; + int best_possible_rating = 0; + char last_lvl = 0; + + /* We look at the frames in reversed order, since: + * - rate_line() checks starting from the first line of the frame + * (note: it may need to look at more than one line!) + * - we increase weight (multiplier) for every frame, + * so that topmost frames end up most important + */ + len = 0; + for (i = strlen(backtrace) - 1; i >= 0; i--) + { + if (backtrace[i] == '#' + && (backtrace[i+1] >= '0' && backtrace[i+1] <= '9') /* #N */ + && (i == 0 || backtrace[i-1] == '\n') /* it's at line start */ + ) { + /* For one, "#0 xxx" always repeats, skip repeats */ + if (backtrace[i+1] == last_lvl) + continue; + last_lvl = backtrace[i+1]; + + char *s = xstrndup(backtrace + i + 1, len); + /* Replace tabs with spaces, rate_line() does not expect tabs. + * Actually, even newlines may be there. Example of multiline frame + * where " at SRCFILE" is on 2nd line: + * #3 0x0040b35d in __libc_message (do_abort=<value optimized out>, + * fmt=<value optimized out>) at ../sysdeps/unix/sysv/linux/libc_fatal.c:186 + */ + for (char *p = s; *p; p++) + if (*p == '\t' || *p == '\n') + *p = ' '; + int lrate = rate_line(s); + multiplier++; + rating += lrate * multiplier; + best_possible_rating += BestRating * multiplier; + //log("lrate:%d rating:%d best_possible_rating:%d s:'%-.40s'", lrate, rating, best_possible_rating, s); + free(s); + len = 0; /* starting new line */ + } + else + { + len++; + } + } + + /* Bogus 'backtrace' with zero frames? */ + if (best_possible_rating == 0) + return 0; + + /* Returning number of "stars" to show */ + if (rating*10 >= best_possible_rating*8) /* >= 0.8 */ + return 4; + if (rating*10 >= best_possible_rating*6) + return 3; + if (rating*10 >= best_possible_rating*4) + return 2; + if (rating*10 >= best_possible_rating*2) + return 1; + + return 0; +} + +int main(int argc, char **argv) +{ + if (argc != 2) + { + fprintf(stderr, "Usage: %s BACKTRACE_FILE\n", argv[0]); + return 1; + } + + FILE *fp = fopen(argv[1], "r"); + if (!fp) + { + fprintf(stderr, "Cannot open the input file.\n"); + return 2; + } + fseek(fp, 0, SEEK_END); + size_t size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + char *file = malloc(size + 1); + if (!file) + { + fprintf(stderr, "Malloc error.\n"); + return 3; + } + size_t read = fread(file, size, 1, fp); + if (read != 1) + { + fprintf(stderr, "Error while reading file.\n", argv[0]); + return 4; + } + fclose(fp); + file[size] = '\0'; + + int rating = 4; + /* Do not rate Python backtraces. */ + if (NULL == strstr(file, "Local variables in innermost frame:\n")) + rating = rate_backtrace(file); + + free(file); + fprintf(stdout, "%d", rating); + return 0; +} diff --git a/src/Backtrace/abrt-bz-dupchecker b/src/Backtrace/abrt-bz-dupchecker index 344d1326..cbdafc53 100755 --- a/src/Backtrace/abrt-bz-dupchecker +++ b/src/Backtrace/abrt-bz-dupchecker @@ -24,6 +24,7 @@ from optparse import OptionParser import sys import os.path import subprocess +import cPickle parser = OptionParser(version="%prog 1.0") parser.add_option("-u", "--user", dest="user", @@ -34,6 +35,8 @@ parser.add_option("-b", "--bugzilla", dest="bugzilla", help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL") parser.add_option("-v", "--verbose", dest="verbose", help="Detailed output") +parser.add_option("-i", "--wiki", help="Generate output in wiki syntax", + action="store_true", default=False, dest="wiki") (options, args) = parser.parse_args() @@ -54,9 +57,38 @@ buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboar print "{0} bugs found.".format(len(buginfos)) +# +# Load cache from previous run. Speeds up the case Bugzilla closes connection. +# The cache should be manually removed after a day or so, because the data in it +# are no longer valid. +# database = {} - +ids = {} +CACHE_FILE = "abrt-bz-dupchecker-cache.tmp" +if os.path.isfile(CACHE_FILE): + f = open(CACHE_FILE, 'r') + database = cPickle.load(f) + ids = cPickle.load(f) + f.close() + +def save_to_cache(): + global database + f = open(CACHE_FILE, 'w') + cPickle.dump(database, f, 2) + cPickle.dump(ids, f, 2) + f.close() + +count = 0 for buginfo in buginfos: + count += 1 + print "{0}/{1}".format(count, len(buginfos)) + if count % 100 == 0: + save_to_cache() + + if ids.has_key(buginfo.bug_id): + continue + ids[buginfo.bug_id] = True + if not buginfo.bug_status in ["NEW", "ASSIGNED", "MODIFIED", "VERIFIED"]: if options.verbose: print "Bug {0} has status {1}, skipping.".format(buginfo.bug_id, buginfo.bug_status) @@ -127,21 +159,38 @@ print "========================================================================= # The number of duplicates. dupcount = 0 +# The number of duplicates that can be closed. +dupclosecount = 0 for backtrace, components in database.items(): for component, bugitems in components.items(): - if len(bugitems) > 1: - dupcount += len(bugitems) - 1 - + dupcount += len(bugitems) - 1 + dupclosecount += min(len(filter(lambda x: x <= 2, + map(lambda x: x["comments"], + bugitems))), + len(bugitems) - 1) + print "Total number of duplicate bugs detected: {0}".format(dupcount) +print "Number of duplicate bugs that will be closed : {0}".format(dupclosecount) print "------------------------------" # Print the duplicates for backtrace, components in database.items(): for component, bugitems in components.items(): if len(bugitems) > 1: - print "Component: {0}".format(component) - print "Duplicates: {0}".format( - reduce(lambda x,y: x+", "+y, - map(lambda x: "{0} ({1})".format(x['id'],x['comments']), - bugitems))) - print "Backtrace: {0}".format(backtrace) + if options.wiki: + print "----" + print "* component: '''{0}'''".format(component) + print "* duplicates: {0}".format( + reduce(lambda x,y: x+", "+y, + map(lambda x: "#[https://bugzilla.redhat.com/show_bug.cgi?id={0} {0}] ({1} comments)".format(x['id'],x['comments']), + bugitems))) + print "* backtrace:" + for line in backtrace.replace("Thread\n", "").splitlines(): + print "*# {0}".format(line) + else: + print "Component: {0}".format(component) + print "Duplicates: {0}".format( + reduce(lambda x,y: x+", "+y, + map(lambda x: "{0} ({1})".format(x['id'],x['comments']), + bugitems))) + print "Backtrace: {0}".format(backtrace) |