path: root/scripts/abrt-bz-stats

#!/usr/bin/python
# -*- mode:python -*-
# ABRT Bugzilla Statistics script
#
# Please do not run this script unless it's neccessary to do so.
# It forces Bugzilla to send info about thousands of bug reports.

from bugzilla import RHBugzilla
from optparse import OptionParser
import sys
import os.path
import subprocess
from datetime import datetime
import pickle

#
# Parse the command line input
#
parser = OptionParser(version="%prog 1.0")
parser.add_option("-u", "--user", dest="user",
                  help="Bugzilla user name (REQUIRED)", metavar="USERNAME")
parser.add_option("-p", "--password", dest="password",
                  help="Bugzilla password (REQUIRED)", metavar="PASSWORD")
parser.add_option("-b", "--bugzilla", dest="bugzilla",
                  help="Bugzilla URL (defaults to Red Hat Bugzilla)", metavar="URL")
# Weekly stats shows the impact of changes in ABRT early.
parser.add_option("-w", "--weekly", help="Generate weekly report instead of monthly",
                  action="store_true", default=False, dest="weekly")
# HTML output for blogs etc.
parser.add_option("-t", "--html", help="Generate HTML output",
                  action="store_true", default=False, dest="html")
parser.add_option("-i", "--wiki", help="Generate output in wiki syntax",
                  action="store_true", default=False, dest="wiki")
# Newest stats first
parser.add_option("-r", "--reversed", help="Display the newest stats first",
                  action="store_true", default=False, dest="reversed")
(options, args) = parser.parse_args()
if not options.user or len(options.user) == 0:
  parser.error("User name is required.\nTry {0} --help".format(sys.argv[0]))
if not options.password or len(options.password) == 0:
  parser.error("Password is required.\nTry {0} --help".format(sys.argv[0]))
if not options.bugzilla or len(options.bugzilla) == 0:
  options.bugzilla = "https://bugzilla.redhat.com/xmlrpc.cgi"

#
# Connect to Bugzilla and get the list of all bugs reported by ABRT
#
bz = RHBugzilla()
bz.connect(options.bugzilla)
bz.login(options.user, options.password)

buginfos = bz.query({'status_whiteboard_type':'allwordssubstr','status_whiteboard':'abrt_hash'})
total = len(buginfos)
print "{0} bugs found.".format(total)

#
# Load cache from previous run. Speeds up the case Bugzilla closes connection.
#
buginfos_loaded = {}
CACHE_FILE = "abrt-bz-stats-cache.tmp"
if os.path.isfile(CACHE_FILE):
  f = open(CACHE_FILE, 'r')
  buginfos_loaded = pickle.load(f)
  f.close()

def save_to_cache():
  global buginfos_loaded
  f = open(CACHE_FILE, 'w')
  pickle.dump(buginfos_loaded, f, 2)
  f.close()

#
# Load data from Bugzilla
#
count = 0
for buginfo in buginfos:
  count += 1
  print "{0}/{1}".format(count, total)

  if count % 100 == 0:
    save_to_cache()

  if buginfos_loaded.has_key(buginfo.bug_id):
    continue

  # creation date, format YEAR-MONTH-DAY
  created = buginfo.creation_ts[0:10].replace(".", "-")
  # last change to bug, format YEAR-MONTH-DAY
  lastchange = buginfo.delta_ts[0:10].replace(".", "-")
  status = buginfo.bug_status # status during the last change
  if buginfo.resolution != "":
    status += "_" + buginfo.resolution
  buginfos_loaded[buginfo.bug_id] = {
    'created':created,
    'lastchange':lastchange,
    'status':status,
    'component':buginfo.component}

bz.logout()
save_to_cache()

#
# Interpret data from Bugzilla
#
# Bugs reported this month/week by ABRT
# Bugs closed as useful this month/week by ABRT
# Bugs closed as waste this month/week by ABRT
# Top crashers this month/week.
#
class TimeSpan:
  """
  It's either a week or month.
  """
  def __init__(self):
    # Number of bugs reported to certain component this month.
    self.components = {}

    self.closed_as_useful = 0
    self.closed_as_waste = 0
    self.closed_as_other = 0

  def bugs_reported(self):
    result = 0
    for component in self.components.values():
      result += component
    return result

  def top_crashers(self, n = 10):
    """
    Top n components causing crash this month.
    Returns list of tuples (component, number of crashes)
    """
    result = sorted(self.components.items(), key=lambda x: x[1])
    result.reverse()
    return result[0:n]

  def closed_as_useful_percentage(self):
    return int(100 * self.closed_as_useful / self.closed())

  def closed_as_waste_percentage(self):
    return int(100 * self.closed_as_waste / self.closed())

  def closed_as_other_percentage(self):
    return 100 - self.closed_as_useful_percentage() \
        - self.closed_as_waste_percentage()

  def closed(self):
    return self.closed_as_useful + self.closed_as_waste + self.closed_as_other

  def add_component_crash(self, component):
    if component in self.components:
      self.components[component] += 1
    else:
      self.components[component] = 1

  def add_resolution(self, resolution):
    # Catches only resolutions starting with "CLOSED_"
    if resolution in ["CLOSED_CURRENTRELEASE", "CLOSED_RAWHIDE", "CLOSED_ERRATA",
                      "CLOSED_UPSTREAM", "CLOSED_NEXTRELEASE"]:
      self.closed_as_useful += 1
    elif resolution in ["CLOSED_DUPLICATE", "CLOSED_CANTFIX",
                        "CLOSED_INSUFFICIENT_DATA"]:
      self.closed_as_waste += 1
    elif resolution in ["CLOSED_NOTABUG", "CLOSED_WONTFIX",
                           "CLOSED_DEFERRED", "CLOSED_WORKSFORME"]:
      self.closed_as_other += 1

  def __str__(self):
    def bug(count):
      if count == 1:
        return "%d bug" % count
      else:
        return "%d bugs" % count

    def crash(count):
      if count == 1:
        return "%d crash" % count
      else:
        return "%d crashes" % count

    start = ""
    bugs_reported = " - %s reported\n"
    bugs_closed = " - %s closed\n"
    bugs_cl_useful =  "   - %s (%d%%) as fixed, so ABRT was useful\n"
    bugs_cl_notuseful = "   - %s (%d%%) as duplicate, can't fix, insuf. data, so ABRT was not useful\n"
    bugs_cl_other = "   - %s (%d%%) as notabug, wontfix, worksforme\n"
    bugs_closed_end = ""
    top_crashers = " - top crashers:\n"
    top_crasher_item = "    # %s: %s\n"
    top_crashers_end = ""
    end = ""
    if options.html:
      start = "<ul>\n"
      bugs_reported = "<li>%s reported</li>\n"
      bugs_closed = "<li>%s closed\n<ul>\n"
      bugs_cl_useful = "  <li>%s (%d%%) as fixed, so ABRT was useful</li>\n"
      bugs_cl_notuseful = "   <li>%s (%d%%) as duplicate, can't fix, insuf. data, so ABRT was not useful</li>\n"
      bugs_cl_other = "   <li>%s (%d%%) as notabug, wontfix, worksforme</li>\n"
      bugs_closed_end = "</ul></li>\n"
      top_crashers = "<li>top crashers:\n<ol>\n"
      top_crasher_item = "    <li>%s: %s</li>\n"
      top_crashers_end = "</ol></li>\n"
      end = "</ul>\n"
    elif options.wiki:
      start = ""
      bugs_reported = "* %s reported\n"
      bugs_closed = "* %s closed\n"
      bugs_cl_useful =  "** %s (%d%%) as fixed, so ABRT was useful\n"
      bugs_cl_notuseful = "** %s (%d%%) as duplicate, can't fix, insuf. data, so ABRT was not useful\n"
      bugs_cl_other = "** %s (%d%%) as notabug, wontfix, worksforme\n"
      bugs_closed_end = ""
      top_crashers = "* top crashers:\n"
      top_crasher_item = "*# %s: %s\n"
      top_crashers_end = ""
      end = ""


    str = start
    str += bugs_reported % bug(self.bugs_reported())
    if self.closed() > 0:
      str += bugs_closed % bug(self.closed())
      if self.closed_as_useful > 0:
        str += bugs_cl_useful % (bug(self.closed_as_useful), self.closed_as_useful_percentage())
      if self.closed_as_waste > 0:
        str += bugs_cl_notuseful % (bug(self.closed_as_waste), self.closed_as_waste_percentage())
      if self.closed_as_other > 0:
        str += bugs_cl_other % (bug(self.closed_as_other), self.closed_as_other_percentage())
      str += bugs_closed_end
    if len(self.top_crashers()) > 0:
      str += top_crashers
      for (component, num_crashes) in self.top_crashers():
        str += top_crasher_item % (component, crash(num_crashes))
      str += top_crashers_end
    str += end
    return str

monthly_stats = {} # key == YEAR-MONTH, value == Month()
weekly_stats = {} # key == YEAR-WEEK, value == Month()

def get_month(month):
  global monthly_stats
  if month in monthly_stats:
    return monthly_stats[month]
  else:
    monthly_stats[month] = TimeSpan()
    return monthly_stats[month]

def get_week(week):
  global weekly_stats
  if week in weekly_stats:
    return weekly_stats[week]
  else:
    weekly_stats[week] = TimeSpan()
    return weekly_stats[week]

for buginfo in buginfos_loaded.values():
  # Bugs reported this month by ABRT
  # Top crashers this month
  month_key = buginfo['created'][0:7]
  month = get_month(month_key)
  month.add_component_crash(buginfo['component'])

  # Bugs reported this week by ABRT
  # Top crashers this week
  week_key = datetime.strptime(buginfo['created'], "%Y-%m-%d").strftime("%Y-%W")
  week = get_week(week_key)
  week.add_component_crash(buginfo['component'])

  # Bugs closed as useful this month by ABRT
  # Bugs closed as waste this month by ABRT
  month_key = buginfo['lastchange'][0:7]
  month = get_month(month_key)
  month.add_resolution(buginfo['status'])

  # Bugs closed as useful this week by ABRT
  # Bugs closed as waste this week by ABRT
  week_key = datetime.strptime(buginfo['lastchange'], "%Y-%m-%d").strftime("%Y-%W")
  week = get_week(week_key)
  week.add_resolution(buginfo['status'])

#
# Print interpreted data
#
print "STATS"
print "=========================================================================="
if not options.weekly:
  months = monthly_stats.keys()
  months.sort()
  if options.reversed:
    months.reverse()
  for month in months:
    m = monthly_stats[month]
    if options.html:
      print "<h2>Month %s</h2>" % month
    elif options.wiki:
      print "==Month %s==" % month
    else:
      print "MONTH %s" % month
    print m
else:
  weeks = weekly_stats.keys()
  weeks.sort()
  if options.reversed:
    weeks.reverse()
  for week in weeks:
    w = weekly_stats[week]
    if options.html:
      print "<h2>Week %s</h2>" % week
    elif options.wiki:
      print "==Week %s==" % week
    else:
      print "WEEK %s" % week
    print w