#!/usr/bin/python # Copyright © 2012 Red Hat Inc. # Author(s): Josh Boyer # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; version 2 of the License. # See http://www.gnu.org/copyleft/gpl.html for the full text of the license. import sys import ConfigParser import argparse import rpmUtils.miscutils import koji import os from functools import cmp_to_key import cPickle import urlgrabber.grabber as grabber import urlgrabber.progress as progress import shutil class Options: debug = True server = None weburl = None pkgurl = None topdir = None cert = None ca = None serverca = None authtype = None noauth = None user = None runas = None bisect_log_file = None def get_options(): global options # load local config defaults = { 'server' : 'http://localhost/kojihub', 'weburl' : 'http://localhost/koji', 'pkgurl' : 'http://localhost/packages', 'topdir' : '/mnt/koji', 'max_retries' : None, 'retry_interval': None, 'anon_retry' : None, 'offline_retry' : None, 'offline_retry_interval' : None, 'poll_interval': 5, 'cert': '~/.koji/client.crt', 'ca': '~/.koji/clientca.crt', 'serverca': '~/.koji/serverca.crt', 'authtype': None } # grab settings from /etc/koji.conf first, and allow them to be # overridden by user config progname = 'koji' for configFile in ('/etc/koji.conf',): if os.access(configFile, os.F_OK): f = open(configFile) config = ConfigParser.ConfigParser() config.readfp(f) f.close() if config.has_section(progname): for name, value in config.items(progname): #note the defaults dictionary also serves to indicate which #options *can* be set via the config file. Such options should #not have a default value set in the option parser. if defaults.has_key(name): if name in ('anon_retry', 'offline_retry'): defaults[name] = config.getboolean(progname, name) elif name in ('max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval'): try: defaults[name] = int(value) except ValueError: parser.error("value for %s config option must be a valid integer" % name) assert False else: defaults[name] = value for name, value in defaults.iteritems(): if getattr(options, name, None) is None: # print '%s' % getattr(options, name, None) setattr(options, name, value) # print '%s' % getattr(options, name, None) dir_opts = ('topdir', 'cert', 'ca', 'serverca') for name in dir_opts: # expand paths here, so we don't have to worry about it later value = os.path.expanduser(getattr(options, name)) setattr(options, name, value) #honor topdir if options.topdir: koji.BASEDIR = options.topdir koji.pathinfo.topdir = options.topdir return options def ensure_connection(session): try: ret = session.getAPIVersion() except xmlrpclib.ProtocolError: error(_("Error: Unable to connect to server")) if ret != koji.API_VERSION: warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) def activate_session(session): """Test and login the session is applicable""" global options if options.authtype == "noauth" or options.noauth: #skip authentication pass elif options.authtype == "ssl" or os.path.isfile(options.cert) and options.authtype is None: # authenticate using SSL client cert session.ssl_login(options.cert, options.ca, options.serverca, proxyuser=options.runas) elif options.authtype == "password" or options.user and options.authtype is None: # authenticate using user/password session.login() if not options.noauth and options.authtype != "noauth" and not session.logged_in: error(_("Unable to log in, no authentication methods available")) ensure_connection(session) if options.debug: print "successfully connected to hub" # This is copied almost directly from koji. Sigh again. def download_build(session, build, arch): activate_session(session) info = session.getBuild(build) if info is None: print "No such build: %s" % build print "Email Josh because he messed up somewhere." sys.exit(1) rpms = session.listRPMs(buildID=info['id'], arches=arch) if not rpms: print "No packages available for %s" % koji.buildLabel(info) sys.exit(1) urls = [] for rpm in rpms: # Skip debuginfo because it's huge if koji.is_debuginfo(rpm['name']): continue fname = koji.pathinfo.rpm(rpm) url = '%s/%s/%s/%s/%s' % (options.pkgurl, info['name'], info['version'], info['release'], fname) urls.append((url, 'downloads/%s/%s' % (build,os.path.basename(fname)))) pg = progress.TextMeter() for url, relpath in urls: file = grabber.urlopen(url, progress_obj=pg, text=relpath) if '/' in relpath: koji.ensuredir(os.path.dirname(relpath)) out = os.open(relpath, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666) try: while 1: buf = file.read(4096) if not buf: break os.write(out, buf) finally: os.close(out) file.close() # Everything above this line is all koji session bullshit that we shouldn't # have to copy, but do because koji doesn't put it in a damn module somewhere # ANGRY CODING def sort_builds(builds): build_tuples=[] pkg = builds[0]['package_name'] # compareEVR takes tuples consisting of epoch, version, release # we need to split the 'nvr' string we got from koji up into that # so we can get it sorted in RPM order for build in builds: bld = build['nvr'].split('-') bld.insert(1,build['epoch']) build_tuples.append(tuple(bld[1:])) build_list = sorted(build_tuples, key=cmp_to_key(rpmUtils.miscutils.compareEVR)) # reassemble the builds now. Because sigh. nvr_list = [] for build in build_list: nvr_list.append(pkg + '-' + build[1] + '-' + build[2]) return nvr_list def filter_dist(builds, dist): dist_builds = [] for build in builds: if build['nvr'].endswith(args.dist): dist_builds.append(build) return dist_builds def get_args(): parser = argparse.ArgumentParser(description='Bisect koji builds') parser.add_argument('--good', action='store', help='good kernel NVR') parser.add_argument('--bad', action='store', help='bad kernel NVR') parser.add_argument('--list', action='store_true', help='list of builds remaining') parser.add_argument('--dist', action='store', help='disttag of specific release') parser.add_argument('--start', action='store_true', help='start bisecting') parser.add_argument('--reset', action='store_true', help='cancel bisecting') return parser.parse_args() def list_builds(bisect_data): for build in bisect_data['builds_left']: if build == bisect_data['good']: print build + ' good build' elif build == bisect_data['bad']: print build + ' bad build' elif build == bisect_data['current']: print build + ' current build' else: print build def bisect_log(string): global bisect_log_file if bisect_log_file == None: try: bisect_log_file = open('.koji-bisect.log', 'a+') except: print "Wtf. No log. Weird." sys.exit(1) bisect_log_file.write(string + "\n") bisect_log_file.flush() def bisect_save(data): dfile = open('.koji-bisect.data', 'w') cPickle.dump(data, dfile) dfile.flush() dfile.close() if data['good']: bisect_log("Good build %s" % data['good']) if data['bad']: bisect_log("Bad build %s" % data['bad']) if data['current']: bisect_log("Current build %s" % data['current']) if data['builds_left']: bisect_log("Builds left:") for build in data['builds_left']: bisect_log(build) def bisect_load(): bisect_data_def = { 'good': None, 'bad': None, 'current' : None, 'builds_left': None } bisect_data = bisect_data_def try: dfile = open('.koji-bisect.data', 'r') except: return bisect_data try: bisect_data = cPickle.load(dfile) except: bisect_data = bisect_data_def dfile.close() return bisect_data def mark_build(build_list, build, goodbad): try: i = build_list.index(build) print "Marking %s as %s" % (build_list[i], goodbad) bisect_log("Marking %s as %s" % (build_list[i], goodbad)) return build except ValueError: raise if __name__ == "__main__": global options options = Options() options = get_options() args = get_args() if args.reset: if args.start or args.good or args.bad: print "Can't reset and specify something else too" sys.exit(1) # Clear out the files. We dont' care if they didn't exist try: os.remove('.koji-bisect.log') except: pass try: os.remove('.koji-bisect.data') except: pass # remove any existing downloads shutil.rmtree('downloads', ignore_errors=True) sys.exit(0) bisect_data = bisect_load() session = koji.ClientSession(options.server) if bisect_data['builds_left'] == None: activate_session(session) all_builds = session.listBuilds(packageID=8, state=1) builds = all_builds if args.dist: builds = filter_dist(builds, args.dist) build_list = sort_builds(builds) builds_left = build_list bisect_data['builds_left'] = builds_left if args.start: if os.path.exists('.koji-bisect.log') or os.path.exists('.koji-bisect.data'): print "Koji-bisect already in progress. Please reset with --reset" sys.exit(1) else: bisect_log_file = open('.koji-bisect.log', 'w') bisect_data_file = open('.koji-bisect.data', 'w') bisect_log_file.close() bisect_log_file = None bisect_data_file.close() bisect_data_file = None if not args.good and not args.bad: if args.list: list_builds(bisect_data) bisect_save(bisect_data) sys.exit(0) else: print "Must have a good or bad index" bisect_save(bisect_data) sys.exit(1) build_list = bisect_data['builds_left'] if args.good: try: bisect_data['good'] = mark_build(build_list, args.good, "good") except ValueError: print "Invalid build %s. Please pick from list:" % args.good list_builds(bisect_data) sys.exit(1) if args.bad: try: bisect_data['bad'] = mark_build(build_list, args.bad, "bad") except ValueError: print "Invalid build %s. Please pick from list:" % args.bad list_builds(bisect_data) sys.exit(1) if bisect_data['good'] and bisect_data['bad']: builds_left = bisect_data['builds_left'] good = builds_left.index(bisect_data['good']) bad = builds_left.index(bisect_data['bad']) if good < bad: if len(build_list) == bad: builds_left = build_list[good:] else: builds_left = build_list[good:bad+1] elif good == bad: print "Shit yo, is %s bad or good ?!" % build_list[good] else: if len(build_list) == good: builds_left = build_list[bad:] else: builds_left = build_list[bad:good+1] bisect_data['builds_left'] = builds_left current = builds_left[len(builds_left)/2] if current == bisect_data['current']: print "Build %s is the first bad build" % bisect_data['bad'] bisect_save(bisect_data) sys.exit(0) else: bisect_data['current'] = current os.makedirs('downloads/%s' % bisect_data['current']) download_build(session, bisect_data['current'], 'x86_64') print "-------------------------------------------------------" print "%s is now available for install." % bisect_data['current'] bisect_save(bisect_data) if args.list: list_builds(bisect_data)