From 29d02fe91aa12b5e871617789f3c46304730fd36 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Fri, 5 Feb 2010 17:07:24 -0800 Subject: Rename the module 'pyfedpkg' to save headaches. Having the script and the module named the same was just causing too many weird python issues, so I gave up. --- src/fedpkg/__init__.py | 786 ----------------------------------------------- src/pyfedpkg/__init__.py | 786 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 786 insertions(+), 786 deletions(-) delete mode 100644 src/fedpkg/__init__.py create mode 100644 src/pyfedpkg/__init__.py diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py deleted file mode 100644 index b822388..0000000 --- a/src/fedpkg/__init__.py +++ /dev/null @@ -1,786 +0,0 @@ -# fedpkg - a Python library for Fedora Packagers -# -# Copyright (C) 2009 Red Hat Inc. -# Author(s): Jesse Keating -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - -import os -#import pycurl -import subprocess -import hashlib -import koji -import rpm -import logging -import git -import ConfigParser - -# Define some global variables, put them here to make it easy to change -LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs' -LOOKASIDEHASH = 'md5' -GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s' -ANONGITURL = 'git://pkgs.stg.fedoraproject.org/%(module)s' - -# Define our own error class -class FedpkgError(Exception): - pass - -# Setup our logger -# Null logger to avoid spurrious messages, add a handler in app code -class NullHandler(logging.Handler): - def emit(self, record): - pass - -h = NullHandler() -# This is our log object, clients of this library can use this object to -# define their own logging needs -log = logging.getLogger("fedpkg") -# Add the null handler -log.addHandler(h) - -# Define some helper functions, they start with _ -def _hash_file(file, hashtype): - """Return the hash of a file given a hash type""" - - try: - sum = hashlib.new(hashtype) - except ValueError: - raise FedpkgError('Invalid hash type: %s' % hashtype) - - input = open(file, 'rb') - # Loop through the file reading chunks at a time as to not - # put the entire file in memory. That would suck for DVDs - while True: - chunk = input.read(8192) # magic number! Taking suggestions - if not chunk: - break # we're done with the file - sum.update(chunk) - input.close() - return sum.hexdigest() - -def _verify_file(file, hash, hashtype): - """Given a file, a hash of that file, and a hashtype, verify. - - Returns True if the file verifies, False otherwise - - """ - - # get the hash - sum = _hash_file(file, hashtype) - # now do the comparison - if sum == hash: - return True - return False - -def _get_build_arches_from_srpm(srpm, arches): - """Given the path to an srpm, determine the possible build arches - - Use supplied arches as a filter, only return compatible arches - - """ - - archlist = arches - hdr = koji.get_rpm_header(srpm) - if hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1: - raise FedpkgError('%s is not a source package.' % srpm) - buildarchs = hdr[rpm.RPMTAG_BUILDARCHS] - exclusivearch = hdr[rpm.RPMTAG_EXCLUSIVEARCH] - excludearch = hdr[rpm.RPMTAG_EXCLUDEARCH] - # Reduce by buildarchs - if buildarchs: - archlist = [a for a in archlist if a in buildarchs] - # Reduce by exclusive arches - if exclusivearch: - archlist = [a for a in archlist if a in exclusivearch] - # Reduce by exclude arch - if excludearch: - archlist = [a for a in archlist if a not in excludearch] - # do the noarch thing - if 'noarch' not in excludearch and ('noarch' in buildarchs or \ - 'noarch' in exclusivearch): - archlist.append('noarch') - # See if we have anything compatible. Should we raise here? - if not archlist: - raise FedpkgError('No compatible build arches found in %s' % srpm) - return archlist - -def clean(dry=False, useignore=True): - """Clean a module checkout of untracked files. - - Can optionally perform a dry-run - - Can optionally not use the ignore rules - - Logs output and returns the returncode - - """ - - # setup the command, this could probably be done with some python api... - cmd = ['git', 'clean', '-f', '-d'] - if dry: - cmd.append('--dry-run') - if not useignore: - cmd.append('-x') - # Run it! - log.debug('Running: %s' % subprocess.list2cmdline(cmd)) - try: - proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - log.info(output) - if error: - log.error(error) - return proc.returncode - -def clone(module, user, path=os.getcwd(), branch=None): - """Clone a repo, optionally check out a specific branch. - - module is the name of the module to clone - - path is the basedir to perform the clone in - - branch is the name of a branch to checkout instead of origin/master - - Logs the output and returns the return code - - """ - - # construct the git url - giturl = GITBASEURL % {'user': user, 'module': module} - # Create the git object - mygit = git.Git(path) - # do the clone and capture the output - try: - if branch: - log.debug('Cloning %s with branch %s' % (giturl, branch)) - retcode, output, error = mygit.clone('--branch', branch, - giturl, - with_extended_output=True) - else: - log.debug('Cloning %s' % giturl) - retcode, output, error = mygit.clone(giturl, - with_extended_output=True) - except (git.GitCommandError, OSError), e: - raise FedpkgError('Could not clone %s: %s' % (giturl, e)) - log.info(output) - if error: - log.error(error) - return retcode - -def clone_with_dirs(module, user): - """Clone a repo old style with subdirs for each branch. - - module is the name of the module to clone - - gitargs is an option list of arguments to git clone - - """ - - # not implemented yet - print('would have cloned %s with dirs as user %s' % - (module, user)) - return - -def new(path=os.getcwd()): - """Return changes in a repo since the last tag""" - - # setup the repo object based on our path - try: - repo = git.Repo(path) - except git.errors.InvalidGitRepositoryError: - raise FedpkgError('%s is not a valid repo' % path) - # Find the latest tag - tag = repo.git.describe('--tags', '--abbrev=0') - # Now get the diff - log.debug('Diffing from tag %s' % tag) - return repo.git.diff('-M', tag) - -# Create a class for package module -class PackageModule: - def _findbranch(self): - """Find the branch we're on""" - - if not os.path.exists(os.path.join(self.path, 'branch')): - return 'devel' - branch = open(os.path.join(self.path, 'branch'), 'r').read().strip() - return branch - - def _getlocalarch(self): - """Get the local arch as defined by rpm""" - - return subprocess.Popen(['rpm --eval %{_arch}'], shell=True, - stdout=subprocess.PIPE).communicate()[0].strip('\n') - - def __init__(self, path=os.getcwd()): - # Initiate a PackageModule object in a given path - # Set some global variables used throughout - log.debug('Creating module object from %s' % path) - self.path = path - self.lookaside = LOOKASIDE - self.lookasidehash = LOOKASIDEHASH - self.spec = self.gimmespec() - self.module = self.spec.split('.spec')[0] - self.localarch = self._getlocalarch() - # Set the default mock config to None, not all branches have a config - self.mockconfig = None - # Set a place holder for kojisession - self.kojisession = None - # Find the branch and set things based from that - # Still requires a 'branch' file in each branch - self.branch = self._findbranch() - if self.branch.startswith('F-'): - self.distval = self.branch.split('-')[1] - self.distvar = 'fedora' - self.dist = '.fc%s' % self.distval - self.target = 'dist-f%s-updates-candidate' % self.distval - self.mockconfig = 'fedora-%s-%s' % (self.distval, self.localarch) - elif self.branch.startswith('EL-'): - self.distval = self.branch.split('-')[1] - self.distvar = 'epel' - self.dist = '.el%s' % self.distval - self.target = 'dist-%sE-epel-testing-candidate' % self.distval - self.mockconfig = 'epel-%s-%s' % (self.distval, self.localarch) - elif self.branch.startswith('OLPC-'): - self.distval = self.branch.split('-')[1] - self.distvar = 'olpc' - self.dist = '.olpc%s' % self.distval - self.target = 'dist-olpc%s' % self.distval - # Need to do something about no branch here - elif self.branch == 'devel': - self.distval = '13' # this is hardset for now, which is bad - self.distvar = 'fedora' - self.dist = '.fc%s' % self.distval - self.target = 'dist-f%s' % self.distval # will be dist-rawhide - self.mockconfig = 'fedora-devel-%s' % self.localarch - self.rpmdefines = ["--define '_sourcedir %s'" % path, - "--define '_specdir %s'" % path, - "--define '_builddir %s'" % path, - "--define '_srcrpmdir %s'" % path, - "--define '_rpmdir %s'" % path, - "--define 'dist %s'" % self.dist, - "--define '%s %s'" % (self.distvar, self.distval), - "--define '%s 1'" % self.distvar] - self.ver = self.getver() - self.rel = self.getrel() - try: - self.repo = git.Repo(path) - except git.errors.InvalidGitRepositoryError: - raise FedpkgError('%s is not a valid repo' % path) - - def build(self, skip_tag=False, scratch=False, background=False, - url=None): - """Initiate a build of the module. Available options are: - - skip_tag: Skip the tag action after the build - - scratch: Perform a scratch build - - background: Perform the build with a low priority - - This function submits the task to koji and returns the taskID - - It is up to the client to wait or watch the task. - - """ - - # Make sure we have a valid session. - if not self.kojisession: - raise FedpkgError('No koji session found.') - # construct the url - if not url: - # We don't have a url, so build from the latest commit - # Check to see if the tree is dirty - if self.repo.is_dirty: - raise FedpkgError('There are uncommitted changes in your repo') - # Need to check here to see if the local commit you want to build is - # pushed or not - # This doesn't work if the local branch name doesn't match the remote - if self.repo.git.rev_list('...origin/%s' % self.repo.active_branch): - raise FedpkgError('There are unpushed changes in your repo') - # Get the commit hash to build - commit = self.repo.commits(max_count=1)[0].id - url = ANONGITURL % {'module': self.module} + '#%s' % commit - # Check to see if the target is valid - build_target = self.kojisession.getBuildTarget(self.target) - if not build_target: - raise FedpkgError('Unknown build target: %s' % self.target) - # see if the dest tag is locked - dest_tag = self.kojisession.getTag(build_target['dest_tag_name']) - if not dest_tag: - raise FedpkgError('Unknown destination tag %s' % - build_target['dest_tag_name']) - if dest_tag['locked'] and not scratch: - raise FedpkgError('Destination tag %s is locked' % dest_tag['name']) - # define our dictionary for options - opts = {} - # Set a placeholder for the build priority - priority = None - if skip_tag: - opts['skip_tag'] = True - if scratch: - opts['scratch'] = True - if background: - priority = 5 # magic koji number :/ - - log.debug('Building %s for %s with options %s and a priority of %s' % - (url, self.target, opts, priority)) - # Now submit the task and get the task_id to return - task_id = self.kojisession.build(url, self.target, opts, - priority=priority) - log.info('Created task: %s' % task_id) - log.info('Task info: %s/taskinfo?taskID=%s' % (self.kojiweburl, - task_id)) - return task_id - - def clog(self): - """Write the latest spec changelog entry to a clog file""" - - # This is a little ugly. We want to find where %changelog starts, - # then only deal with the content up to the first empty newline. - # Then remove any lines that start with $ or %, and then replace - # %% with % - - # This should probably change behavior from dist-cvs and not print - # the first line with the date/name/version as git has that info - # already and it would be redundant. - - cloglines = [] - spec = open(os.path.join(self.path, self.spec), 'r').readlines() - for line in spec: - if line.startswith('%changelog'): - # Grab all the lines below changelog - for line2 in spec[spec.index(line):]: - if line2.startswith('\n'): - break - if line2.startswith('$'): - continue - if line2.startswith('%'): - continue - cloglines.append(line2.replace('%%', '%')) - # Now open the clog file and write out the lines - clogfile = open(os.path.join(self.path, 'clog'), 'w') - clogfile.writelines(cloglines) - return - - def compile(self, arch=None, short=False): - """Run rpm -bc on a module - - optionally for a specific arch, or short-circuit it - - Logs the output and returns the returncode - - """ - - # Get the sources - self.sources() - # setup the rpm command - cmd = ['rpmbuild'] - cmd.extend(self.rpmdefines) - if arch: - cmd.extend(['--target', arch]) - if short: - cmd.append('--short-circuit') - cmd.extend(['-bc', os.path.join(self.path, self.spec)]) - # Run the command and capture output - log.debug('Running: %s' % ' '.join(cmd)) - try: - proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, - stdout=subprocess.PIPE, shell=True) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - log.info(output) - if error: - log.error(error) - return proc.returncode - - def getver(self): - """Return the version-release of a package module.""" - - cmd = ['rpm'] - cmd.extend(self.rpmdefines) - cmd.extend(['-q', '--qf', '%{VERSION}', '--specfile', - os.path.join(self.path, self.spec)]) - try: - output = subprocess.Popen(' '.join(cmd), shell=True, - stdout=subprocess.PIPE).communicate() - except subprocess.CalledProcessError, e: - raise FedpkgError('Could not get version of %s: %s' % (self.module, e)) - return output[0] - - def getrel(self): - """Return the version-release of a package module.""" - - cmd = ['rpm'] - cmd.extend(self.rpmdefines) - cmd.extend(['-q', '--qf', '%{RELEASE}', '--specfile', - os.path.join(self.path, self.spec)]) - try: - output = subprocess.Popen(' '.join(cmd), shell=True, - stdout=subprocess.PIPE).communicate() - except subprocess.CalledProcessError, e: - raise FedpkgError('Could not get release of %s: %s' % (self.module, e)) - return output[0] - - def gimmespec(self): - """Return the name of a specfile within a package module""" - - # Get a list of files in the path we're looking at - files = os.listdir(self.path) - # Search the files for the first one that ends with ".spec" - for f in files: - if f.endswith('.spec'): - return f - raise FedpkgError('No spec file found.') - - def koji_upload(self, file, path, callback=None): - """Upload a file to koji - - file is the file you wish to upload - - path is the relative path on the server to upload to - - callback is the progress callback to use, if any - - Returns nothing or raises - - """ - - # See if we actually have a file - if not os.path.exists(file): - raise FedpkgError('No such file: %s' % file) - if not self.kojisession: - raise FedpkgError('No active koji session.') - # This should have a try and catch koji errors - self.kojisession.uploadWrapper(file, path, callback = callback) - return - - def init_koji(self, user, kojiconfig=None, url=None): - """Initiate a koji session. Available options are: - - user: User to log into koji as - - kojiconfig: Use an alternate koji config file - - This function attempts to log in and returns nothing or raises. - - """ - - # Stealing a bunch of code from /usr/bin/koji here, too bad it isn't - # in a more usable library form - defaults = { - 'server' : 'http://localhost/kojihub', - 'weburl' : 'http://localhost/koji', - 'pkgurl' : 'http://localhost/packages', - 'topdir' : '/mnt/koji', - 'cert': '~/.koji/client.crt', - 'ca': '~/.koji/clientca.crt', - 'serverca': '~/.koji/serverca.crt', - 'authtype': None - } - # Process the configs in order, global, user, then any option passed - configs = ['/etc/koji.conf', os.path.expanduser('~/.koji/config')] - if kojiconfig: - configs.append(os.path.join(kojiconfig)) - for configFile in configs: - if os.access(configFile, os.F_OK): - f = open(configFile) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - if config.has_section('koji'): - for name, value in config.items('koji'): - if defaults.has_key(name): - defaults[name] = value - # Expand out the directory options - for name in ('topdir', 'cert', 'ca', 'serverca'): - defaults[name] = os.path.expanduser(defaults[name]) - session_opts = {'user': user} - # We assign the kojisession to our self as it can be used later to - # watch the tasks. - self.kojisession = koji.ClientSession(defaults['server'], session_opts) - # save the weburl for later use too - self.kojiweburl = defaults['weburl'] - # log in using ssl - self.kojisession.ssl_login(defaults['cert'], defaults['ca'], - defaults['serverca']) - if not self.kojisession.logged_in: - raise FedpkgError('Could not auth with koji as %s' % user) - return - - def install(self, arch=None, short=False): - """Run rpm -bi on a module - - optionally for a specific arch, or short-circuit it - - Logs the output and returns the returncode - - """ - - # Get the sources - self.sources() - # setup the rpm command - cmd = ['rpmbuild'] - cmd.extend(self.rpmdefines) - if arch: - cmd.extend(['--target', arch]) - if short: - cmd.append('--short-circuit') - cmd.extend(['-bi', os.path.join(self.path, self.spec)]) - # Run the command and capture output - log.debug('Running: %s' % ' '.join(cmd)) - try: - proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, - stdout=subprocess.PIPE, shell=True) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - log.info(output) - if error: - log.error(error) - return proc.returncode - - def lint(self): - """Run rpmlint over a built srpm - - Log the output and return the returncode - - """ - - # Make sure we have rpms to run on - srpm = "%s-%s-%s.src.rpm" % (self.module, self.ver, self.rel) - if not os.path.exists(os.path.join(self.path, srpm)): - raise FedpkgError('Need to build srpm and rpm first') - # Get the possible built arches - arches = _get_build_arches_from_srpm(os.path.join(self.path, srpm), - [self.localarch]) - rpms = [] - for arch in arches: - rpms.extend([os.path.join(self.path, arch, file) for file in - os.listdir(os.path.join(self.path, arch)) - if file.endswith('.rpm')]) - cmd = ['rpmlint', os.path.join(self.path, srpm)] - cmd.extend(rpms) - # Run the command - log.debug('Running: %s' % subprocess.list2cmdline(cmd)) - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - log.info(output) - if error: - log.error(error) - return proc.returncode - - def local(self, arch=None, hashtype='sha256'): - """rpmbuild locally for given arch. - - Takes arch to build for, and hashtype to build with. - - Writes output to a log file and logs it to the logger - - Returns the returncode from the build call - - """ - - # This could really use a list of arches to build for and loop over - # Get the sources - self.sources() - # Determine arch to build for - if not arch: - arch = self.localarch - # build up the rpm command - cmd = ['rpmbuild'] - cmd.extend(self.rpmdefines) - # This may need to get updated if we ever change our checksum default - if not hashtype == 'sha256': - cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, - "--define '_binary_filedigest_algorithm %s'" % hashtype]) - cmd.extend(['--target', arch, '-ba', - os.path.join(self.path, self.spec)]) - # Run the command - log.debug('Running: %s' % ' '.join(cmd)) - try: - proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, - stdout=subprocess.PIPE, shell=True) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - outfile = open(os.path.join(self.path, '.build-%s-%s.log' % (self.ver, - self.rel)), 'w') - outfile.writelines(output) - log.info(output) - if error: - outfile.writelines(error) - log.error(error) - outfile.close() - return proc.returncode - - def mockbuild(self, mockargs=[]): - """Build the package in mock, using mockargs - - Log the output and return the returncode - - """ - - # Make sure we have an srpm to run on - srpm = os.path.join(self.path, - "%s-%s-%s.src.rpm" % (self.module, - self.ver, self.rel)) - if not os.path.exists(srpm): - raise FedpkgError('Need to build srpm first') - - # setup the command - cmd = ['mock'] - cmd.extend(mockargs) - cmd.extend(['-r', self.mockconfig, '--resultdir', - os.path.join(self.path, self.module, self.ver, self.rel), - '--rebuild', srpm]) - # Run the command - log.debug('Running: %s' % subprocess.list2cmdline(cmd)) - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - log.info(output) - if error: - log.error(error) - return proc.returncode - - def new_sources(self, files): - """Replace source file(s) in the lookaside cache""" - - # Not fully implimented yet - for file in files: - hash = _hash_file(file, self.lookasidehash) - print "Would upload %s:%s" % (hash, file) - return - - def prep(self, arch=None): - """Run rpm -bp on a module - - optionally for a specific arch - - Logs the output and returns the returncode from the prep call - - """ - - # Get the sources - self.sources() - # setup the rpm command - cmd = ['rpmbuild'] - cmd.extend(self.rpmdefines) - if arch: - cmd.extend(['--target', arch]) - cmd.extend(['--nodeps', '-bp', os.path.join(self.path, self.spec)]) - # Run the command and capture output - log.debug('Running: %s' % ' '.join(cmd)) - try: - proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, - stdout=subprocess.PIPE, shell=True) - output, error = proc.communicate() - except OSError, e: - raise FedpkgError(e) - log.info(output) - if error: - log.error(error) - return proc.returncode - - def sources(self, outdir=None): - """Download source files""" - - archives = open(os.path.join(self.path, 'sources'), - 'r').readlines() - # Default to putting the files where the module is - if not outdir: - outdir = self.path - for archive in archives: - csum, file = archive.split() - # See if we already have a valid copy downloaded - outfile = os.path.join(outdir, file) - if os.path.exists(outfile): - if _verify_file(outfile, csum, self.lookasidehash): - continue - url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module, file, csum, - file) - # There is some code here for using pycurl, but for now, - # just use subprocess - #output = open(file, 'wb') - #curl = pycurl.Curl() - #curl.setopt(pycurl.URL, url) - #curl.setopt(pycurl.FOLLOWLOCATION, 1) - #curl.setopt(pycurl.MAXREDIRS, 5) - #curl.setopt(pycurl.CONNECTTIMEOUT, 30) - #curl.setopt(pycurl.TIMEOUT, 300) - #curl.setopt(pycurl.WRITEDATA, output) - #try: - # curl.perform() - #except: - # print "Problems downloading %s" % url - # curl.close() - # output.close() - # return 1 - #curl.close() - #output.close() - # These options came from Makefile.common. - # Probably need to support wget too - command = ['curl', '-H', 'Pragma:', '-O', '-R', '-S', '--fail', - '--show-error', url] - try: - subprocess.check_call(command, cwd=outdir) - except subprocess.CalledProcessError, e: - raise FedpkgError('Could not download %s: %s' % (url, e)) - if not _verify_file(outfile, csum, self.lookasidehash): - raise FedpkgError('%s failed checksum' % file) - return - - def srpm(self, hashtype='sha256'): - """Create an srpm using hashtype from content in the module - - Requires sources already downloaded. - - """ - - cmd = ['rpmbuild'] - cmd.extend(self.rpmdefines) - # This may need to get updated if we ever change our checksum default - if not hashtype == 'sha256': - cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, - "--define '_binary_filedigest_algorithm %s'" % hashtype]) - cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)]) - try: - subprocess.check_call(' '.join(cmd), shell=True) - except subprocess.CalledProcessError, e: - raise FedpkgError('Could not build %s: %s' % (self.module, e)) - return - - def unused_patches(self): - """Discover patches checked into source control that are not used - - Returns a list of unused patches, which may be empty. - - """ - - # Create a list for unused patches - unused = [] - # Get the content of spec into memory for fast searching - spec = open(self.spec, 'r').read() - # Get a list of files tracked in source control - files = self.repo.git.ls_files('--exclude-standard').split() - for file in files: - # throw out non patches - if not file.endswith('.patch'): - continue - if file not in spec: - unused.append(file) - return unused \ No newline at end of file diff --git a/src/pyfedpkg/__init__.py b/src/pyfedpkg/__init__.py new file mode 100644 index 0000000..b822388 --- /dev/null +++ b/src/pyfedpkg/__init__.py @@ -0,0 +1,786 @@ +# fedpkg - a Python library for Fedora Packagers +# +# Copyright (C) 2009 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +import os +#import pycurl +import subprocess +import hashlib +import koji +import rpm +import logging +import git +import ConfigParser + +# Define some global variables, put them here to make it easy to change +LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs' +LOOKASIDEHASH = 'md5' +GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s' +ANONGITURL = 'git://pkgs.stg.fedoraproject.org/%(module)s' + +# Define our own error class +class FedpkgError(Exception): + pass + +# Setup our logger +# Null logger to avoid spurrious messages, add a handler in app code +class NullHandler(logging.Handler): + def emit(self, record): + pass + +h = NullHandler() +# This is our log object, clients of this library can use this object to +# define their own logging needs +log = logging.getLogger("fedpkg") +# Add the null handler +log.addHandler(h) + +# Define some helper functions, they start with _ +def _hash_file(file, hashtype): + """Return the hash of a file given a hash type""" + + try: + sum = hashlib.new(hashtype) + except ValueError: + raise FedpkgError('Invalid hash type: %s' % hashtype) + + input = open(file, 'rb') + # Loop through the file reading chunks at a time as to not + # put the entire file in memory. That would suck for DVDs + while True: + chunk = input.read(8192) # magic number! Taking suggestions + if not chunk: + break # we're done with the file + sum.update(chunk) + input.close() + return sum.hexdigest() + +def _verify_file(file, hash, hashtype): + """Given a file, a hash of that file, and a hashtype, verify. + + Returns True if the file verifies, False otherwise + + """ + + # get the hash + sum = _hash_file(file, hashtype) + # now do the comparison + if sum == hash: + return True + return False + +def _get_build_arches_from_srpm(srpm, arches): + """Given the path to an srpm, determine the possible build arches + + Use supplied arches as a filter, only return compatible arches + + """ + + archlist = arches + hdr = koji.get_rpm_header(srpm) + if hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1: + raise FedpkgError('%s is not a source package.' % srpm) + buildarchs = hdr[rpm.RPMTAG_BUILDARCHS] + exclusivearch = hdr[rpm.RPMTAG_EXCLUSIVEARCH] + excludearch = hdr[rpm.RPMTAG_EXCLUDEARCH] + # Reduce by buildarchs + if buildarchs: + archlist = [a for a in archlist if a in buildarchs] + # Reduce by exclusive arches + if exclusivearch: + archlist = [a for a in archlist if a in exclusivearch] + # Reduce by exclude arch + if excludearch: + archlist = [a for a in archlist if a not in excludearch] + # do the noarch thing + if 'noarch' not in excludearch and ('noarch' in buildarchs or \ + 'noarch' in exclusivearch): + archlist.append('noarch') + # See if we have anything compatible. Should we raise here? + if not archlist: + raise FedpkgError('No compatible build arches found in %s' % srpm) + return archlist + +def clean(dry=False, useignore=True): + """Clean a module checkout of untracked files. + + Can optionally perform a dry-run + + Can optionally not use the ignore rules + + Logs output and returns the returncode + + """ + + # setup the command, this could probably be done with some python api... + cmd = ['git', 'clean', '-f', '-d'] + if dry: + cmd.append('--dry-run') + if not useignore: + cmd.append('-x') + # Run it! + log.debug('Running: %s' % subprocess.list2cmdline(cmd)) + try: + proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + log.info(output) + if error: + log.error(error) + return proc.returncode + +def clone(module, user, path=os.getcwd(), branch=None): + """Clone a repo, optionally check out a specific branch. + + module is the name of the module to clone + + path is the basedir to perform the clone in + + branch is the name of a branch to checkout instead of origin/master + + Logs the output and returns the return code + + """ + + # construct the git url + giturl = GITBASEURL % {'user': user, 'module': module} + # Create the git object + mygit = git.Git(path) + # do the clone and capture the output + try: + if branch: + log.debug('Cloning %s with branch %s' % (giturl, branch)) + retcode, output, error = mygit.clone('--branch', branch, + giturl, + with_extended_output=True) + else: + log.debug('Cloning %s' % giturl) + retcode, output, error = mygit.clone(giturl, + with_extended_output=True) + except (git.GitCommandError, OSError), e: + raise FedpkgError('Could not clone %s: %s' % (giturl, e)) + log.info(output) + if error: + log.error(error) + return retcode + +def clone_with_dirs(module, user): + """Clone a repo old style with subdirs for each branch. + + module is the name of the module to clone + + gitargs is an option list of arguments to git clone + + """ + + # not implemented yet + print('would have cloned %s with dirs as user %s' % + (module, user)) + return + +def new(path=os.getcwd()): + """Return changes in a repo since the last tag""" + + # setup the repo object based on our path + try: + repo = git.Repo(path) + except git.errors.InvalidGitRepositoryError: + raise FedpkgError('%s is not a valid repo' % path) + # Find the latest tag + tag = repo.git.describe('--tags', '--abbrev=0') + # Now get the diff + log.debug('Diffing from tag %s' % tag) + return repo.git.diff('-M', tag) + +# Create a class for package module +class PackageModule: + def _findbranch(self): + """Find the branch we're on""" + + if not os.path.exists(os.path.join(self.path, 'branch')): + return 'devel' + branch = open(os.path.join(self.path, 'branch'), 'r').read().strip() + return branch + + def _getlocalarch(self): + """Get the local arch as defined by rpm""" + + return subprocess.Popen(['rpm --eval %{_arch}'], shell=True, + stdout=subprocess.PIPE).communicate()[0].strip('\n') + + def __init__(self, path=os.getcwd()): + # Initiate a PackageModule object in a given path + # Set some global variables used throughout + log.debug('Creating module object from %s' % path) + self.path = path + self.lookaside = LOOKASIDE + self.lookasidehash = LOOKASIDEHASH + self.spec = self.gimmespec() + self.module = self.spec.split('.spec')[0] + self.localarch = self._getlocalarch() + # Set the default mock config to None, not all branches have a config + self.mockconfig = None + # Set a place holder for kojisession + self.kojisession = None + # Find the branch and set things based from that + # Still requires a 'branch' file in each branch + self.branch = self._findbranch() + if self.branch.startswith('F-'): + self.distval = self.branch.split('-')[1] + self.distvar = 'fedora' + self.dist = '.fc%s' % self.distval + self.target = 'dist-f%s-updates-candidate' % self.distval + self.mockconfig = 'fedora-%s-%s' % (self.distval, self.localarch) + elif self.branch.startswith('EL-'): + self.distval = self.branch.split('-')[1] + self.distvar = 'epel' + self.dist = '.el%s' % self.distval + self.target = 'dist-%sE-epel-testing-candidate' % self.distval + self.mockconfig = 'epel-%s-%s' % (self.distval, self.localarch) + elif self.branch.startswith('OLPC-'): + self.distval = self.branch.split('-')[1] + self.distvar = 'olpc' + self.dist = '.olpc%s' % self.distval + self.target = 'dist-olpc%s' % self.distval + # Need to do something about no branch here + elif self.branch == 'devel': + self.distval = '13' # this is hardset for now, which is bad + self.distvar = 'fedora' + self.dist = '.fc%s' % self.distval + self.target = 'dist-f%s' % self.distval # will be dist-rawhide + self.mockconfig = 'fedora-devel-%s' % self.localarch + self.rpmdefines = ["--define '_sourcedir %s'" % path, + "--define '_specdir %s'" % path, + "--define '_builddir %s'" % path, + "--define '_srcrpmdir %s'" % path, + "--define '_rpmdir %s'" % path, + "--define 'dist %s'" % self.dist, + "--define '%s %s'" % (self.distvar, self.distval), + "--define '%s 1'" % self.distvar] + self.ver = self.getver() + self.rel = self.getrel() + try: + self.repo = git.Repo(path) + except git.errors.InvalidGitRepositoryError: + raise FedpkgError('%s is not a valid repo' % path) + + def build(self, skip_tag=False, scratch=False, background=False, + url=None): + """Initiate a build of the module. Available options are: + + skip_tag: Skip the tag action after the build + + scratch: Perform a scratch build + + background: Perform the build with a low priority + + This function submits the task to koji and returns the taskID + + It is up to the client to wait or watch the task. + + """ + + # Make sure we have a valid session. + if not self.kojisession: + raise FedpkgError('No koji session found.') + # construct the url + if not url: + # We don't have a url, so build from the latest commit + # Check to see if the tree is dirty + if self.repo.is_dirty: + raise FedpkgError('There are uncommitted changes in your repo') + # Need to check here to see if the local commit you want to build is + # pushed or not + # This doesn't work if the local branch name doesn't match the remote + if self.repo.git.rev_list('...origin/%s' % self.repo.active_branch): + raise FedpkgError('There are unpushed changes in your repo') + # Get the commit hash to build + commit = self.repo.commits(max_count=1)[0].id + url = ANONGITURL % {'module': self.module} + '#%s' % commit + # Check to see if the target is valid + build_target = self.kojisession.getBuildTarget(self.target) + if not build_target: + raise FedpkgError('Unknown build target: %s' % self.target) + # see if the dest tag is locked + dest_tag = self.kojisession.getTag(build_target['dest_tag_name']) + if not dest_tag: + raise FedpkgError('Unknown destination tag %s' % + build_target['dest_tag_name']) + if dest_tag['locked'] and not scratch: + raise FedpkgError('Destination tag %s is locked' % dest_tag['name']) + # define our dictionary for options + opts = {} + # Set a placeholder for the build priority + priority = None + if skip_tag: + opts['skip_tag'] = True + if scratch: + opts['scratch'] = True + if background: + priority = 5 # magic koji number :/ + + log.debug('Building %s for %s with options %s and a priority of %s' % + (url, self.target, opts, priority)) + # Now submit the task and get the task_id to return + task_id = self.kojisession.build(url, self.target, opts, + priority=priority) + log.info('Created task: %s' % task_id) + log.info('Task info: %s/taskinfo?taskID=%s' % (self.kojiweburl, + task_id)) + return task_id + + def clog(self): + """Write the latest spec changelog entry to a clog file""" + + # This is a little ugly. We want to find where %changelog starts, + # then only deal with the content up to the first empty newline. + # Then remove any lines that start with $ or %, and then replace + # %% with % + + # This should probably change behavior from dist-cvs and not print + # the first line with the date/name/version as git has that info + # already and it would be redundant. + + cloglines = [] + spec = open(os.path.join(self.path, self.spec), 'r').readlines() + for line in spec: + if line.startswith('%changelog'): + # Grab all the lines below changelog + for line2 in spec[spec.index(line):]: + if line2.startswith('\n'): + break + if line2.startswith('$'): + continue + if line2.startswith('%'): + continue + cloglines.append(line2.replace('%%', '%')) + # Now open the clog file and write out the lines + clogfile = open(os.path.join(self.path, 'clog'), 'w') + clogfile.writelines(cloglines) + return + + def compile(self, arch=None, short=False): + """Run rpm -bc on a module + + optionally for a specific arch, or short-circuit it + + Logs the output and returns the returncode + + """ + + # Get the sources + self.sources() + # setup the rpm command + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + if arch: + cmd.extend(['--target', arch]) + if short: + cmd.append('--short-circuit') + cmd.extend(['-bc', os.path.join(self.path, self.spec)]) + # Run the command and capture output + log.debug('Running: %s' % ' '.join(cmd)) + try: + proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, + stdout=subprocess.PIPE, shell=True) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + log.info(output) + if error: + log.error(error) + return proc.returncode + + def getver(self): + """Return the version-release of a package module.""" + + cmd = ['rpm'] + cmd.extend(self.rpmdefines) + cmd.extend(['-q', '--qf', '%{VERSION}', '--specfile', + os.path.join(self.path, self.spec)]) + try: + output = subprocess.Popen(' '.join(cmd), shell=True, + stdout=subprocess.PIPE).communicate() + except subprocess.CalledProcessError, e: + raise FedpkgError('Could not get version of %s: %s' % (self.module, e)) + return output[0] + + def getrel(self): + """Return the version-release of a package module.""" + + cmd = ['rpm'] + cmd.extend(self.rpmdefines) + cmd.extend(['-q', '--qf', '%{RELEASE}', '--specfile', + os.path.join(self.path, self.spec)]) + try: + output = subprocess.Popen(' '.join(cmd), shell=True, + stdout=subprocess.PIPE).communicate() + except subprocess.CalledProcessError, e: + raise FedpkgError('Could not get release of %s: %s' % (self.module, e)) + return output[0] + + def gimmespec(self): + """Return the name of a specfile within a package module""" + + # Get a list of files in the path we're looking at + files = os.listdir(self.path) + # Search the files for the first one that ends with ".spec" + for f in files: + if f.endswith('.spec'): + return f + raise FedpkgError('No spec file found.') + + def koji_upload(self, file, path, callback=None): + """Upload a file to koji + + file is the file you wish to upload + + path is the relative path on the server to upload to + + callback is the progress callback to use, if any + + Returns nothing or raises + + """ + + # See if we actually have a file + if not os.path.exists(file): + raise FedpkgError('No such file: %s' % file) + if not self.kojisession: + raise FedpkgError('No active koji session.') + # This should have a try and catch koji errors + self.kojisession.uploadWrapper(file, path, callback = callback) + return + + def init_koji(self, user, kojiconfig=None, url=None): + """Initiate a koji session. Available options are: + + user: User to log into koji as + + kojiconfig: Use an alternate koji config file + + This function attempts to log in and returns nothing or raises. + + """ + + # Stealing a bunch of code from /usr/bin/koji here, too bad it isn't + # in a more usable library form + defaults = { + 'server' : 'http://localhost/kojihub', + 'weburl' : 'http://localhost/koji', + 'pkgurl' : 'http://localhost/packages', + 'topdir' : '/mnt/koji', + 'cert': '~/.koji/client.crt', + 'ca': '~/.koji/clientca.crt', + 'serverca': '~/.koji/serverca.crt', + 'authtype': None + } + # Process the configs in order, global, user, then any option passed + configs = ['/etc/koji.conf', os.path.expanduser('~/.koji/config')] + if kojiconfig: + configs.append(os.path.join(kojiconfig)) + for configFile in configs: + if os.access(configFile, os.F_OK): + f = open(configFile) + config = ConfigParser.ConfigParser() + config.readfp(f) + f.close() + if config.has_section('koji'): + for name, value in config.items('koji'): + if defaults.has_key(name): + defaults[name] = value + # Expand out the directory options + for name in ('topdir', 'cert', 'ca', 'serverca'): + defaults[name] = os.path.expanduser(defaults[name]) + session_opts = {'user': user} + # We assign the kojisession to our self as it can be used later to + # watch the tasks. + self.kojisession = koji.ClientSession(defaults['server'], session_opts) + # save the weburl for later use too + self.kojiweburl = defaults['weburl'] + # log in using ssl + self.kojisession.ssl_login(defaults['cert'], defaults['ca'], + defaults['serverca']) + if not self.kojisession.logged_in: + raise FedpkgError('Could not auth with koji as %s' % user) + return + + def install(self, arch=None, short=False): + """Run rpm -bi on a module + + optionally for a specific arch, or short-circuit it + + Logs the output and returns the returncode + + """ + + # Get the sources + self.sources() + # setup the rpm command + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + if arch: + cmd.extend(['--target', arch]) + if short: + cmd.append('--short-circuit') + cmd.extend(['-bi', os.path.join(self.path, self.spec)]) + # Run the command and capture output + log.debug('Running: %s' % ' '.join(cmd)) + try: + proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, + stdout=subprocess.PIPE, shell=True) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + log.info(output) + if error: + log.error(error) + return proc.returncode + + def lint(self): + """Run rpmlint over a built srpm + + Log the output and return the returncode + + """ + + # Make sure we have rpms to run on + srpm = "%s-%s-%s.src.rpm" % (self.module, self.ver, self.rel) + if not os.path.exists(os.path.join(self.path, srpm)): + raise FedpkgError('Need to build srpm and rpm first') + # Get the possible built arches + arches = _get_build_arches_from_srpm(os.path.join(self.path, srpm), + [self.localarch]) + rpms = [] + for arch in arches: + rpms.extend([os.path.join(self.path, arch, file) for file in + os.listdir(os.path.join(self.path, arch)) + if file.endswith('.rpm')]) + cmd = ['rpmlint', os.path.join(self.path, srpm)] + cmd.extend(rpms) + # Run the command + log.debug('Running: %s' % subprocess.list2cmdline(cmd)) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + log.info(output) + if error: + log.error(error) + return proc.returncode + + def local(self, arch=None, hashtype='sha256'): + """rpmbuild locally for given arch. + + Takes arch to build for, and hashtype to build with. + + Writes output to a log file and logs it to the logger + + Returns the returncode from the build call + + """ + + # This could really use a list of arches to build for and loop over + # Get the sources + self.sources() + # Determine arch to build for + if not arch: + arch = self.localarch + # build up the rpm command + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + # This may need to get updated if we ever change our checksum default + if not hashtype == 'sha256': + cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, + "--define '_binary_filedigest_algorithm %s'" % hashtype]) + cmd.extend(['--target', arch, '-ba', + os.path.join(self.path, self.spec)]) + # Run the command + log.debug('Running: %s' % ' '.join(cmd)) + try: + proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, + stdout=subprocess.PIPE, shell=True) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + outfile = open(os.path.join(self.path, '.build-%s-%s.log' % (self.ver, + self.rel)), 'w') + outfile.writelines(output) + log.info(output) + if error: + outfile.writelines(error) + log.error(error) + outfile.close() + return proc.returncode + + def mockbuild(self, mockargs=[]): + """Build the package in mock, using mockargs + + Log the output and return the returncode + + """ + + # Make sure we have an srpm to run on + srpm = os.path.join(self.path, + "%s-%s-%s.src.rpm" % (self.module, + self.ver, self.rel)) + if not os.path.exists(srpm): + raise FedpkgError('Need to build srpm first') + + # setup the command + cmd = ['mock'] + cmd.extend(mockargs) + cmd.extend(['-r', self.mockconfig, '--resultdir', + os.path.join(self.path, self.module, self.ver, self.rel), + '--rebuild', srpm]) + # Run the command + log.debug('Running: %s' % subprocess.list2cmdline(cmd)) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + log.info(output) + if error: + log.error(error) + return proc.returncode + + def new_sources(self, files): + """Replace source file(s) in the lookaside cache""" + + # Not fully implimented yet + for file in files: + hash = _hash_file(file, self.lookasidehash) + print "Would upload %s:%s" % (hash, file) + return + + def prep(self, arch=None): + """Run rpm -bp on a module + + optionally for a specific arch + + Logs the output and returns the returncode from the prep call + + """ + + # Get the sources + self.sources() + # setup the rpm command + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + if arch: + cmd.extend(['--target', arch]) + cmd.extend(['--nodeps', '-bp', os.path.join(self.path, self.spec)]) + # Run the command and capture output + log.debug('Running: %s' % ' '.join(cmd)) + try: + proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE, + stdout=subprocess.PIPE, shell=True) + output, error = proc.communicate() + except OSError, e: + raise FedpkgError(e) + log.info(output) + if error: + log.error(error) + return proc.returncode + + def sources(self, outdir=None): + """Download source files""" + + archives = open(os.path.join(self.path, 'sources'), + 'r').readlines() + # Default to putting the files where the module is + if not outdir: + outdir = self.path + for archive in archives: + csum, file = archive.split() + # See if we already have a valid copy downloaded + outfile = os.path.join(outdir, file) + if os.path.exists(outfile): + if _verify_file(outfile, csum, self.lookasidehash): + continue + url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module, file, csum, + file) + # There is some code here for using pycurl, but for now, + # just use subprocess + #output = open(file, 'wb') + #curl = pycurl.Curl() + #curl.setopt(pycurl.URL, url) + #curl.setopt(pycurl.FOLLOWLOCATION, 1) + #curl.setopt(pycurl.MAXREDIRS, 5) + #curl.setopt(pycurl.CONNECTTIMEOUT, 30) + #curl.setopt(pycurl.TIMEOUT, 300) + #curl.setopt(pycurl.WRITEDATA, output) + #try: + # curl.perform() + #except: + # print "Problems downloading %s" % url + # curl.close() + # output.close() + # return 1 + #curl.close() + #output.close() + # These options came from Makefile.common. + # Probably need to support wget too + command = ['curl', '-H', 'Pragma:', '-O', '-R', '-S', '--fail', + '--show-error', url] + try: + subprocess.check_call(command, cwd=outdir) + except subprocess.CalledProcessError, e: + raise FedpkgError('Could not download %s: %s' % (url, e)) + if not _verify_file(outfile, csum, self.lookasidehash): + raise FedpkgError('%s failed checksum' % file) + return + + def srpm(self, hashtype='sha256'): + """Create an srpm using hashtype from content in the module + + Requires sources already downloaded. + + """ + + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + # This may need to get updated if we ever change our checksum default + if not hashtype == 'sha256': + cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, + "--define '_binary_filedigest_algorithm %s'" % hashtype]) + cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)]) + try: + subprocess.check_call(' '.join(cmd), shell=True) + except subprocess.CalledProcessError, e: + raise FedpkgError('Could not build %s: %s' % (self.module, e)) + return + + def unused_patches(self): + """Discover patches checked into source control that are not used + + Returns a list of unused patches, which may be empty. + + """ + + # Create a list for unused patches + unused = [] + # Get the content of spec into memory for fast searching + spec = open(self.spec, 'r').read() + # Get a list of files tracked in source control + files = self.repo.git.ls_files('--exclude-standard').split() + for file in files: + # throw out non patches + if not file.endswith('.patch'): + continue + if file not in spec: + unused.append(file) + return unused \ No newline at end of file -- cgit