path: root/src/fedpkg/__init__.py

# fedpkg - a Python library for Fedora Packagers
#
# Copyright (C) 2009 Red Hat Inc.
# Author(s): Jesse Keating <jkeating@redhat.com>
# 
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.

import os
#import pycurl
import subprocess
import hashlib
import koji
import rpm
import logging
import git

# Define some global variables, put them here to make it easy to change
LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs'
LOOKASIDEHASH = 'md5'
GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s'
ANONGITURL = 'git://pkgs.stg.fedoraproject.org/%(module)s'

# Define our own error class
class FedpkgError(Exception):
    pass

# Setup our logger
# Null logger to avoid spurrious messages, add a handler in app code
class NullHandler(logging.Handler):
    def emit(self, record):
        pass

h = NullHandler()
# This is our log object, clients of this library can use this object to
# define their own logging needs
log = logging.getLogger("fedpkg")
# Add the null handler
log.addHandler(h)

# Define some helper functions, they start with _
def _hash_file(file, hashtype):
    """Return the hash of a file given a hash type"""

    try:
        sum = hashlib.new(hashtype)
    except ValueError:
        raise FedpkgError('Invalid hash type: %s' % hashtype)

    input = open(file, 'rb')
    # Loop through the file reading chunks at a time as to not
    # put the entire file in memory.  That would suck for DVDs
    while True:
        chunk = input.read(8192) # magic number!  Taking suggestions
        if not chunk:
            break # we're done with the file
        sum.update(chunk)
    input.close()
    return sum.hexdigest()

def _verify_file(file, hash, hashtype):
    """Given a file, a hash of that file, and a hashtype, verify.

    Returns True if the file verifies, False otherwise

    """

    # get the hash
    sum = _hash_file(file, hashtype)
    # now do the comparison
    if sum == hash:
        return True
    return False

def _get_build_arches_from_srpm(srpm, arches):
    """Given the path to an srpm, determine the possible build arches

    Use supplied arches as a filter, only return compatible arches

    """

    archlist = arches
    hdr = koji.get_rpm_header(srpm)
    if hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1:
        raise FedpkgError('%s is not a source package.' % srpm)
    buildarchs = hdr[rpm.RPMTAG_BUILDARCHS]
    exclusivearch = hdr[rpm.RPMTAG_EXCLUSIVEARCH]
    excludearch = hdr[rpm.RPMTAG_EXCLUDEARCH]
    # Reduce by buildarchs
    if buildarchs:
        archlist = [a for a in archlist if a in buildarchs]
    # Reduce by exclusive arches
    if exclusivearch:
        archlist = [a for a in archlist if a in exclusivearch]
    # Reduce by exclude arch
    if excludearch:
        archlist = [a for a in archlist if a not in excludearch]
    # do the noarch thing
    if 'noarch' not in excludearch and ('noarch' in buildarchs or \
                                        'noarch' in exclusivearch):
        archlist.append('noarch')
    # See if we have anything compatible.  Should we raise here?
    if not archlist:
        raise FedpkgError('No compatible build arches found in %s' % srpm)
    return archlist

def clean(dry=False, useignore=True):
    """Clean a module checkout of untracked files.

    Can optionally perform a dry-run

    Can optionally not use the ignore rules

    Logs output and returns the returncode

    """

    # setup the command, this could probably be done with some python api...
    cmd = ['git', 'clean', '-f', '-d']
    if dry:
        cmd.append('--dry-run')
    if not useignore:
        cmd.append('-x')
    # Run it!
    log.debug('Running: %s' % subprocess.list2cmdline(cmd))
    try:
        proc = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                                stdout=subprocess.PIPE)
        output, error = proc.communicate()
    except OSError, e:
        raise FedpkgError(e)
    log.info(output)
    if error:
        log.error(error)
    return proc.returncode

def clone(module, user, path=os.getcwd(), branch=None):
    """Clone a repo, optionally check out a specific branch.

    module is the name of the module to clone

    path is the basedir to perform the clone in

    branch is the name of a branch to checkout instead of origin/master

    Logs the output and returns the return code

    """

    # construct the git url
    giturl = GITBASEURL % {'user': user, 'module': module}
    # Create the git object
    mygit = git.Git(path)
    # do the clone and capture the output
    try:
        if branch:
            log.debug('Cloning %s with branch %s' % (giturl, branch))
            retcode, output, error = mygit.clone('--branch', branch,
                                                 giturl,
                                                 with_extended_output=True)
        else:
            log.debug('Cloning %s' % giturl)
            retcode, output, error = mygit.clone(giturl,
                                                 with_extended_output=True)
    except (git.GitCommandError, OSError), e:
        raise FedpkgError('Could not clone %s: %s' % (giturl, e))
    log.info(output)
    if error:
        log.error(error)
    return retcode

def clone_with_dirs(module, user):
    """Clone a repo old style with subdirs for each branch.

    module is the name of the module to clone

    gitargs is an option list of arguments to git clone

    """

    # not implemented yet
    print('would have cloned %s with dirs as user %s' % 
          (module, user))
    return

def new(path=os.getcwd()):
    """Return changes in a repo since the last tag"""

    # setup the repo object based on our path
    try:
        repo = git.Repo(path)
    except git.errors.InvalidGitRepositoryError:
        raise FedpkgError('%s is not a valid repo' % path)
    # Find the latest tag
    tag = repo.git.describe('--tags', '--abbrev=0')
    # Now get the diff
    log.debug('Diffing from tag %s' % tag)
    return repo.git.diff('-M', tag)

# Create a class for package module
class PackageModule:
    def _findbranch(self):
        """Find the branch we're on"""

        if not os.path.exists(os.path.join(self.path, 'branch')):
            return 'devel'
        branch = open(os.path.join(self.path, 'branch'), 'r').read().strip()
        return branch

    def _getlocalarch(self):
        """Get the local arch as defined by rpm"""
        
        return subprocess.Popen(['rpm --eval %{_arch}'], shell=True,
                        stdout=subprocess.PIPE).communicate()[0].strip('\n')

    def __init__(self, path=os.getcwd()):
        # Initiate a PackageModule object in a given path
        # Set some global variables used throughout
        log.debug('Creating module object from %s' % path)
        self.path = path
        self.lookaside = LOOKASIDE
        self.lookasidehash = LOOKASIDEHASH
        self.spec = self.gimmespec()
        self.module = self.spec.split('.spec')[0]
        self.localarch = self._getlocalarch()
        # Set the default mock config to None, not all branches have a config
        self.mockconfig = None
        # Find the branch and set things based from that
        # Still requires a 'branch' file in each branch
        self.branch = self._findbranch()
        if self.branch.startswith('F-'):
            self.distval = self.branch.split('-')[1]
            self.distvar = 'fedora'
            self.dist = '.fc%s' % self.distval
            self.target = 'dist-f%s-updates-candidate' % self.distval
            self.mockconfig = 'fedora-%s-%s' % (self.distval, self.localarch)
        elif self.branch.startswith('EL-'):
            self.distval = self.branch.split('-')[1]
            self.distvar = 'epel'
            self.dist = '.el%s' % self.distval
            self.target = 'dist-%sE-epel-testing-candidate' % self.distval
            self.mockconfig = 'epel-%s-%s' % (self.distval, self.localarch)
        elif self.branch.startswith('OLPC-'):
            self.distval = self.branch.split('-')[1]
            self.distvar = 'olpc'
            self.dist = '.olpc%s' % self.distval
            self.target = 'dist-olpc%s' % self.distval
        # Need to do something about no branch here
        elif self.branch == 'devel':
            self.distval = '13' # this is hardset for now, which is bad
            self.distvar = 'fedora'
            self.dist = '.fc%s' % self.distval
            self.target = 'dist-f%s' % self.distval # will be dist-rawhide
            self.mockconfig = 'fedora-devel-%s' % self.localarch
        self.rpmdefines = ["--define '_sourcedir %s'" % path,
                           "--define '_specdir %s'" % path,
                           "--define '_builddir %s'" % path,
                           "--define '_srcrpmdir %s'" % path,
                           "--define '_rpmdir %s'" % path,
                           "--define 'dist %s'" % self.dist,
                           "--define '%s %s'" % (self.distvar, self.distval),
                           "--define '%s 1'" % self.distvar]
        self.ver = self.getver()
        self.rel = self.getrel()

    def clog(self):
        """Write the latest spec changelog entry to a clog file"""

        # This is a little ugly.  We want to find where %changelog starts,
        # then only deal with the content up to the first empty newline.
        # Then remove any lines that start with $ or %, and then replace
        # %% with %

        # This should probably change behavior from dist-cvs and not print
        # the first line with the date/name/version as git has that info
        # already and it would be redundant.

        cloglines = []
        spec = open(os.path.join(self.path, self.spec), 'r').readlines()
        for line in spec:
            if line.startswith('%changelog'):
                # Grab all the lines below changelog
                for line2 in spec[spec.index(line):]:
                    if line2.startswith('\n'):
                        break
                    if line2.startswith('$'):
                        continue
                    if line2.startswith('%'):
                        continue
                    cloglines.append(line2.replace('%%', '%'))
        # Now open the clog file and write out the lines
        clogfile = open(os.path.join(self.path, 'clog'), 'w')
        clogfile.writelines(cloglines)
        return

    def compile(self, arch=None, short=False):
        """Run rpm -bc on a module

        optionally for a specific arch, or short-circuit it

        Logs the output and returns the returncode

        """

        # Get the sources
        self.sources()
        # setup the rpm command
        cmd = ['rpmbuild']
        cmd.extend(self.rpmdefines)
        if arch:
            cmd.extend(['--target', arch])
        if short:
            cmd.append('--short-circuit')
        cmd.extend(['-bc', os.path.join(self.path, self.spec)])
        # Run the command and capture output
        log.debug('Running: %s' % ' '.join(cmd))
        try:
            proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE, shell=True)
            output, error = proc.communicate()
        except OSError, e:
            raise FedpkgError(e)
        log.info(output)
        if error:
            log.error(error)
        return proc.returncode

    def getver(self):
        """Return the version-release of a package module."""

        cmd = ['rpm']
        cmd.extend(self.rpmdefines)
        cmd.extend(['-q', '--qf', '%{VERSION}', '--specfile',
                    os.path.join(self.path, self.spec)])
        try:
            output = subprocess.Popen(' '.join(cmd), shell=True,
                                      stdout=subprocess.PIPE).communicate()
        except subprocess.CalledProcessError, e:
            raise FedpkgError('Could not get version of %s: %s' % (self.module, e))
        return output[0]

    def getrel(self):
        """Return the version-release of a package module."""

        cmd = ['rpm']
        cmd.extend(self.rpmdefines)
        cmd.extend(['-q', '--qf', '%{RELEASE}', '--specfile',
                    os.path.join(self.path, self.spec)])
        try:
            output = subprocess.Popen(' '.join(cmd), shell=True,
                                      stdout=subprocess.PIPE).communicate()
        except subprocess.CalledProcessError, e:
            raise FedpkgError('Could not get release of %s: %s' % (self.module, e))
        return output[0]

    def gimmespec(self):
        """Return the name of a specfile within a package module"""
    
        # Get a list of files in the path we're looking at
        files = os.listdir(self.path)
        # Search the files for the first one that ends with ".spec"
        for f in files:
            if f.endswith('.spec'):
                return f
        raise FedpkgError('No spec file found.')

    def install(self, arch=None, short=False):
        """Run rpm -bi on a module

        optionally for a specific arch, or short-circuit it

        Logs the output and returns the returncode

        """

        # Get the sources
        self.sources()
        # setup the rpm command
        cmd = ['rpmbuild']
        cmd.extend(self.rpmdefines)
        if arch:
            cmd.extend(['--target', arch])
        if short:
            cmd.append('--short-circuit')
        cmd.extend(['-bi', os.path.join(self.path, self.spec)])
        # Run the command and capture output
        log.debug('Running: %s' % ' '.join(cmd))
        try:
            proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE, shell=True)
            output, error = proc.communicate()
        except OSError, e:
            raise FedpkgError(e)
        log.info(output)
        if error:
            log.error(error)
        return proc.returncode

    def lint(self):
        """Run rpmlint over a built srpm

        Log the output and return the returncode

        """

        # Make sure we have rpms to run on
        srpm = "%s-%s-%s.src.rpm" % (self.module, self.ver, self.rel)
        if not os.path.exists(os.path.join(self.path, srpm)):
            raise FedpkgError('Need to build srpm and rpm first')
        # Get the possible built arches
        arches = _get_build_arches_from_srpm(os.path.join(self.path, srpm),
                                             [self.localarch])
        rpms = []
        for arch in arches:
            rpms.extend([os.path.join(self.path, arch, file) for file in
                         os.listdir(os.path.join(self.path, arch))
                         if file.endswith('.rpm')])
        cmd = ['rpmlint', os.path.join(self.path, srpm)]
        cmd.extend(rpms)
        # Run the command
        log.debug('Running: %s' % subprocess.list2cmdline(cmd))
        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            output, error = proc.communicate()
        except OSError, e:
            raise FedpkgError(e)
        log.info(output)
        if error:
            log.error(error)
        return proc.returncode

    def local(self, arch=None, hashtype='sha256'):
        """rpmbuild locally for given arch.

        Takes arch to build for, and hashtype to build with.

        Writes output to a log file and logs it to the logger

        Returns the returncode from the build call

        """

        # This could really use a list of arches to build for and loop over
        # Get the sources
        self.sources()
        # Determine arch to build for
        if not arch:
            arch = self.localarch
        # build up the rpm command
        cmd = ['rpmbuild']
        cmd.extend(self.rpmdefines)
        # This may need to get updated if we ever change our checksum default
        if not hashtype == 'sha256':
            cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype,
                        "--define '_binary_filedigest_algorithm %s'" % hashtype])
        cmd.extend(['--target', arch, '-ba',
                    os.path.join(self.path, self.spec)])
        # Run the command
        log.debug('Running: %s' % ' '.join(cmd))
        try:
            proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE, shell=True)
            output, error = proc.communicate()
        except OSError, e:
            raise FedpkgError(e)
        outfile = open(os.path.join(self.path, '.build-%s-%s.log' % (self.ver,
                       self.rel)), 'w')
        outfile.writelines(output)
        log.info(output)
        if error:
            outfile.writelines(error)
            log.error(error)
        outfile.close()
        return proc.returncode

    def mockbuild(self, mockargs=[]):
        """Build the package in mock, using mockargs

        Log the output and return the returncode

        """

        # Make sure we have an srpm to run on
        srpm = os.path.join(self.path,
                            "%s-%s-%s.src.rpm" % (self.module,
                                                  self.ver, self.rel))
        if not os.path.exists(srpm):
            raise FedpkgError('Need to build srpm first')

        # setup the command
        cmd = ['mock']
        cmd.extend(mockargs)
        cmd.extend(['-r', self.mockconfig, '--resultdir',
                    os.path.join(self.path, self.module, self.ver, self.rel),
                    '--rebuild', srpm])
        # Run the command
        log.debug('Running: %s' % subprocess.list2cmdline(cmd))
        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            output, error = proc.communicate()
        except OSError, e:
            raise FedpkgError(e)
        log.info(output)
        if error:
            log.error(error)
        return proc.returncode

    def new_sources(self, files):
        """Replace source file(s) in the lookaside cache"""
    
        # Not fully implimented yet
        for file in files:
            hash = _hash_file(file, self.lookasidehash)
            print "Would upload %s:%s" % (hash, file)
        return

    def prep(self, arch=None):
        """Run rpm -bp on a module

        optionally for a specific arch

        Logs the output and returns the returncode from the prep call

        """

        # Get the sources
        self.sources()
        # setup the rpm command
        cmd = ['rpmbuild']
        cmd.extend(self.rpmdefines)
        if arch:
            cmd.extend(['--target', arch])
        cmd.extend(['--nodeps', '-bp', os.path.join(self.path, self.spec)])
        # Run the command and capture output
        log.debug('Running: %s' % ' '.join(cmd))
        try:
            proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE, shell=True)
            output, error = proc.communicate()
        except OSError, e:
            raise FedpkgError(e)
        log.info(output)
        if error:
            log.error(error)
        return proc.returncode
               
    def sources(self, outdir=None):
        """Download source files"""
    
        archives = open(os.path.join(self.path, 'sources'),
                        'r').readlines()
        # Default to putting the files where the module is
        if not outdir:
            outdir = self.path
        for archive in archives:
            csum, file = archive.split()
            # See if we already have a valid copy downloaded
            outfile = os.path.join(outdir, file)
            if os.path.exists(outfile):
                if _verify_file(outfile, csum, self.lookasidehash):
                    continue
            url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module, file, csum,
                                      file)
            # There is some code here for using pycurl, but for now,
            # just use subprocess
            #output = open(file, 'wb')
            #curl = pycurl.Curl()
            #curl.setopt(pycurl.URL, url)
            #curl.setopt(pycurl.FOLLOWLOCATION, 1)
            #curl.setopt(pycurl.MAXREDIRS, 5)
            #curl.setopt(pycurl.CONNECTTIMEOUT, 30)
            #curl.setopt(pycurl.TIMEOUT, 300)
            #curl.setopt(pycurl.WRITEDATA, output)
            #try:
            #    curl.perform()
            #except:
            #    print "Problems downloading %s" % url
            #    curl.close()
            #    output.close()
            #    return 1
            #curl.close()
            #output.close()
            # These options came from Makefile.common.
            # Probably need to support wget too
            command = ['curl', '-H',  'Pragma:', '-O', '-R', '-S',  '--fail',
                       '--show-error', url]
            try:
                subprocess.check_call(command, cwd=outdir)
            except subprocess.CalledProcessError, e:
                raise FedpkgError('Could not download %s: %s' % (url, e))
            if not _verify_file(outfile, csum, self.lookasidehash):
                raise FedpkgError('%s failed checksum' % file)
        return

    def srpm(self, hashtype='sha256'):
        """Create an srpm using hashtype from content in the module
    
        Requires sources already downloaded.
    
        """

        cmd = ['rpmbuild']
        cmd.extend(self.rpmdefines)
        # This may need to get updated if we ever change our checksum default
        if not hashtype == 'sha256':
            cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype,
                    "--define '_binary_filedigest_algorithm %s'" % hashtype])
        cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)])
        try:
            subprocess.check_call(' '.join(cmd), shell=True)
        except subprocess.CalledProcessError, e:
            raise FedpkgError('Could not build %s: %s' % (self.module, e))
        return