path: root/fedpkg-make-pull

#!/usr/bin/python

# fedpkg-make-pull:
# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
# Copyright (C) 2010 Red Hat, Inc.
# Written by Colin Walters <walters@verbum.org>
#
# Using a key in the .spec file "#VCS", support various operations using
# the upstream version control repository:
#
# $ make-pull 
#   Create a .srpm of the latest upstream code
# $ make-pull --apply
#   Patch the existing .spec, sources files for the latest upstream
# $ make-pull --tag 0xdeadbeef
#   Create a .srpm from the tag/branch identifier
# $ make-pull --apply --patch 0xdeadbeef
#   Update the .spec file to include patch 0xdeadbeef from upstream
#

import os
import sys
import re
import urlparse
import getopt
import subprocess
import shutil
import hashlib

class Vcs(object):
    def __init__(self, parsedurl):
        self._parsed_url = parsedurl
        # Deliberately drop params/query
        self._nonfragment_url_string = urlparse.urlunparse((parsedurl.scheme,
                                                            parsedurl.netloc,
                                                            parsedurl.path,
                                                            '', '', ''))
        self._branch = self._parsed_url.fragment
        
    def checkout(self, destdir):
        """Retrieve a new copy of the source tree, saving as destdir"""
        pass
        
    def update(self, directory):
        """Update directory from the latest upstream"""
        pass
        
    def get_scheme(self):
        return self._parsed_url.scheme
        
    def get_id(self, directory):
        pass
        
    def _vcs_exec(self, *args, **kwargs):
        print "Running: %r" % (args[0], )
        kwargs['stdout'] = sys.stdout
        kwargs['stderr'] = subprocess.STDOUT
        subprocess.check_call(*args, **kwargs)
        
    @classmethod
    def new_from_url(cls, url):
        orig = urlparse.urlsplit(url)
        # We want to support fragments, even if the URL type isn't recognized.  So change the
        # scheme to http temporarily.
        temp = urlparse.urlunsplit(('http', orig.netloc, orig.path, orig.query, orig.fragment))
        new = urlparse.urlsplit(temp)
        combined = urlparse.SplitResult(orig.scheme, new.netloc, new.path, new.query, new.fragment)
        if combined.scheme == 'git':
            return GitVcs(combined)
        
class GitVcs(Vcs):
    def checkout(self, destdir):
        self._vcs_exec(['git', 'clone', '--depth=1', self._nonfragment_url_string, destdir])
        if self._branch:
            self._vcs_exec(['git', 'checkout', self._branch], cwd=destdir)
        
    def update(self, directory):
        if self._branch:
            self._vcs_exec(['git', 'checkout', self._branch], cwd=directory)
        self._vcs_exec(['git', 'pull', '-r'], cwd=directory)
        
    def get_id(self, directory):
        output = subprocess.Popen(['git', 'show', '--format=%H'], stdout=subprocess.PIPE, cwd=directory).communicate()[0]
        return output.split('\n')[0]
        
class BuildSystem(object):
    def __init__(self, directory):
        self._directory = directory
        
    @classmethod
    def new_from_directory(cls, directory):
        autogen_path = os.path.join(directory, 'autogen.sh')
        if os.path.exists(autogen_path):
            return AutogenAutotools(directory)
        if os.path.exists(os.path.join(directory, 'Makefile.am')):
            return Autotools(directory)
            
    def get_bootstrap_buildrequires(self):
        return []
        
    def get_substitutions(self):
        return []
        
class Autotools(BuildSystem):
    def get_bootstrap_buildrequires(self):
        return ['libtool', 'automake', 'autoconf']
        
    def get_substitutions(self):
        return [(re.compile('^%configure'), 'autoreconf -f -i\n%configure')]
        
class AutogenAutotools(Autotools):
    def get_bootstrap_buildrequires(self):
        bootstrap = super(AutogenAutotools, self).get_bootstrap_buildrequires()
        bootstrap.append('gnome-common')
        bootstrap.append('intltool')
        return bootstrap
        
    def get_substitutions(self):
        # We'll configure twice with this, but oh well.  Need this in RPM.
        return [(re.compile('^%configure'), './autogen.sh && %configure')]
        
class Spec(object):
    def __init__(self, filename):
        self._filename = filename
        f = open(filename)
        self._lines = f.readlines()
        f.close()
        
        self._append_buildrequires = []
        self._new_release = None
        self._source_dirname = None
        self._source_archivename = None
        self._substitutions = []
        
    def add_buildrequires(self, new_buildrequires):
        current_buildrequires = self.get_key_allvalues('BuildRequires')
        new_buildrequires = filter(lambda x: x not in current_buildrequires, new_buildrequires)
        self._append_buildrequires = new_buildrequires
        
    def increment_release_snapshot(self, identifier):
        cur_release = self.get_key('Release')
        release_has_dist = cur_release.endswith('%{?dist}')
        if release_has_dist:
            cur_release = cur_release[:-8]
        snapshot_release_re = re.compile(r'^([0-9]+)\.([0-9]+)\.')
        numeric_re = re.compile(r'^([0-9]+)$')
        match = snapshot_release_re.match(cur_release)
        if match:
            firstint = int(match.group(1))
            relint = int(match.group(2)) + 1
            new_release = '%d.%d.%s' % (firstint, relint, identifier)
        else:
            match = numeric_re.match(cur_release)
            if not match:
                raise ValueError("Can't handle Release value: %r" % (cur_release, ))
            new_release = '%s.0.%s' % (cur_release, identifier)
        if release_has_dist:
            new_release += '%{?dist}'
            
        self._new_release = new_release
        
    def set_source(self, dirname, archivename):
        self._source_dirname = dirname
        self._source_archivename = archivename
        
    def substitute(self, substitutions):
        self._substitutions = substitutions
        
    def save(self, new_filename):
        wrote_buildrequires = False
        output = open(new_filename, 'w')
        for line in self._lines:
            replacement_matched = False
            for sub_re, replacement in self._substitutions:
                (line, subcount) = sub_re.subn(replacement, line)
                if subcount > 0:
                    replacement_matched = True
                    break
            if replacement_matched:
                output.write(line)
            elif line.startswith('Release:'):
                output.write('Release: %s\n' % self._new_release)
            elif line.startswith('Source0:'):
                output.write('Source0: %s\n' % self._source_archivename)
            elif line.startswith('%setup'):  # This is dumb, need to automate this in RPM
                output.write('%%setup -q -n %s\n' % self._source_dirname)
            elif line.startswith('BuildRequires:') and not wrote_buildrequires:
                output.write(line)
                for req in self._append_buildrequires:
                    output.write('BuildRequires: %s\n' % req)
                wrote_buildrequires = True
            else:
                output.write(line)
                
        output.close()

    def get_version(self):
        return self.get_key('Version')
        
    def get_vcs(self):
        for line in self._lines:
            if line.startswith('#VCS:'):
                return line[5:].strip()
        raise ValueError("No such key #VCS in file %r" % (self._filename, ))
        
    def get_key(self, key):
        key = key + ':'
        for line in self._lines:
            if line.startswith(key):
                return line[len(key):].strip()
        raise ValueError("No such key %r in file %r" % (key, self._filename))
        
    def get_key_allvalues(self, key):
        key = key + ':'
        result = []
        for line in self._lines:
            if line.startswith(key):
                result.append(line[len(key):].strip())
        return result
        
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'f', ['force', 'status-file='])
    except getopt.GetoptError, e:
        print unicode(e)
        print ""
        print "Usage: make-pull.py [-f]"
        sys.exit(1)
        
    force = False
    opt_statusfile = None
    for o, a in opts:
        if o in ('-f', '--force'):
            force = True
        elif o in ('--status-file', ):
            opt_statusfile = a

    targetspec = None
    for filename in os.listdir('.'):
        if filename.endswith('.spec'):
            targetspec = filename
    
    if targetspec is None:
        sys.stderr.write("Couldn't find spec file\n")
        sys.exit(1)
    spec = Spec(targetspec)
    name = targetspec[:-5]
    version = spec.get_version()

    f = open('sources')
    lines = f.readlines()
    f.close()
    if len(lines) != 1:
        print "Must have exactly one source in sources file"
        sys.exit(1)
    
    try:
        vcsurl = spec.get_vcs()
    except ValueError, e:
        sys.stderr.write(unicode(e) + '\n')
        sys.exit(1)
    
    vcs = Vcs.new_from_url(vcsurl)
   
    targetdir = '%s.%s' % (name, vcs.get_scheme())
    
    if os.path.exists(targetdir):
        print "Updating from %r existing directory %r" % (vcsurl, targetdir)
        oldid = vcs.get_id(targetdir)
        vcs.update(targetdir)
        newid = vcs.get_id(targetdir)
        if oldid == newid and not force:
            print "No changes upstream"
            if opt_statusfile is not None:
                f = open(opt_statusfile, 'w')
                f.write('unchanged')
                f.close()
            sys.exit(0)
    else:
        print "Checking out from %r into new directory %r" % (vcsurl, targetdir)
        vcs.checkout(targetdir)
        newid = vcs.get_id(targetdir)
   
    snapshot_dirname = '%s-%s%s%s' % (name, version, vcs.get_scheme(), newid)
    snapshot_archivename = snapshot_dirname + '.tar.bz2'
    subprocess.check_call(['tar', '-cj', r'--transform=s,^\.,' + snapshot_dirname + ',', '-f', '../' + snapshot_archivename, '.'], cwd=targetdir)
    
    new_specname = '%s-%s%s.spec.tmp' % (name, vcs.get_scheme(), newid)
    
    buildsys = BuildSystem.new_from_directory(targetdir)
    if buildsys is None:
        print "WARNING: Unrecognized buildsystem in directory %r" % (targetdir, )
    else:
        spec.add_buildrequires(buildsys.get_bootstrap_buildrequires())
        spec.substitute(buildsys.get_substitutions())

    spec.set_source(snapshot_dirname, snapshot_archivename)
    spec.increment_release_snapshot(newid)
    spec.save(new_specname)

    if False:  # This stuff is buggy...
        tempdir_name = new_specname + '.dir'
        os.mkdir(tempdir_name)
        # Ok, this is a gross hack...parse Source/Patch? from .spec?
        for filename in os.listdir('.'):
            if filename.endswith('.patch'):
                shutil.copy(filename, os.path.join(tempdir_name, filename))
        os.rename(snapshot_archivename, os.path.join(tempdir_name, snapshot_archivename))
        os.rename(new_specname, os.path.join(tempdir_name, targetspec))
        subprocess.check_call(['rpmbuild', '--nodeps', 
                               '--define', '_sourcedir .',
                               '--define', '_specdir .',
                               '--define', '_builddir .',
                               '--define', '_rpmdir .',
                               '--define', '_srcrpmdir .',
                               '-bs', targetspec], cwd=tempdir_name)
        srpm_name = filter(lambda x: x.endswith('.src.rpm'), os.listdir(tempdir_name))[0]
        os.rename(os.path.join(tempdir_name, srpm_name), srpm_name)
        shutil.rmtree(tempdir_name)
        print "Created SRPM: %s" % (srpm_name, )
    elif True:
        os.rename(new_specname, targetspec)
        
        snapshot_md5 = hashlib.md5()
        f = open(snapshot_archivename)
        b = f.read(8192)
        while b != '':
            snapshot_md5.update(b)
            b = f.read(8192)
        f.close()
        
        snapshot_md5 = snapshot_md5.hexdigest()
        
        f = open('sources', 'w')
        f.write(snapshot_md5)
        f.write(' ')
        f.write(snapshot_archivename)
        f.write('\n')
        f.close()
            
        print "Updated %s and sources file" % (targetspec, )
        print "If you want to upload to Fedora, you'll need to run:"
        print "  make upload FILE=%s" % (snapshot_archivename, )
        print "  cvs commit && make tag build"
        if opt_statusfile is not None:
            f = open(opt_statusfile, 'w')
            f.write('updated')
            f.close()   

    sys.exit(0)
        
if __name__ == '__main__':
    main()