Add fedpkg module and script

fedpkg is a replacement for the Make system used in dist-cvs.

2 files changed, 538 insertions, 0 deletions

diff --git a/src/fedpkg.py b/src/fedpkg.py
new file mode 100755
index 0000000..6683f65
--- /dev/null
+++ b/src/fedpkg.py
@@ -0,0 +1,309 @@
+#!/usr/bin/python
+# fedpkg - a script to interact with the Fedora Packaging system
+#
+# Copyright (C) 2009 Red Hat Inc.
+# Author(s): Jesse Keating <jkeating@redhat.com>
+# 
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
+# the full text of the license.
+
+import argparse
+import fedpkg
+import os
+
+# Add a simple function to print usage, for the 'help' command
+def usage(args):
+    parser.print_help()
+
+# Define our stub functions
+def build(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def chainbuild(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def check(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def clean(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def clog(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def clone(args):
+    if not args.user:
+        # Use a method to scrape user from fedora cert here
+        args.user = os.getlogin()
+    if args.branches:
+        fedpkg.clone_with_dirs(args.module, args.user)
+    else:
+        fedpkg.clone(args.module, args.user, args.branch)
+
+def compile(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def export(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def gimmespec(args):
+    mymodule = fedpkg.PackageModule(args.path)
+    print(mymodule.spec)
+
+def install(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def lint(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def local(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def mockbuild(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def new(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def new_sources(args):
+    mymodule = fedpkg.PackageModule(args.path)
+    mymodule.new_sources(args.files)
+
+def patch(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def prep(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def scratchbuild(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def sources(args):
+    mymodule = fedpkg.PackageModule(args.path)
+    mymodule.sources(args.outdir)
+
+def srpm(args):
+    mymodule = fedpkg.PackageModule(args.path)
+    mymodule.sources(args.path)
+    if args.md5:
+        mymodule.srpm('md5')
+    else:
+        mymodule.srpm()
+
+def tagrequest(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def unusedfedpatches(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def unusedpatches(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def update(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+def verrel(args):
+    # not implimented
+    print('Not implimented yet, got %s' % args)
+
+# THe main code goes here
+if __name__ == '__main__':
+    # Create the parser object
+    parser = argparse.ArgumentParser(description = 'Fedora Packaging utility')
+
+    # Add top level arguments
+    # Let somebody override the username found in fedora cert
+    parser.add_argument('-u', '--user')
+    # Let the user define which path to look at instead of pwd
+    parser.add_argument('--path', default = os.curdir,
+                    help='Directory to interact with instead of current dir')
+    # Verbosity
+    parser.add_argument('-v', action = 'count',
+                        help = 'Verbosity, may be specified multiple times')
+
+    # Add a subparsers object to use for the actions
+    subparsers = parser.add_subparsers(title = 'Targets')
+
+    # Set up the various actions
+    # Add help to -h and --help
+    parser_help = subparsers.add_parser('help', help = 'Show usage')
+    parser_help.set_defaults(command = usage)
+
+    # build target
+    parser_build = subparsers.add_parser('build',
+                                         help = 'Request build')
+    parser_build.set_defaults(command = build)
+
+    # chain build
+    parser_chainbuild = subparsers.add_parser('chain-build',
+                help = 'Build current package in order with other packages')
+    parser_chainbuild.set_defaults(command = chainbuild)
+
+    # check preps
+    parser_check = subparsers.add_parser('check',
+                                help = 'Check test srpm preps on all arches')
+    parser_check.set_defaults(command = check)
+
+    # clean things up
+    parser_clean = subparsers.add_parser('clean',
+                                         help = 'Remove untracked files')
+    parser_clean.set_defaults(command = clean)
+
+    # Create a changelog stub
+    parser_clog = subparsers.add_parser('clog',
+                    help = 'Make a clog file containing top changelog entry')
+    parser_clog.set_defaults(command = clog)
+
+    # clone take some options, and then passes the rest on to git
+    parser_clone = subparsers.add_parser('clone',
+                                         help = 'Clone and checkout a module')
+    # Allow an old style clone with subdirs for branches
+    parser_clone.add_argument('--branches', '-B',
+                action = 'store_true',
+                help = 'Do an old style checkout with subdirs for branches')
+    # provide a convenient way to get to a specific branch
+    parser_clone.add_argument('--branch', '-b',
+                              help = 'Check out a specific branch')
+    # store the module to be cloned
+    parser_clone.add_argument('--module', '-m', required = True,
+                              help = 'Name of the module to clone')
+    parser_clone.set_defaults(command = clone)
+
+    # compile locally
+    parser_compile = subparsers.add_parser('compile',
+                                        help = 'Local test rpmbuild compile')
+    parser_compile.add_argument('--short-circuit', action = 'store_true',
+                                help = 'short-circuit compile')
+    parser_compile.set_defaults(command = compile)
+
+    # export the module
+    parser_export = subparsers.add_parser('export',
+                                          help = 'Create a clean export')
+    parser_export.set_defaults(command = export)
+
+    # gimmespec takes an optional path argument, defaults to cwd
+    parser_gimmespec = subparsers.add_parser('gimmespec',
+                                             help = 'print spec file name')
+    parser_gimmespec.set_defaults(command = gimmespec)
+
+    # install locally
+    parser_install = subparsers.add_parser('install',
+                                        help = 'Local test rpmbuild install')
+    parser_install.add_argument('--short-circuit', action = 'store_true',
+                                help = 'short-circuit install')
+    parser_install.set_defaults(command = install)
+
+    # rpmlint target
+    parser_lint = subparsers.add_parser('lint',
+                            help = 'Run rpmlint against local build output')
+    parser_lint.set_defaults(command = lint)
+
+    # Build locally
+    parser_local = subparsers.add_parser('local',
+                                         help = 'Local test rpmbuild binary')
+    parser_local.set_defaults(command = local)
+
+    # Build in mock
+    parser_mockbuild = subparsers.add_parser('mockbuild',
+                                        help = 'Local test build using mock')
+    parser_mockbuild.set_defaults(command = mockbuild)
+
+    # See what's different
+    parser_new = subparsers.add_parser('new',
+                                       help = 'Diff against last tag')
+    parser_new.set_defaults(command = new)
+
+    # newsources target takes one or more files as input
+    parser_newsources = subparsers.add_parser('new-sources',
+                                              help = 'Upload new source files')
+    parser_newsources.add_argument('files', nargs = '+')
+    parser_newsources.set_defaults(command = new_sources)
+
+    # patch
+    parser_patch = subparsers.add_parser('patch',
+                                help = 'Create and add a gendiff patch file')
+    parser_patch.add_argument('--suffix')
+    parser_patch.add_argument('--rediff', action = 'store_true',
+                            help = 'Recreate gendiff file retaining comments')
+    parser_patch.set_defaults(command = patch)
+
+    # Prep locally
+    parser_prep = subparsers.add_parser('prep',
+                                        help = 'Local test rpmbuild prep')
+    parser_prep.set_defaults(command = prep)
+
+    # scratch build
+    parser_scratchbuild = subparsers.add_parser('scratch-build',
+                                                help = 'Request scratch build')
+    parser_scratchbuild.add_argument('--arches', nargs = '*',
+                                     help = 'Build for specific arches')
+    parser_scratchbuild.add_argument('--srpm', help='Build from srpm')
+    parser_scratchbuild.set_defaults(command = scratchbuild)
+
+    # sources downloads all the source files, into an optional output dir
+    parser_sources = subparsers.add_parser('sources',
+                                           help = 'Download source files')
+    parser_sources.add_argument('--outdir',
+                default = os.curdir,
+                help = 'Directory to download files into (defaults to pwd)')
+    parser_sources.set_defaults(command = sources)
+
+    # srpm creates a source rpm from the module content
+    parser_srpm = subparsers.add_parser('srpm',
+                                        help = 'Create a source rpm')
+    # optionally define old style hashsums
+    parser_srpm.add_argument('--md5', action = 'store_true',
+                             help = 'Use md5 checksums (for older rpm hosts)')
+    parser_srpm.set_defaults(command = srpm)
+
+    # Create a releng tag request
+    parser_tagrequest = subparsers.add_parser('tag-request',
+                            help = 'Submit last build as a releng tag request')
+    parser_tagrequest.set_defaults(command = tagrequest)
+
+    # Show unused Fedora patches
+    parser_unusedfedpatches = subparsers.add_parser('unused-fedora-patches',
+            help = 'Print Fedora patches not used by Patch and/or ApplyPatch'
+                   ' directives')
+    parser_unusedfedpatches.set_defaults(command = unusedfedpatches)
+
+    # Show unused patches
+    parser_unusedpatches = subparsers.add_parser('unused-patches',
+            help = 'Print list of patches not referenced by name in specfile')
+    parser_unusedpatches.set_defaults(command = unusedpatches)
+
+    # Submit to bodhi for update
+    parser_update = subparsers.add_parser('update',
+                                    help = 'Submit last build as an update')
+    parser_update.set_defaults(command = update)
+
+    # Get version and release
+    parser_verrel = subparsers.add_parser('verrel',
+                                          help = 'Print the version-release')
+    parser_verrel.set_defaults(command = verrel)
+
+    # Parse the args and run the necessary command
+    args = parser.parse_args()
+    args.command(args)
\ No newline at end of file
diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py
new file mode 100644
index 0000000..750bea3
--- /dev/null
+++ b/src/fedpkg/__init__.py
@@ -0,0 +1,229 @@
+# fedpkg - a Python library for Fedora Packagers
+#
+# Copyright (C) 2009 Red Hat Inc.
+# Author(s): Jesse Keating <jkeating@redhat.com>
+# 
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
+# the full text of the license.
+
+import os
+#import pycurl
+import subprocess
+import hashlib
+
+# Define some global variables, put them here to make it easy to change
+LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs'
+LOOKASIDEHASH = 'md5'
+GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s'
+
+# Define some helper functions, they start with _
+def _hash_file(file, hashtype):
+    """Return the hash of a file given a hash type"""
+
+    try:
+        sum = hashlib.new(hashtype)
+    except ValueError:
+        print("Invalid hash type: 0%s" % hashtype)
+        return False
+
+    input = open(file, 'rb')
+    # Loop through the file reading chunks at a time as to not
+    # put the entire file in memory.  That would suck for DVDs
+    while True:
+        chunk = input.read(8192) # magic number!  Taking suggestions
+        if not chunk:
+            break # we're done with the file
+        sum.update(chunk)
+    input.close()
+    return sum.hexdigest()
+
+def _verify_file(file, hash, hashtype):
+    """Given a file, a hash of that file, and a hashtype, verify.
+
+    Returns True if the file verifies, False otherwise
+
+    """
+
+    # get the hash
+    sum = _hash_file(file, hashtype)
+    # now do the comparison
+    if sum == hash:
+        return True
+    return False
+
+def clone(module, user, branch=None):
+    """Clone a repo, optionally check out a specific branch.
+
+    module is the name of the module to clone
+
+    branch is the name of a branch to checkout instead of origin/master
+
+    gitargs is an optional list of arguments to git clone
+
+    """
+
+    # not implemented yet
+    # construct the git url
+    giturl = GITBASEURL % {'user': user, 'module': module}
+    cmd = ['git', 'clone']
+    if branch:
+        cmd.extend(['--branch', branch])
+    cmd.append(giturl)
+    print('Would have ran %s' % subprocess.list2cmdline(cmd))
+    return
+
+def clone_with_dirs(module, user):
+    """Clone a repo old style with subdirs for each branch.
+
+    module is the name of the module to clone
+
+    gitargs is an option list of arguments to git clone
+
+    """
+
+    # not implemented yet
+    print('would have cloned %s with dirs as user %s' % 
+          (module, user))
+    return
+
+# Create a class for package module
+class PackageModule:
+    def _findbranch(self):
+        """Find the branch we're on"""
+        if not os.path.exists(os.path.join(self.path, 'branch')):
+            return 'devel'
+        branch = open(os.path.join(self.path, 'branch'), 'r').read().strip()
+        return branch
+
+    def __init__(self, path=os.curdir):
+        # Initiate a PackageModule object in a given path
+        # Set some global variables used throughout
+        self.path = path
+        self.lookaside = LOOKASIDE
+        self.lookasidehash = LOOKASIDEHASH
+        self.spec = self.gimmespec()
+        self.module = self.spec.split('.spec')[0]
+        # Find the branch and set things based from that
+        # Still requires a 'branch' file in each branch
+        self.branch = self._findbranch()
+        if self.branch.startswith('F-'):
+            self.distval = self.branch.split('-')[1]
+            self.distvar = 'fedora'
+            self.dist = '.fc%s' % self.distval
+        elif self.branch.startswith('EL-'):
+            self.distval = self.branch.split('-')[1]
+            self.distvar = 'epel'
+            self.dist = '.el%s' % self.distval
+        elif self.branch.startswith('OLPC-'):
+            self.distval = self.branch.split('-')[1]
+            self.distvar = 'olpc'
+            self.dist = '.olpc%s' % self.distval
+        # Need to do something about no branch here
+        elif self.branch == 'devel':
+            self.distval = '13' # this is hardset for now, which is bad
+            self.distvar = 'fedora'
+            self.dist = '.fc%s' % self.distval
+        self.rpmdefines = ['--define', '_sourcedir %s' % path,
+                           '--define', '_specdir %s' % path,
+                           '--define', '_builddir %s' % path,
+                           '--define', '_srcrpmdir %s' % path,
+                           '--define', '_rpmdir %s' % path,
+                           '--define', 'dist %s' % self.dist,
+                           '--define', '%s %s' % (self.distvar, self.distval),
+                           '--define', '%s 1' % self.distvar]
+
+    def gimmespec(self):
+        """Print the name of a specfile within a package module"""
+    
+        # Get a list of files in the path we're looking at
+        files = os.listdir(self.path)
+        # Search the files for the first one that ends with ".spec"
+        for f in files:
+            if f.endswith('.spec'):
+                return f
+        return None
+
+    def new_sources(self, files):
+        """Replace source file(s) in the lookaside cache"""
+    
+        # Not fully implimented yet
+        for file in files:
+            hash = _hash_file(file, self.lookasidehash)
+            print "Would upload %s:%s" % (hash, file)
+        return
+               
+    def sources(self, outdir=None):
+        """Download source files"""
+    
+        archives = open(os.path.join(self.path, 'sources'),
+                        'r').readlines()
+        # Default to putting the files where the module is
+        if not outdir:
+            outdir = self.path
+        for archive in archives:
+            csum, file = archive.split()
+            # See if we already have a valid copy downloaded
+            outfile = os.path.join(outdir, file)
+            if os.path.exists(outfile):
+                if _verify_file(outfile, csum, self.lookasidehash):
+                    continue
+            url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module, file, csum,
+                                      file)
+            # There is some code here for using pycurl, but for now,
+            # just use subprocess
+            #output = open(file, 'wb')
+            #curl = pycurl.Curl()
+            #curl.setopt(pycurl.URL, url)
+            #curl.setopt(pycurl.FOLLOWLOCATION, 1)
+            #curl.setopt(pycurl.MAXREDIRS, 5)
+            #curl.setopt(pycurl.CONNECTTIMEOUT, 30)
+            #curl.setopt(pycurl.TIMEOUT, 300)
+            #curl.setopt(pycurl.WRITEDATA, output)
+            #try:
+            #    curl.perform()
+            #except:
+            #    print "Problems downloading %s" % url
+            #    curl.close()
+            #    output.close()
+            #    return 1
+            #curl.close()
+            #output.close()
+            # These options came from Makefile.common.
+            # Probably need to support wget too
+            command = ['curl', '-H',  'Pragma:', '-O', '-R', '-S',  '--fail',
+                       '--show-error', url]
+            try:
+                subprocess.check_call(command, cwd=outdir)
+            except subprocess.CalledProcessError, e:
+                print "Could not download %s: %s" % (url, e)
+                return 1
+            if not _verify_file(outfile, csum, self.lookasidehash):
+                print "%s failed checksum" % file
+                return 1
+        return
+
+    def srpm(self, hashtype='sha256'):
+        """Create an srpm using hashtype from content in the module
+    
+        Requires sources already downloaded.
+    
+        """
+    
+        cmd = ['rpmbuild']
+        cmd.extend(self.rpmdefines)
+        # This may need to get updated if we ever change our checksum default
+        if not hashtype == 'sha256':
+            cmd.extend(['--define',
+                        '_source_filedigest_algorithm %s' % hashtype,
+                        '--define',
+                        '_binary_filedigest_algorithm %s' % hashtype])
+        cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)])
+        try:
+            subprocess.check_call(cmd)
+        except subprocess.CalledProcessError, e:
+            print "Could not build %s: %s" % (self.module, e)
+            return 1
+        return
\ No newline at end of file