From 4fc1cfc6549a5f2850d16ea39d185dc17b2c70ec Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 11:04:15 -0800 Subject: Add fedpkg module and script fedpkg is a replacement for the Make system used in dist-cvs. --- src/fedpkg.py | 309 +++++++++++++++++++++++++++++++++++++++++++++++++ src/fedpkg/__init__.py | 229 ++++++++++++++++++++++++++++++++++++ 2 files changed, 538 insertions(+) create mode 100755 src/fedpkg.py create mode 100644 src/fedpkg/__init__.py (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py new file mode 100755 index 0000000..6683f65 --- /dev/null +++ b/src/fedpkg.py @@ -0,0 +1,309 @@ +#!/usr/bin/python +# fedpkg - a script to interact with the Fedora Packaging system +# +# Copyright (C) 2009 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +import argparse +import fedpkg +import os + +# Add a simple function to print usage, for the 'help' command +def usage(args): + parser.print_help() + +# Define our stub functions +def build(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def chainbuild(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def check(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def clean(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def clog(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def clone(args): + if not args.user: + # Use a method to scrape user from fedora cert here + args.user = os.getlogin() + if args.branches: + fedpkg.clone_with_dirs(args.module, args.user) + else: + fedpkg.clone(args.module, args.user, args.branch) + +def compile(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def export(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def gimmespec(args): + mymodule = fedpkg.PackageModule(args.path) + print(mymodule.spec) + +def install(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def lint(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def local(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def mockbuild(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def new(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def new_sources(args): + mymodule = fedpkg.PackageModule(args.path) + mymodule.new_sources(args.files) + +def patch(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def prep(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def scratchbuild(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def sources(args): + mymodule = fedpkg.PackageModule(args.path) + mymodule.sources(args.outdir) + +def srpm(args): + mymodule = fedpkg.PackageModule(args.path) + mymodule.sources(args.path) + if args.md5: + mymodule.srpm('md5') + else: + mymodule.srpm() + +def tagrequest(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def unusedfedpatches(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def unusedpatches(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def update(args): + # not implimented + print('Not implimented yet, got %s' % args) + +def verrel(args): + # not implimented + print('Not implimented yet, got %s' % args) + +# THe main code goes here +if __name__ == '__main__': + # Create the parser object + parser = argparse.ArgumentParser(description = 'Fedora Packaging utility') + + # Add top level arguments + # Let somebody override the username found in fedora cert + parser.add_argument('-u', '--user') + # Let the user define which path to look at instead of pwd + parser.add_argument('--path', default = os.curdir, + help='Directory to interact with instead of current dir') + # Verbosity + parser.add_argument('-v', action = 'count', + help = 'Verbosity, may be specified multiple times') + + # Add a subparsers object to use for the actions + subparsers = parser.add_subparsers(title = 'Targets') + + # Set up the various actions + # Add help to -h and --help + parser_help = subparsers.add_parser('help', help = 'Show usage') + parser_help.set_defaults(command = usage) + + # build target + parser_build = subparsers.add_parser('build', + help = 'Request build') + parser_build.set_defaults(command = build) + + # chain build + parser_chainbuild = subparsers.add_parser('chain-build', + help = 'Build current package in order with other packages') + parser_chainbuild.set_defaults(command = chainbuild) + + # check preps + parser_check = subparsers.add_parser('check', + help = 'Check test srpm preps on all arches') + parser_check.set_defaults(command = check) + + # clean things up + parser_clean = subparsers.add_parser('clean', + help = 'Remove untracked files') + parser_clean.set_defaults(command = clean) + + # Create a changelog stub + parser_clog = subparsers.add_parser('clog', + help = 'Make a clog file containing top changelog entry') + parser_clog.set_defaults(command = clog) + + # clone take some options, and then passes the rest on to git + parser_clone = subparsers.add_parser('clone', + help = 'Clone and checkout a module') + # Allow an old style clone with subdirs for branches + parser_clone.add_argument('--branches', '-B', + action = 'store_true', + help = 'Do an old style checkout with subdirs for branches') + # provide a convenient way to get to a specific branch + parser_clone.add_argument('--branch', '-b', + help = 'Check out a specific branch') + # store the module to be cloned + parser_clone.add_argument('--module', '-m', required = True, + help = 'Name of the module to clone') + parser_clone.set_defaults(command = clone) + + # compile locally + parser_compile = subparsers.add_parser('compile', + help = 'Local test rpmbuild compile') + parser_compile.add_argument('--short-circuit', action = 'store_true', + help = 'short-circuit compile') + parser_compile.set_defaults(command = compile) + + # export the module + parser_export = subparsers.add_parser('export', + help = 'Create a clean export') + parser_export.set_defaults(command = export) + + # gimmespec takes an optional path argument, defaults to cwd + parser_gimmespec = subparsers.add_parser('gimmespec', + help = 'print spec file name') + parser_gimmespec.set_defaults(command = gimmespec) + + # install locally + parser_install = subparsers.add_parser('install', + help = 'Local test rpmbuild install') + parser_install.add_argument('--short-circuit', action = 'store_true', + help = 'short-circuit install') + parser_install.set_defaults(command = install) + + # rpmlint target + parser_lint = subparsers.add_parser('lint', + help = 'Run rpmlint against local build output') + parser_lint.set_defaults(command = lint) + + # Build locally + parser_local = subparsers.add_parser('local', + help = 'Local test rpmbuild binary') + parser_local.set_defaults(command = local) + + # Build in mock + parser_mockbuild = subparsers.add_parser('mockbuild', + help = 'Local test build using mock') + parser_mockbuild.set_defaults(command = mockbuild) + + # See what's different + parser_new = subparsers.add_parser('new', + help = 'Diff against last tag') + parser_new.set_defaults(command = new) + + # newsources target takes one or more files as input + parser_newsources = subparsers.add_parser('new-sources', + help = 'Upload new source files') + parser_newsources.add_argument('files', nargs = '+') + parser_newsources.set_defaults(command = new_sources) + + # patch + parser_patch = subparsers.add_parser('patch', + help = 'Create and add a gendiff patch file') + parser_patch.add_argument('--suffix') + parser_patch.add_argument('--rediff', action = 'store_true', + help = 'Recreate gendiff file retaining comments') + parser_patch.set_defaults(command = patch) + + # Prep locally + parser_prep = subparsers.add_parser('prep', + help = 'Local test rpmbuild prep') + parser_prep.set_defaults(command = prep) + + # scratch build + parser_scratchbuild = subparsers.add_parser('scratch-build', + help = 'Request scratch build') + parser_scratchbuild.add_argument('--arches', nargs = '*', + help = 'Build for specific arches') + parser_scratchbuild.add_argument('--srpm', help='Build from srpm') + parser_scratchbuild.set_defaults(command = scratchbuild) + + # sources downloads all the source files, into an optional output dir + parser_sources = subparsers.add_parser('sources', + help = 'Download source files') + parser_sources.add_argument('--outdir', + default = os.curdir, + help = 'Directory to download files into (defaults to pwd)') + parser_sources.set_defaults(command = sources) + + # srpm creates a source rpm from the module content + parser_srpm = subparsers.add_parser('srpm', + help = 'Create a source rpm') + # optionally define old style hashsums + parser_srpm.add_argument('--md5', action = 'store_true', + help = 'Use md5 checksums (for older rpm hosts)') + parser_srpm.set_defaults(command = srpm) + + # Create a releng tag request + parser_tagrequest = subparsers.add_parser('tag-request', + help = 'Submit last build as a releng tag request') + parser_tagrequest.set_defaults(command = tagrequest) + + # Show unused Fedora patches + parser_unusedfedpatches = subparsers.add_parser('unused-fedora-patches', + help = 'Print Fedora patches not used by Patch and/or ApplyPatch' + ' directives') + parser_unusedfedpatches.set_defaults(command = unusedfedpatches) + + # Show unused patches + parser_unusedpatches = subparsers.add_parser('unused-patches', + help = 'Print list of patches not referenced by name in specfile') + parser_unusedpatches.set_defaults(command = unusedpatches) + + # Submit to bodhi for update + parser_update = subparsers.add_parser('update', + help = 'Submit last build as an update') + parser_update.set_defaults(command = update) + + # Get version and release + parser_verrel = subparsers.add_parser('verrel', + help = 'Print the version-release') + parser_verrel.set_defaults(command = verrel) + + # Parse the args and run the necessary command + args = parser.parse_args() + args.command(args) \ No newline at end of file diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py new file mode 100644 index 0000000..750bea3 --- /dev/null +++ b/src/fedpkg/__init__.py @@ -0,0 +1,229 @@ +# fedpkg - a Python library for Fedora Packagers +# +# Copyright (C) 2009 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +import os +#import pycurl +import subprocess +import hashlib + +# Define some global variables, put them here to make it easy to change +LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs' +LOOKASIDEHASH = 'md5' +GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s' + +# Define some helper functions, they start with _ +def _hash_file(file, hashtype): + """Return the hash of a file given a hash type""" + + try: + sum = hashlib.new(hashtype) + except ValueError: + print("Invalid hash type: 0%s" % hashtype) + return False + + input = open(file, 'rb') + # Loop through the file reading chunks at a time as to not + # put the entire file in memory. That would suck for DVDs + while True: + chunk = input.read(8192) # magic number! Taking suggestions + if not chunk: + break # we're done with the file + sum.update(chunk) + input.close() + return sum.hexdigest() + +def _verify_file(file, hash, hashtype): + """Given a file, a hash of that file, and a hashtype, verify. + + Returns True if the file verifies, False otherwise + + """ + + # get the hash + sum = _hash_file(file, hashtype) + # now do the comparison + if sum == hash: + return True + return False + +def clone(module, user, branch=None): + """Clone a repo, optionally check out a specific branch. + + module is the name of the module to clone + + branch is the name of a branch to checkout instead of origin/master + + gitargs is an optional list of arguments to git clone + + """ + + # not implemented yet + # construct the git url + giturl = GITBASEURL % {'user': user, 'module': module} + cmd = ['git', 'clone'] + if branch: + cmd.extend(['--branch', branch]) + cmd.append(giturl) + print('Would have ran %s' % subprocess.list2cmdline(cmd)) + return + +def clone_with_dirs(module, user): + """Clone a repo old style with subdirs for each branch. + + module is the name of the module to clone + + gitargs is an option list of arguments to git clone + + """ + + # not implemented yet + print('would have cloned %s with dirs as user %s' % + (module, user)) + return + +# Create a class for package module +class PackageModule: + def _findbranch(self): + """Find the branch we're on""" + if not os.path.exists(os.path.join(self.path, 'branch')): + return 'devel' + branch = open(os.path.join(self.path, 'branch'), 'r').read().strip() + return branch + + def __init__(self, path=os.curdir): + # Initiate a PackageModule object in a given path + # Set some global variables used throughout + self.path = path + self.lookaside = LOOKASIDE + self.lookasidehash = LOOKASIDEHASH + self.spec = self.gimmespec() + self.module = self.spec.split('.spec')[0] + # Find the branch and set things based from that + # Still requires a 'branch' file in each branch + self.branch = self._findbranch() + if self.branch.startswith('F-'): + self.distval = self.branch.split('-')[1] + self.distvar = 'fedora' + self.dist = '.fc%s' % self.distval + elif self.branch.startswith('EL-'): + self.distval = self.branch.split('-')[1] + self.distvar = 'epel' + self.dist = '.el%s' % self.distval + elif self.branch.startswith('OLPC-'): + self.distval = self.branch.split('-')[1] + self.distvar = 'olpc' + self.dist = '.olpc%s' % self.distval + # Need to do something about no branch here + elif self.branch == 'devel': + self.distval = '13' # this is hardset for now, which is bad + self.distvar = 'fedora' + self.dist = '.fc%s' % self.distval + self.rpmdefines = ['--define', '_sourcedir %s' % path, + '--define', '_specdir %s' % path, + '--define', '_builddir %s' % path, + '--define', '_srcrpmdir %s' % path, + '--define', '_rpmdir %s' % path, + '--define', 'dist %s' % self.dist, + '--define', '%s %s' % (self.distvar, self.distval), + '--define', '%s 1' % self.distvar] + + def gimmespec(self): + """Print the name of a specfile within a package module""" + + # Get a list of files in the path we're looking at + files = os.listdir(self.path) + # Search the files for the first one that ends with ".spec" + for f in files: + if f.endswith('.spec'): + return f + return None + + def new_sources(self, files): + """Replace source file(s) in the lookaside cache""" + + # Not fully implimented yet + for file in files: + hash = _hash_file(file, self.lookasidehash) + print "Would upload %s:%s" % (hash, file) + return + + def sources(self, outdir=None): + """Download source files""" + + archives = open(os.path.join(self.path, 'sources'), + 'r').readlines() + # Default to putting the files where the module is + if not outdir: + outdir = self.path + for archive in archives: + csum, file = archive.split() + # See if we already have a valid copy downloaded + outfile = os.path.join(outdir, file) + if os.path.exists(outfile): + if _verify_file(outfile, csum, self.lookasidehash): + continue + url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module, file, csum, + file) + # There is some code here for using pycurl, but for now, + # just use subprocess + #output = open(file, 'wb') + #curl = pycurl.Curl() + #curl.setopt(pycurl.URL, url) + #curl.setopt(pycurl.FOLLOWLOCATION, 1) + #curl.setopt(pycurl.MAXREDIRS, 5) + #curl.setopt(pycurl.CONNECTTIMEOUT, 30) + #curl.setopt(pycurl.TIMEOUT, 300) + #curl.setopt(pycurl.WRITEDATA, output) + #try: + # curl.perform() + #except: + # print "Problems downloading %s" % url + # curl.close() + # output.close() + # return 1 + #curl.close() + #output.close() + # These options came from Makefile.common. + # Probably need to support wget too + command = ['curl', '-H', 'Pragma:', '-O', '-R', '-S', '--fail', + '--show-error', url] + try: + subprocess.check_call(command, cwd=outdir) + except subprocess.CalledProcessError, e: + print "Could not download %s: %s" % (url, e) + return 1 + if not _verify_file(outfile, csum, self.lookasidehash): + print "%s failed checksum" % file + return 1 + return + + def srpm(self, hashtype='sha256'): + """Create an srpm using hashtype from content in the module + + Requires sources already downloaded. + + """ + + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + # This may need to get updated if we ever change our checksum default + if not hashtype == 'sha256': + cmd.extend(['--define', + '_source_filedigest_algorithm %s' % hashtype, + '--define', + '_binary_filedigest_algorithm %s' % hashtype]) + cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)]) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError, e: + print "Could not build %s: %s" % (self.module, e) + return 1 + return \ No newline at end of file -- cgit From 8da0910f372a68bbef0315a2a7c427bd69f2d8ab Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 11:53:07 -0800 Subject: Add getver and getrel Useful for getting the verrel --- src/fedpkg/__init__.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 750bea3..b84a85a 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -134,6 +134,38 @@ class PackageModule: '--define', 'dist %s' % self.dist, '--define', '%s %s' % (self.distvar, self.distval), '--define', '%s 1' % self.distvar] + self.ver = self.getver() + self.rel = self.getrel() + + def getver(self): + """Return the version-release of a package module.""" + + cmd = ['rpm'] + cmd.extend(self.rpmdefines) + cmd.extend(['-q', '--qf', '%{VERSION}', '--specfile', + os.path.join(self.path, self.spec)]) + try: + output = subprocess.Popen(cmd, + stdout=subprocess.PIPE).communicate() + except subprocess.CalledProcessError, e: + print("Could not get version-release of %s: %s" % (self.module, e)) + return 1 + return output[0] + + def getrel(self): + """Return the version-release of a package module.""" + + cmd = ['rpm'] + cmd.extend(self.rpmdefines) + cmd.extend(['-q', '--qf', '%{RELEASE}', '--specfile', + os.path.join(self.path, self.spec)]) + try: + output = subprocess.Popen(cmd, + stdout=subprocess.PIPE).communicate() + except subprocess.CalledProcessError, e: + print("Could not get version-release of %s: %s" % (self.module, e)) + return 1 + return output[0] def gimmespec(self): """Print the name of a specfile within a package module""" -- cgit From 466878ceaddb00c4ba748c1c091f8c235fc5759f Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 11:53:20 -0800 Subject: Wire up the verrel command --- src/fedpkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index 6683f65..93a4337 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -125,8 +125,8 @@ def update(args): print('Not implimented yet, got %s' % args) def verrel(args): - # not implimented - print('Not implimented yet, got %s' % args) + mymodule = fedpkg.PackageModule(args.path) + print('%s-%s' % (mymodule.ver, mymodule.rel)) # THe main code goes here if __name__ == '__main__': -- cgit From 2129555c4d841a1781bc69c7c222f5c9a563433d Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 11:54:46 -0800 Subject: Add the name- to verrel, that's what make does --- src/fedpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index 93a4337..b84b81d 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -126,7 +126,7 @@ def update(args): def verrel(args): mymodule = fedpkg.PackageModule(args.path) - print('%s-%s' % (mymodule.ver, mymodule.rel)) + print('%s-%s-%s' % (mymodule.module, mymodule.ver, mymodule.rel)) # THe main code goes here if __name__ == '__main__': -- cgit From e74c9cf416b50d4a1c286af187a47e3296c91308 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 11:55:54 -0800 Subject: Update the help output on verrel to match reality --- src/fedpkg.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index b84b81d..c6b5115 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -301,7 +301,8 @@ if __name__ == '__main__': # Get version and release parser_verrel = subparsers.add_parser('verrel', - help = 'Print the version-release') + help = 'Print the' + ' name-version-release') parser_verrel.set_defaults(command = verrel) # Parse the args and run the necessary command -- cgit From 14cc21fc32e8e44d4278aaeaa5136e2b1bb66b1c Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:10:37 -0800 Subject: Add a lint function. Needs better error handling, will add a custom error class --- src/fedpkg/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index b84a85a..ceffbbc 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -178,6 +178,22 @@ class PackageModule: return f return None + def lint(self): + """Run rpmlint over a built srpm""" + + # Make sure we have a srpm to run on + srpm = "%s-%s-%s.src.rpm" % (self.module, self.ver, self.rel) + rpm = "%s-%s-%s.%s.rpm" % (self.module, self.ver, self.rel, + os.uname()[4]) + if not os.path.exists(os.path.join(self.path, srpm)) and not \ + os.path.exists(os.path.join(self.path, rpm)): + return "Need to build srpm and rpm first" + cmd = ['rpmlint', os.path.join(self.path, srpm), + os.path.join(self.path, rpm)] + output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() + return output[0] + + def new_sources(self, files): """Replace source file(s) in the lookaside cache""" -- cgit From 536ca47f4b5ba5ad9f968cb10c30cc191549e486 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:10:45 -0800 Subject: Wire up lint in fedpkg.py --- src/fedpkg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index c6b5115..f2131cf 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -65,8 +65,8 @@ def install(args): print('Not implimented yet, got %s' % args) def lint(args): - # not implimented - print('Not implimented yet, got %s' % args) + mymodule = fedpkg.PackageModule(args.path) + print(mymodule.lint()) def local(args): # not implimented -- cgit From 76cb0035e4928cf7f6c32e239cd35515c0a92161 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:17:45 -0800 Subject: Add a custom error class This makes it easier to catch errors in the calling tools. They won't have to know about the internals of the library, just catch the custom error and deal with it. --- src/fedpkg/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index ceffbbc..17ea192 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -19,6 +19,10 @@ LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs' LOOKASIDEHASH = 'md5' GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s' +# Define our own error class +class FedpkgError(Exception): + pass + # Define some helper functions, they start with _ def _hash_file(file, hashtype): """Return the hash of a file given a hash type""" @@ -187,7 +191,7 @@ class PackageModule: os.uname()[4]) if not os.path.exists(os.path.join(self.path, srpm)) and not \ os.path.exists(os.path.join(self.path, rpm)): - return "Need to build srpm and rpm first" + raise FedpkgError("Need to build srpm and rpm first") cmd = ['rpmlint', os.path.join(self.path, srpm), os.path.join(self.path, rpm)] output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() -- cgit From fe2f05f3d8f0a14ec70cc6ad768f8f24225b290a Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:17:56 -0800 Subject: Catch an error in the lint call --- src/fedpkg.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index f2131cf..e80b982 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -66,7 +66,11 @@ def install(args): def lint(args): mymodule = fedpkg.PackageModule(args.path) - print(mymodule.lint()) + try: + print(mymodule.lint()) + except fedpkg.FedpkgError, e: + print('Could not run rpmlint: %s' % e) + return 1 def local(args): # not implimented -- cgit From e71b6bebab3df14e75f12990f7914bc8b36f5347 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:30:34 -0800 Subject: Catch subprocess errors and re-raise our error --- src/fedpkg/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 17ea192..0e668b9 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -194,7 +194,10 @@ class PackageModule: raise FedpkgError("Need to build srpm and rpm first") cmd = ['rpmlint', os.path.join(self.path, srpm), os.path.join(self.path, rpm)] - output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() + try: + output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() + except subprocess.CalledProcessError, e: + raise FedpkgError(e) return output[0] -- cgit From 7d0c07c2170c16b4b4502cafee38f56fcede80b3 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:39:24 -0800 Subject: Use FedpkgError for error checking --- src/fedpkg/__init__.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 0e668b9..05d83ff 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -30,8 +30,7 @@ def _hash_file(file, hashtype): try: sum = hashlib.new(hashtype) except ValueError: - print("Invalid hash type: 0%s" % hashtype) - return False + raise FedpkgError('Invalid hash type: %s' % hashtype) input = open(file, 'rb') # Loop through the file reading chunks at a time as to not @@ -152,8 +151,7 @@ class PackageModule: output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() except subprocess.CalledProcessError, e: - print("Could not get version-release of %s: %s" % (self.module, e)) - return 1 + raise FedpkgError('Could not get version-release of %s: %s' % (self.module, e)) return output[0] def getrel(self): @@ -167,8 +165,7 @@ class PackageModule: output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() except subprocess.CalledProcessError, e: - print("Could not get version-release of %s: %s" % (self.module, e)) - return 1 + raise FedpkgError('Could not get version-release of %s: %s' % (self.module, e)) return output[0] def gimmespec(self): @@ -180,7 +177,7 @@ class PackageModule: for f in files: if f.endswith('.spec'): return f - return None + raise FedpkgError('No spec file found.') def lint(self): """Run rpmlint over a built srpm""" @@ -191,7 +188,7 @@ class PackageModule: os.uname()[4]) if not os.path.exists(os.path.join(self.path, srpm)) and not \ os.path.exists(os.path.join(self.path, rpm)): - raise FedpkgError("Need to build srpm and rpm first") + raise FedpkgError('Need to build srpm and rpm first') cmd = ['rpmlint', os.path.join(self.path, srpm), os.path.join(self.path, rpm)] try: @@ -253,11 +250,9 @@ class PackageModule: try: subprocess.check_call(command, cwd=outdir) except subprocess.CalledProcessError, e: - print "Could not download %s: %s" % (url, e) - return 1 + raise FedpkgError('Could not download %s: %s' % (url, e)) if not _verify_file(outfile, csum, self.lookasidehash): - print "%s failed checksum" % file - return 1 + raise FedpkgError('%s failed checksum' % file) return def srpm(self, hashtype='sha256'): @@ -279,6 +274,5 @@ class PackageModule: try: subprocess.check_call(cmd) except subprocess.CalledProcessError, e: - print "Could not build %s: %s" % (self.module, e) - return 1 + raise FedpkgError('Could not build %s: %s' % (self.module, e)) return \ No newline at end of file -- cgit From 8ff2ad2b965e0ab027f4bb397f8b03ccc3f2c7a0 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:40:14 -0800 Subject: Fix the error messages to match the functions --- src/fedpkg/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 05d83ff..296e343 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -151,7 +151,7 @@ class PackageModule: output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() except subprocess.CalledProcessError, e: - raise FedpkgError('Could not get version-release of %s: %s' % (self.module, e)) + raise FedpkgError('Could not get version of %s: %s' % (self.module, e)) return output[0] def getrel(self): @@ -165,7 +165,7 @@ class PackageModule: output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate() except subprocess.CalledProcessError, e: - raise FedpkgError('Could not get version-release of %s: %s' % (self.module, e)) + raise FedpkgError('Could not get release of %s: %s' % (self.module, e)) return output[0] def gimmespec(self): -- cgit From 58385c0d32910558c18502c585eb30fab6380e97 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:49:36 -0800 Subject: Wrap fedpkg calls in try statements to catch errors nicely --- src/fedpkg.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index e80b982..a447c1a 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -57,16 +57,20 @@ def export(args): print('Not implimented yet, got %s' % args) def gimmespec(args): - mymodule = fedpkg.PackageModule(args.path) - print(mymodule.spec) + try: + mymodule = fedpkg.PackageModule(args.path) + print(mymodule.spec) + except fedpkg.FedpkgError, e: + print('Could not get spec file: %s' % e) + return 1 def install(args): # not implimented print('Not implimented yet, got %s' % args) def lint(args): - mymodule = fedpkg.PackageModule(args.path) try: + mymodule = fedpkg.PackageModule(args.path) print(mymodule.lint()) except fedpkg.FedpkgError, e: print('Could not run rpmlint: %s' % e) @@ -101,16 +105,24 @@ def scratchbuild(args): print('Not implimented yet, got %s' % args) def sources(args): - mymodule = fedpkg.PackageModule(args.path) - mymodule.sources(args.outdir) + try: + mymodule = fedpkg.PackageModule(args.path) + mymodule.sources(args.outdir) + except fedpkg.FedpkgError, e: + print('Could not download sources: %s' % e) + return 1 def srpm(args): - mymodule = fedpkg.PackageModule(args.path) - mymodule.sources(args.path) - if args.md5: - mymodule.srpm('md5') - else: - mymodule.srpm() + try: + mymodule = fedpkg.PackageModule(args.path) + mymodule.sources(args.path) + if args.md5: + mymodule.srpm('md5') + else: + mymodule.srpm() + except fedpkg.FedpkgError, e: + print('Could not make an srpm: %s' % e) + return 1 def tagrequest(args): # not implimented -- cgit From 76a0095c50ac1ef170d33b744123c4aa123d7ae8 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 14:51:46 -0800 Subject: Whitespace clean up --- src/fedpkg/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 296e343..f3a3a40 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -96,6 +96,7 @@ def clone_with_dirs(module, user): class PackageModule: def _findbranch(self): """Find the branch we're on""" + if not os.path.exists(os.path.join(self.path, 'branch')): return 'devel' branch = open(os.path.join(self.path, 'branch'), 'r').read().strip() @@ -261,7 +262,7 @@ class PackageModule: Requires sources already downloaded. """ - + cmd = ['rpmbuild'] cmd.extend(self.rpmdefines) # This may need to get updated if we ever change our checksum default -- cgit From a52412f0975f2c2b192e8a56f9654e45f4087116 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 15:08:38 -0800 Subject: Make localarch a module attribute --- src/fedpkg/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index f3a3a40..4f5e559 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -102,6 +102,12 @@ class PackageModule: branch = open(os.path.join(self.path, 'branch'), 'r').read().strip() return branch + def _getlocalarch(self): + """Get the local arch as defined by rpm""" + + return subprocess.Popen(['rpm', '--eval', '%{_arch}'], + stdout=subprocess.PIPE).communicate()[0] + def __init__(self, path=os.curdir): # Initiate a PackageModule object in a given path # Set some global variables used throughout @@ -140,6 +146,7 @@ class PackageModule: '--define', '%s 1' % self.distvar] self.ver = self.getver() self.rel = self.getrel() + self.localarch = self._getlocalarch() def getver(self): """Return the version-release of a package module.""" @@ -186,7 +193,7 @@ class PackageModule: # Make sure we have a srpm to run on srpm = "%s-%s-%s.src.rpm" % (self.module, self.ver, self.rel) rpm = "%s-%s-%s.%s.rpm" % (self.module, self.ver, self.rel, - os.uname()[4]) + self.localarch) if not os.path.exists(os.path.join(self.path, srpm)) and not \ os.path.exists(os.path.join(self.path, rpm)): raise FedpkgError('Need to build srpm and rpm first') -- cgit From 5c0a9bef181a3380921b90aafaf985ab0fe07ba1 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 15:59:41 -0800 Subject: Add a local target to build locally This still has some errors, and I think I need to run the rpm stuff in a real shell, which would require re-working the rpm defines and editing the other functions that use them. What fun! --- src/fedpkg/__init__.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 4f5e559..696338f 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -204,7 +204,48 @@ class PackageModule: except subprocess.CalledProcessError, e: raise FedpkgError(e) return output[0] - + + def local(self, arch=None, hashtype='sha256'): + """rpmbuild locally for given arch. + + Takes arch to build for, and hashtype to build with. + + Writes output to a log file and returns output from build. + + """ + + # Get the sources + self.sources() + # Determine arch to build for + if not arch: + arch = self.localarch + # build up the rpm command + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + # This may need to get updated if we ever change our checksum default + if not hashtype == 'sha256': + cmd.extend(['--define', + '_source_filedigest_algorithm %s' % hashtype, + '--define', + '_binary_filedigest_algorithm %s' % hashtype]) + cmd.extend(['--target', arch, '-ba', + os.path.join(self.path, self.spec)]) + try: + proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, + stdout=subprocess.PIPE) + output = proc.communicate() + except OSError, e: + raise FedpkgError(e) + outfile = open(os.path.join(self.path, '.build-%s-%s.log' % (self.ver, + self.rel)), 'w') + outfile.writelines(output[0]) + outfile.close() + # See if we had a good return or not, raise accordingly + if proc.returncode: + raise FedpkgError('%s returned %s: %s' % + (subprocess.list2cmdline(cmd), + proc.returncode, output[0])) + return output[0] def new_sources(self, files): """Replace source file(s) in the lookaside cache""" -- cgit From 7a6a512468692a717f537de087009ba846b295c3 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 15:59:52 -0800 Subject: Wire up local target --- src/fedpkg.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index a447c1a..c393e9c 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -77,8 +77,15 @@ def lint(args): return 1 def local(args): - # not implimented - print('Not implimented yet, got %s' % args) + arch = None + if args.arch: + arch = args.arch + try: + mymodule = fedpkg.PackageModule(args.path) + print(mymodule.local(arch=arch)) + except fedpkg.FedpkgError, e: + print('Could not build locally: %s' % e) + return 1 def mockbuild(args): # not implimented @@ -239,6 +246,7 @@ if __name__ == '__main__': # Build locally parser_local = subparsers.add_parser('local', help = 'Local test rpmbuild binary') + parser_local.add_argument('--arch', help = 'Build for arch') parser_local.set_defaults(command = local) # Build in mock -- cgit From 8b511db3667b9a50601984af4b0a5668f99f9a9f Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 16:57:04 -0800 Subject: Make all rpm calls use shell=True I think this is necessary or some spec junk won't work out right. --- src/fedpkg/__init__.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/fedpkg/__init__.py b/src/fedpkg/__init__.py index 696338f..b4940e9 100644 --- a/src/fedpkg/__init__.py +++ b/src/fedpkg/__init__.py @@ -105,8 +105,8 @@ class PackageModule: def _getlocalarch(self): """Get the local arch as defined by rpm""" - return subprocess.Popen(['rpm', '--eval', '%{_arch}'], - stdout=subprocess.PIPE).communicate()[0] + return subprocess.Popen(['rpm --eval %{_arch}'], shell=True, + stdout=subprocess.PIPE).communicate()[0].strip('\n') def __init__(self, path=os.curdir): # Initiate a PackageModule object in a given path @@ -136,14 +136,14 @@ class PackageModule: self.distval = '13' # this is hardset for now, which is bad self.distvar = 'fedora' self.dist = '.fc%s' % self.distval - self.rpmdefines = ['--define', '_sourcedir %s' % path, - '--define', '_specdir %s' % path, - '--define', '_builddir %s' % path, - '--define', '_srcrpmdir %s' % path, - '--define', '_rpmdir %s' % path, - '--define', 'dist %s' % self.dist, - '--define', '%s %s' % (self.distvar, self.distval), - '--define', '%s 1' % self.distvar] + self.rpmdefines = ["--define '_sourcedir %s'" % path, + "--define '_specdir %s'" % path, + "--define '_builddir %s'" % path, + "--define '_srcrpmdir %s'" % path, + "--define '_rpmdir %s'" % path, + "--define 'dist %s'" % self.dist, + "--define '%s %s'" % (self.distvar, self.distval), + "--define '%s 1'" % self.distvar] self.ver = self.getver() self.rel = self.getrel() self.localarch = self._getlocalarch() @@ -156,7 +156,7 @@ class PackageModule: cmd.extend(['-q', '--qf', '%{VERSION}', '--specfile', os.path.join(self.path, self.spec)]) try: - output = subprocess.Popen(cmd, + output = subprocess.Popen(' '.join(cmd), shell=True, stdout=subprocess.PIPE).communicate() except subprocess.CalledProcessError, e: raise FedpkgError('Could not get version of %s: %s' % (self.module, e)) @@ -170,7 +170,7 @@ class PackageModule: cmd.extend(['-q', '--qf', '%{RELEASE}', '--specfile', os.path.join(self.path, self.spec)]) try: - output = subprocess.Popen(cmd, + output = subprocess.Popen(' '.join(cmd), shell=True, stdout=subprocess.PIPE).communicate() except subprocess.CalledProcessError, e: raise FedpkgError('Could not get release of %s: %s' % (self.module, e)) @@ -224,15 +224,13 @@ class PackageModule: cmd.extend(self.rpmdefines) # This may need to get updated if we ever change our checksum default if not hashtype == 'sha256': - cmd.extend(['--define', - '_source_filedigest_algorithm %s' % hashtype, - '--define', - '_binary_filedigest_algorithm %s' % hashtype]) + cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, + "--define '_binary_filedigest_algorithm %s'" % hashtype]) cmd.extend(['--target', arch, '-ba', os.path.join(self.path, self.spec)]) try: - proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, - stdout=subprocess.PIPE) + proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, shell=True) output = proc.communicate() except OSError, e: raise FedpkgError(e) @@ -315,13 +313,11 @@ class PackageModule: cmd.extend(self.rpmdefines) # This may need to get updated if we ever change our checksum default if not hashtype == 'sha256': - cmd.extend(['--define', - '_source_filedigest_algorithm %s' % hashtype, - '--define', - '_binary_filedigest_algorithm %s' % hashtype]) + cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, + "--define '_binary_filedigest_algorithm %s'" % hashtype]) cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)]) try: - subprocess.check_call(cmd) + subprocess.check_call(' '.join(cmd), shell=True) except subprocess.CalledProcessError, e: raise FedpkgError('Could not build %s: %s' % (self.module, e)) return \ No newline at end of file -- cgit From 4414682024a26492a7ab85b5db346a290140c7b6 Mon Sep 17 00:00:00 2001 From: Jesse Keating Date: Mon, 4 Jan 2010 17:02:19 -0800 Subject: Add md5 option to local build --- src/fedpkg.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/fedpkg.py b/src/fedpkg.py index c393e9c..4c385f0 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -82,7 +82,10 @@ def local(args): arch = args.arch try: mymodule = fedpkg.PackageModule(args.path) - print(mymodule.local(arch=arch)) + if args.md5: + print(mymodule.local(arch=arch, hashtype='md5')) + else: + print(mymodule.local(arch=arch)) except fedpkg.FedpkgError, e: print('Could not build locally: %s' % e) return 1 @@ -247,6 +250,9 @@ if __name__ == '__main__': parser_local = subparsers.add_parser('local', help = 'Local test rpmbuild binary') parser_local.add_argument('--arch', help = 'Build for arch') + # optionally define old style hashsums + parser_local.add_argument('--md5', action = 'store_true', + help = 'Use md5 checksums (for older rpm hosts)') parser_local.set_defaults(command = local) # Build in mock -- cgit