diff options
| -rw-r--r-- | Makefile.am | 18 | ||||
| -rw-r--r-- | src/fedpkg.bash | 7 | ||||
| -rwxr-xr-x | src/fedpkg.py | 66 | ||||
| -rw-r--r-- | src/pyfedpkg/__init__.py | 30 | ||||
| -rwxr-xr-x | src/pyfedpkg/initial_merge.py | 241 | ||||
| -rw-r--r-- | src/pyfedpkg/man_page.py | 162 |
6 files changed, 496 insertions, 28 deletions
diff --git a/Makefile.am b/Makefile.am index 4df6dbb..506ad96 100644 --- a/Makefile.am +++ b/Makefile.am @@ -79,7 +79,23 @@ ppc-koji: $(srcdir)/src/secondary-koji rm -f ppc-koji install -p -m 755 -T $(srcdir)/src/secondary-koji ppc-koji -pyfedpkg_PYTHON = $(wildcard $(srcdir)/src/pyfedpkg/*.py) +if HAVE_PYTHON +CLEANFILES += fedpkg.1 +man1_MANS = fedpkg.1 +fedpkg.1: fedpkg + tmpfile="fedpkg.1.$$$$"; \ + if env PYTHONPATH=$(srcdir)/src python -c 'import fedpkg; fedpkg.parse_cmdline(True)' > "$$tmpfile"; then \ + mv -f "$$tmpfile" "$@"; \ + else \ + rm -f "$$tmpfile"; \ + s="$$?"; \ + echo "Error generating man page: $$s"; \ + exit "$$s"; \ + fi +endif + +noinst_PYTHON = $(srcdir)/src/pyfedpkg/man_page.py +pyfedpkg_PYTHON = $(filter-out $(noinst_PYTHON),$(wildcard $(srcdir)/src/pyfedpkg/*.py)) fedora_cert_PYTHON = $(wildcard $(srcdir)/src/fedora_cert/*.py) fedora_certdir = $(pythondir)/fedora_cert diff --git a/src/fedpkg.bash b/src/fedpkg.bash index e890345..134785c 100644 --- a/src/fedpkg.bash +++ b/src/fedpkg.bash @@ -36,7 +36,7 @@ _fedpkg() local options="--help -v -q" local options_value="--dist --user --path" local commands="build chain-build ci clean clog clone co commit compile diff gimmespec giturl help \ - import install lint local mockbuild new new-sources patch prep pull push retire scratch-build sources \ + import initial-merge install lint local mockbuild new new-sources patch prep pull push retire scratch-build sources \ srpm switch-branch tag tag-request unused-patches update upload verify-files verrel" # parse main options and get command @@ -110,10 +110,13 @@ _fedpkg() options="--dry-run -x" ;; clone|co) - options="--branches --anonymous" + options="--branches --anonymous --initial-merge" options_branch="-b" after="package" ;; + initial-merge) + options="--dry-run" + ;; commit|ci) options="--push --clog --tag" options_string="--message" diff --git a/src/fedpkg.py b/src/fedpkg.py index e12124f..4086562 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -446,6 +446,8 @@ def clone(args): pyfedpkg.clone_with_dirs(args.module[0], user) else: pyfedpkg.clone(args.module[0], user, args.path, args.branch) + if args.initial_merge: + pyfedpkg.initial_merge.handle_repo(args.module[0]) except pyfedpkg.FedpkgError, e: log.error('Could not clone: %s' % e) sys.exit(1) @@ -575,10 +577,7 @@ def local(args): arch = args.arch try: mymodule = pyfedpkg.PackageModule(args.path, args.dist) - if args.md5: - return mymodule.local(arch=arch, hashtype='md5') - else: - return mymodule.local(arch=arch) + return mymodule.local(arch=arch, hashtype=args.hashtype) except pyfedpkg.FedpkgError, e: log.error('Could not build locally: %s' % e) sys.exit(1) @@ -680,10 +679,8 @@ def srpm(args): try: mymodule = pyfedpkg.PackageModule(args.path, args.dist) pyfedpkg.sources(args.path) - if args.md5: - mymodule.srpm('md5') - else: - mymodule.srpm() + mymodule.srpm(hashtype=args.hashtype, + fix_permissions=args.fix_permissions) except pyfedpkg.FedpkgError, e: log.error('Could not make an srpm: %s' % e) sys.exit(1) @@ -854,8 +851,10 @@ def verrel(args): sys.exit(1) print('%s-%s-%s' % (mymodule.module, mymodule.ver, mymodule.rel)) -# THe main code goes here -if __name__ == '__main__': + +def parse_cmdline(generate_manpage = False): + """Parse the command line""" + # Create the parser object parser = argparse.ArgumentParser(description = 'Fedora Packaging utility', prog = 'fedpkg', @@ -867,7 +866,8 @@ if __name__ == '__main__': parser.add_argument('--dist', default=None, help='Override the distribution, eg f15 or el6') # Let somebody override the username found in fedora cert - parser.add_argument('-u', '--user') + parser.add_argument('-u', '--user', + help = "Override the username found in the fedora cert") # Let the user define which path to look at instead of pwd parser.add_argument('--path', default = os.getcwd(), help='Directory to interact with instead of current dir') @@ -955,9 +955,15 @@ packages will be built sequentially. parser_clone = subparsers.add_parser('clone', help = 'Clone and checkout a module') # Allow an old style clone with subdirs for branches - parser_clone.add_argument('--branches', '-B', - action = 'store_true', - help = 'Do an old style checkout with subdirs for branches') + parser_clone_branches_group = parser_clone.add_mutually_exclusive_group() + parser_clone_branches_group.add_argument( + '--branches', '-B', + action = 'store_true', + help = 'Do an old style checkout with subdirs for branches') + parser_clone_branches_group.add_argument( + '--initial-merge', '-i', + action = 'store_true', + help = 'Run initial-merge on the cloned repo immediately') # provide a convenient way to get to a specific branch parser_clone.add_argument('--branch', '-b', help = 'Check out a specific branch') @@ -1060,6 +1066,9 @@ packages will be built sequentially. help = 'Source rpm to import') parser_import_srpm.set_defaults(command = import_srpm) + # Initial branch merges + pyfedpkg.initial_merge.add_parser_to(subparsers) + # install locally parser_install = subparsers.add_parser('install', help = 'Local test rpmbuild install') @@ -1082,7 +1091,8 @@ packages will be built sequentially. help = 'Local test rpmbuild binary') parser_local.add_argument('--arch', help = 'Build for arch') # optionally define old style hashsums - parser_local.add_argument('--md5', action = 'store_true', + parser_local.add_argument('--md5', action = 'store_const', + dest='hashtype', const='md5', default=None, help = 'Use md5 checksums (for older rpm hosts)') parser_local.set_defaults(command = local) @@ -1160,8 +1170,12 @@ packages will be built sequentially. parser_srpm = subparsers.add_parser('srpm', help = 'Create a source rpm') # optionally define old style hashsums - parser_srpm.add_argument('--md5', action = 'store_true', + parser_srpm.add_argument('--md5', action = 'store_const', + dest='hashtype', const='md5', default=None, help = 'Use md5 checksums (for older rpm hosts)') + parser_srpm.add_argument('--fix-permissions', action='store_true', + default=False, + help = 'Fix permissions of files to be put into .src.rpm file') parser_srpm.set_defaults(command = srpm) # switch branches @@ -1241,8 +1255,24 @@ packages will be built sequentially. ' name-version-release') parser_verrel.set_defaults(command = verrel) - # Parse the args - args = parser.parse_args() + if not generate_manpage: + # Parse the args + return parser.parse_args() + else: + # Generate the man page + + # Use the "as man_page" part to avoid overwriting the pyfedpkg + # namespace, which would break all usage of pyfedpkg.* outside + # of this else branch. + import pyfedpkg.man_page as man_page + man_page.generate(parser, subparsers) + sys.exit(0) + # no return possible + + +# The main code goes here +if __name__ == '__main__': + args = parse_cmdline() # setup the logger -- This logger will take things of INFO or DEBUG and # log it to stdout. Anything above that (WARN, ERROR, CRITICAL) will go diff --git a/src/pyfedpkg/__init__.py b/src/pyfedpkg/__init__.py index 0a1e897..b32542a 100644 --- a/src/pyfedpkg/__init__.py +++ b/src/pyfedpkg/__init__.py @@ -48,6 +48,11 @@ BRANCHFILTER = 'f\d\d\/master|master|el\d\/master|olpc\d\/master' class FedpkgError(Exception): pass + +# This module needs FedpkgError to be defined +from . import initial_merge + + # Setup our logger # Null logger to avoid spurrious messages, add a handler in app code class NullHandler(logging.Handler): @@ -131,7 +136,7 @@ def _run_command(cmd, shell=False, env=None, pipe=[], cwd=None): """ - # Process any environment vairables. + # Process any environment variables. environ = os.environ if env: for item in env.keys(): @@ -1070,12 +1075,12 @@ class PackageModule: return subprocess.Popen(['rpm --eval %{_arch}'], shell=True, stdout=subprocess.PIPE).communicate()[0].strip('\n') - def __init__(self, path=None, dist=None): + def __init__(self, path, dist=None): # Initiate a PackageModule object in a given path # Set some global variables used throughout - if not path: - path = os.getcwd() log.debug('Creating module object from %s' % path) + if not os.path.isdir(path): + raise FedpkgError('Module directory not found: %s' % path) self.path = path self.lookaside = LOOKASIDE self.lookasidehash = LOOKASIDEHASH @@ -1536,7 +1541,7 @@ class PackageModule: _run_command(cmd, shell=True) return - def local(self, arch=None, hashtype='sha256'): + def local(self, arch=None, hashtype=None): """rpmbuild locally for given arch. Takes arch to build for, and hashtype to build with. @@ -1547,6 +1552,10 @@ class PackageModule: """ + # Figure out which hashtype to use, if not provided one + if not hashtype: + hashtype = self.hashtype + # This could really use a list of arches to build for and loop over # Get the sources sources(self.path) @@ -1733,7 +1742,7 @@ class PackageModule: _run_command(cmd, shell=True) return - def srpm(self, hashtype=None): + def srpm(self, hashtype=None, fix_permissions=False): """Create an srpm using hashtype from content in the module Requires sources already downloaded. @@ -1751,11 +1760,18 @@ class PackageModule: # srpm is newer, don't redo it return - cmd = ['rpmbuild'] + if fix_permissions: + _run_command(cmd=['git', 'ls-files', '-z'], + pipe= ['xargs', '-0', 'chmod', 'a+r'], + shell=False) + + cmd = ['fakeroot', 'rpmbuild'] cmd.extend(self.rpmdefines) + # Figure out which hashtype to use, if not provided one if not hashtype: hashtype = self.hashtype + # This may need to get updated if we ever change our checksum default if not hashtype == 'sha256': cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype, diff --git a/src/pyfedpkg/initial_merge.py b/src/pyfedpkg/initial_merge.py new file mode 100755 index 0000000..f22453f --- /dev/null +++ b/src/pyfedpkg/initial_merge.py @@ -0,0 +1,241 @@ +# initial_merge.py - perform initial merge after dist-git migration +# +# Copyright (C) 2010 Hans Ulrich Niedermann <hun@n-dimensional.de> +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +# Note that this docstring is used as description for the command. +"""\ +Performs a 'git merge' of all git branches with the same content +(i.e. with the same package spec files, patch files, etc.), regardless +of their history. + +This is useful after Fedora's dist-cvs to dist-git migration, as often +different branches have different histories but the same content on the +filesystem. + +After these initial merges of identical trees, future merges between +the branches will be a lot easier: Easier to follow in the dependency +graph, and easier to perform without conflicts. +""" + +import argparse +import sys +import os +import subprocess +import logging + +import git +import pyfedpkg + + +__all__ = [ + 'add_parser_to', + 'handle_repo', + ] + + +merge_commit_msg = """\ +Initial pseudo merge for dist-git setup + +This git merge exists to make future git merges much easier. +""" + + +log = logging.getLogger("fedpkg") + + +def str_numsplit(s): + """Helper function for properly ordering branch names. + + Converts branch names like 'fc6', 'f8', 'f10', 'master' into + ('fc',6), ('f',8), ('f',10), 'master', respectively. + """ + assert(not s[0].isdigit()) + if not s[-1].isdigit(): + return s + i = len(s) + while i>0: + i = i - 1 + if not s[i].isdigit(): + break + prefix, numstr = s[0:i+1], s[i+1:] + return (prefix, int(numstr)) + + +def cmp_relbranch(a, b): + """Comparison function release branch names (for sort() calls) + + Sorts branch names like 'el4', 'fc6', 'f8', 'f10', 'master' in + that order. + """ + asplit = str_numsplit(a) + bsplit = str_numsplit(b) + if type(asplit) == str and type(bsplit) == str: + return cmp(asplit,bsplit) + elif type(asplit) == str and type(bsplit) == tuple: + return 1 + elif type(asplit) == tuple and type(bsplit) == str: + return -1 + elif type(asplit) == tuple and type(bsplit) == tuple: + if asplit[0] in ('f','fc') and bsplit[0] in ('f','fc'): + return cmp(asplit[1], bsplit[1]) + else: + return cmp(asplit,bsplit) + else: + return cmp(asplit,bsplit) + + +class Branch(object): + + """Convenience class for handling branches to initial-merge""" + + def __init__(self, sha, origbranch): + self.sha, self.origbranch = sha, origbranch + a = self.origbranch.split('/') + assert(a[0] == 'origin') + assert(a[-1] == 'master') + self.localbranch = a[1] + + def __repr__(self): + return "%(sha)s %(origbranch)s" % self.__dict__ + + def __cmp__(self, other): + if self.sha == other.sha: + return cmp_relbranch(self.localbranch, other.localbranch) + else: + return cmp(self.sha, other.sha) + + +class Filter(object): + + """Branch filter + + Feed branches to this filter via eat(), and flush() it after you + are done. The filter will detect branches with the same tree sha, + and call do_initial_merge() for them. + """ + + def __init__(self, repo, dry_run=False): + self.repo = repo + self.dry_run = dry_run + self.__reset() + + def __reset(self): + self.branch_list = [] + + def do_initial_merge(self, into, to_merge): + log.info("#### Merging %s into %s ####", + [ x.localbranch for x in to_merge ], into.localbranch) + pyfedpkg.switch_branch(into.localbranch, self.repo.working_tree_dir) + log.info("Merging %s into %s", + repr([ x.origbranch for x in to_merge]), repr(into.localbranch)) + if not self.dry_run: + self.repo.git.merge('-m', merge_commit_msg, + '-s', 'ours', + *[x.origbranch for x in to_merge]) + for t in to_merge: + pyfedpkg.switch_branch(t.localbranch, self.repo.working_tree_dir) + log.info("Merging %s into %s", repr(into.localbranch), repr(t.localbranch)) + if not self.dry_run: + self.repo.git.merge(into.localbranch) + pyfedpkg.switch_branch(into.localbranch, self.repo.working_tree_dir) + + def flush(self): + if len(self.branch_list) < 2: + return + + head = self.branch_list[-1] + others = self.branch_list[:-1] + self.do_initial_merge(head, others) + + self.__reset() + + def eat(self, item): + """Feed item to the filter. The filter will decide what to do with it.""" + if self.branch_list: + last = self.branch_list[-1] + if last.sha == item.sha: + self.branch_list.extend([item]) + else: + self.flush() + self.branch_list = [item] + else: + self.branch_list = [item] + + +class UnknownRepoTypeError(pyfedpkg.FedpkgError): + pass + + +def handle_repo(repo, dry_run=False): + if type(repo) == str: + repo = git.Repo(repo) + elif isinstance(repo, git.Repo): + pass + else: + raise UnknownRepoTypeError("%s" % repo) + log.info("######## initial-merge %s ########" % repo.working_tree_dir) + _locals, remotes = pyfedpkg._list_branches(repo=repo) + aa = [ Branch(repo.git.rev_parse('%s^{tree}' % b), b) for b in remotes ] + aa.sort() + log.info("Branches sorted by tree sha:") + for x in aa: + log.info(" %s" % x) + + n = 0 + f = Filter(repo, dry_run=dry_run) + while n < len(aa): + f.eat(aa[n]) + n = n + 1 + f.flush() + + log.info("######## /initial-merge %s ########" % repo.working_tree_dir) + + +class DirListAction(argparse.Action): + + def __init__(self, *args, **kwargs): + super(DirListAction, self).__init__(*args, **kwargs) + self.default_value = ['.'] + + def __call__(self, parser, namespace, values, option_string=None): + if values: + setattr(namespace, self.dest, values) + else: + setattr(namespace, self.dest, self.default) + + +def fedpkg_command(args): + is_first = True + for repo_path in args.repo_path: + if is_first: + is_first = False + else: + log.info("") + repo = git.Repo(repo_path) + handle_repo(repo, dry_run=args.dry_run) + + +_module_doc = __doc__ + + +def add_parser_to(subparsers): + sp = subparsers.add_parser('initial-merge', + help = 'git merge to join branches with identical trees', + formatter_class=argparse.RawDescriptionHelpFormatter, + description = _module_doc) + sp.add_argument('repo_path', metavar='repo-path', + nargs='*', default=['.'], + action=DirListAction, + help = 'Path to a repo to initial-merge') + sp.add_argument('-n', '--dry-run', + default=False, const=True, action='store_const', + help="Whether to run without actually merging") + sp.set_defaults(command = fedpkg_command) + return sp diff --git a/src/pyfedpkg/man_page.py b/src/pyfedpkg/man_page.py new file mode 100644 index 0000000..47285f2 --- /dev/null +++ b/src/pyfedpkg/man_page.py @@ -0,0 +1,162 @@ +# Print a man page from the help texts. + + +import argparse +import sys +import datetime + + +# We could substitute the "" in .TH with the fedpkg version if we knew it +man_header = """\ +.\" man page for fedpkg +.TH fedpkg 1 "%(today)s" "" "fedora\-packager" +.SH "NAME" +fedpkg \- Fedora Packaging utility +.SH "SYNOPSIS" +.B "fedpkg" +[ +.I global_options +] +.I "command" +[ +.I command_options +] +[ +.I command_arguments +] +.br +.B "fedpkg" +.B "help" +.br +.B "fedpkg" +.I "command" +.B "\-\-help" +.SH "DESCRIPTION" +.B "fedpkg" +is a script to interact with the Fedora Packaging system. +""" + +man_footer = """\ +.SH "SEE ALSO" +.UR "https://fedorahosted.org/fedora\-packager/" +.BR "https://fedorahosted.org/fedora\-packager/" +""" + +class ManFormatter(object): + + def __init__(self, man): + self.man = man + + def write(self, data): + #print "MF:", repr(data) + for line in data.split('\n'): + #print 'MFL:', line + self.man.write(' %s\n' % line) + + +def strip_usage(s): + """Strip "usage: " string from beginning of string if present""" + if s.startswith('usage: '): + return s.replace('usage: ', '', 1) + else: + return s + + +def man_constants(): + """Global constants for man file templates""" + today = datetime.date.today() + today_manstr = today.strftime('%Y\-%m\-%d') + return {'today': today_manstr} + + +def generate(parser, subparsers): + """\ + Generate the man page on stdout + + Given the argparse based parser and subparsers arguments, generate + the corresponding man page and write it to stdout. + """ + + # Not nice, but works: Redirect any print statement output to + # stderr to avoid clobbering the man page output on stdout. + man_file = sys.stdout + sys.stdout = sys.stderr + + mf = ManFormatter(man_file) + + choices = subparsers.choices + k = choices.keys() + k.sort() + + man_file.write(man_header % man_constants()) + + helptext = parser.format_help() + helptext = strip_usage(helptext) + helptextsplit = helptext.split('\n') + helptextsplit = [ line for line in helptextsplit + if not line.startswith(' -h, --help') ] + + man_file.write('.SS "%s"\n' % ("Global Options",)) + + outflag = False + for line in helptextsplit: + if line == "optional arguments:": + outflag = True + elif line == "": + outflag = False + elif outflag: + man_file.write("%s\n" % line) + + help_texts = {} + for pa in subparsers._choices_actions: + help_texts[pa.dest] = getattr(pa, 'help', None) + + if True: # Either kill THIS + # determine length of longest command and generate format string + commands = help_texts.keys() + commands.sort(lambda a,b: cmp(len(b), len(a))) + max_cmdlen = len(commands[0]) + fmtstring = ' %%-%ds %%s\n' % (max_cmdlen,) + + man_file.write('.SS "Commands"\n') + + for command in k: + cmdparser = choices[command] + if not cmdparser.add_help: + continue + man_file.write(fmtstring % (command, help_texts[command])) + + if True: # Or kill THIS + man_file.write('.SH "COMMAND OVERVIEW"\n') + + for command in k: + cmdparser = choices[command] + if not cmdparser.add_help: + continue + usage = cmdparser.format_usage() + usage = strip_usage(usage) + usage = ''.join(usage.split('\n')) + usage = ' '.join(usage.split()) + if help_texts[command]: + man_file.write('.TP\n.B "%s"\n%s\n' % (usage, help_texts[command])) + else: + man_file.write('.TP\n.B "%s"\n' % (usage)) + + man_file.write('.SH "COMMAND REFERENCE"\n') + for command in k: + cmdparser = choices[command] + if not cmdparser.add_help: + continue + + man_file.write('.SS "%s"\n' % cmdparser.prog) + + help = help_texts[command] + if help and not cmdparser.description: + if not help.endswith('.'): help = "%s." % help + cmdparser.description = help + + formatter = cmdparser.formatter_class(cmdparser.prog) + h = cmdparser.format_help() + mf.write(h) + + man_file.write(man_footer) |