From c139cc05d3101187d7292e220b1734bb03be99da Mon Sep 17 00:00:00 2001 From: Hans Ulrich Niedermann Date: Mon, 9 Aug 2010 15:14:08 +0200 Subject: Add "fedpkg initial-merge" command usage: fedpkg.py initial-merge [-h] [-n] [repo-path [repo-path ...]] Performs a 'git merge' of all git branches with the same content (i.e. with the same package spec files, patch files, etc.), regardless of their history. This is useful after Fedora's dist-cvs to dist-git migration, as often different branches have different histories but the same content on the filesystem. After these initial merges of identical trees, future merges between the branches will be a lot easier: Easier to follow in the dependency graph, and easier to perform without conflicts. positional arguments: repo-path Path to a repo to initial-merge optional arguments: -h, --help show this help message and exit -n, --dry-run Whether to run without actually merging --- src/fedpkg.bash | 5 +- src/fedpkg.py | 3 + src/pyfedpkg/__init__.py | 2 + src/pyfedpkg/initial_merge.py | 231 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 1 deletion(-) create mode 100755 src/pyfedpkg/initial_merge.py diff --git a/src/fedpkg.bash b/src/fedpkg.bash index 90cb2ac..5a20446 100644 --- a/src/fedpkg.bash +++ b/src/fedpkg.bash @@ -36,7 +36,7 @@ _fedpkg() local options="--help -v -q" local options_value="--dist --user --path" local commands="build chain-build ci clean clog clone co commit compile diff gimmespec giturl help \ - import install lint local mockbuild new new-sources patch prep pull push retire scratch-build sources \ + import initial-merge install lint local mockbuild new new-sources patch prep pull push retire scratch-build sources \ srpm switch-branch tag tag-request unused-patches update upload verrel" # parse main options and get command @@ -114,6 +114,9 @@ _fedpkg() options_branch="-b" after="package" ;; + initial-merge) + options="--dry-run" + ;; commit|ci) options="--push --clog --tag" options_string="--message" diff --git a/src/fedpkg.py b/src/fedpkg.py index b9d46c4..90c3be7 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -1046,6 +1046,9 @@ packages will be built sequentially. help = 'Source rpm to import') parser_import_srpm.set_defaults(command = import_srpm) + # Initial branch merges + pyfedpkg.initial_merge.add_parser_to(subparsers) + # install locally parser_install = subparsers.add_parser('install', help = 'Local test rpmbuild install') diff --git a/src/pyfedpkg/__init__.py b/src/pyfedpkg/__init__.py index 76b4e5d..83b54f7 100644 --- a/src/pyfedpkg/__init__.py +++ b/src/pyfedpkg/__init__.py @@ -31,6 +31,8 @@ import OpenSSL import fnmatch import offtrac +from . import initial_merge + # Define some global variables, put them here to make it easy to change LOOKASIDE = 'http://pkgs.fedoraproject.org/repo/pkgs' diff --git a/src/pyfedpkg/initial_merge.py b/src/pyfedpkg/initial_merge.py new file mode 100755 index 0000000..4964745 --- /dev/null +++ b/src/pyfedpkg/initial_merge.py @@ -0,0 +1,231 @@ +# initial_merge.py - perform initial merge after dist-git migration +# +# Copyright (C) 2010 Hans Ulrich Niedermann +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +# Note that this docstring is used as description for the command. +"""\ +Performs a 'git merge' of all git branches with the same content +(i.e. with the same package spec files, patch files, etc.), regardless +of their history. + +This is useful after Fedora's dist-cvs to dist-git migration, as often +different branches have different histories but the same content on the +filesystem. + +After these initial merges of identical trees, future merges between +the branches will be a lot easier: Easier to follow in the dependency +graph, and easier to perform without conflicts. +""" + +import argparse +import sys +import os +import subprocess +import logging + +import git +import pyfedpkg + + +__all__ = [ + 'add_parser_to', + 'handle_repo', + ] + + +merge_commit_msg = """\ +Initial pseudo merge for dist-git setup + +This git merge exists to make future git merges much easier. +""" + + +log = logging.getLogger("fedpkg") + + +def str_numsplit(s): + """Helper function for properly ordering branch names. + + Converts branch names like 'fc6', 'f8', 'f10', 'master' into + ('fc',6), ('f',8), ('f',10), 'master', respectively. + """ + assert(not s[0].isdigit()) + if not s[-1].isdigit(): + return s + i = len(s) + while i>0: + i = i - 1 + if not s[i].isdigit(): + break + prefix, numstr = s[0:i+1], s[i+1:] + return (prefix, int(numstr)) + + +def cmp_relbranch(a, b): + """Comparison function release branch names (for sort() calls) + + Sorts branch names like 'el4', 'fc6', 'f8', 'f10', 'master' in + that order. + """ + asplit = str_numsplit(a) + bsplit = str_numsplit(b) + if type(asplit) == str and type(bsplit) == str: + return cmp(asplit,bsplit) + elif type(asplit) == str and type(bsplit) == tuple: + return 1 + elif type(asplit) == tuple and type(bsplit) == str: + return -1 + elif type(asplit) == tuple and type(bsplit) == tuple: + if asplit[0] in ('f','fc') and bsplit[0] in ('f','fc'): + return cmp(asplit[1], bsplit[1]) + else: + return cmp(asplit,bsplit) + else: + return cmp(asplit,bsplit) + + +class Branch(object): + + """Convenience class for handling branches to initial-merge""" + + def __init__(self, sha, origbranch): + self.sha, self.origbranch = sha, origbranch + a = self.origbranch.split('/') + assert(a[0] == 'origin') + assert(a[-1] == 'master') + self.localbranch = a[1] + + def __repr__(self): + return "%(sha)s %(origbranch)s" % self.__dict__ + + def __cmp__(self, other): + if self.sha == other.sha: + return cmp_relbranch(self.localbranch, other.localbranch) + else: + return cmp(self.sha, other.sha) + + +class Filter(object): + + """Branch filter + + Feed branches to this filter via eat(), and flush() it after you + are done. The filter will detect branches with the same tree sha, + and call do_initial_merge() for them. + """ + + def __init__(self, repo, dry_run=False): + self.repo = repo + self.dry_run = dry_run + self.__reset() + + def __reset(self): + self.branch_list = [] + + def do_initial_merge(self, into, to_merge): + log.info("#### Merging %s into %s ####", + [ x.localbranch for x in to_merge ], into.localbranch) + pyfedpkg.switch_branch(into.localbranch, self.repo.working_tree_dir) + log.info("Merging %s into %s", + repr([ x.origbranch for x in to_merge]), repr(into.localbranch)) + if not self.dry_run: + self.repo.git.merge('-m', merge_commit_msg, + '-s', 'ours', + *[x.origbranch for x in to_merge]) + for t in to_merge: + pyfedpkg.switch_branch(t.localbranch, self.repo.working_tree_dir) + log.info("Merging %s into %s", repr(into.localbranch), repr(t.localbranch)) + if not self.dry_run: + self.repo.git.merge(into.localbranch) + pyfedpkg.switch_branch(into.localbranch, self.repo.working_tree_dir) + + def flush(self): + if len(self.branch_list) < 2: + return + + head = self.branch_list[-1] + others = self.branch_list[:-1] + self.do_initial_merge(head, others) + + self.__reset() + + def eat(self, item): + """Feed item to the filter. The filter will decide what to do with it.""" + if self.branch_list: + last = self.branch_list[-1] + if last.sha == item.sha: + self.branch_list.extend([item]) + else: + self.flush() + self.branch_list = [item] + else: + self.branch_list = [item] + + +def handle_repo(repo, dry_run=False): + log.info("######## initial-merge %s ########" % repo.working_tree_dir) + _locals, remotes = pyfedpkg._list_branches(repo=repo) + aa = [ Branch(repo.git.rev_parse('%s^{tree}' % b), b) for b in remotes ] + aa.sort() + log.info("Branches sorted by tree sha:") + for x in aa: + log.info(" %s" % x) + + n = 0 + f = Filter(repo, dry_run=dry_run) + while n < len(aa): + f.eat(aa[n]) + n = n + 1 + f.flush() + + log.info("######## /initial-merge %s ########" % repo.working_tree_dir) + + +class DirListAction(argparse.Action): + + def __init__(self, *args, **kwargs): + super(DirListAction, self).__init__(*args, **kwargs) + self.default_value = ['.'] + + def __call__(self, parser, namespace, values, option_string=None): + if values: + setattr(namespace, self.dest, values) + else: + setattr(namespace, self.dest, self.default) + + +def fedpkg_command(args): + is_first = True + for repo_path in args.repo_path: + if is_first: + is_first = False + else: + log.info("") + repo = git.Repo(repo_path) + handle_repo(repo, dry_run=args.dry_run) + + +_module_doc = __doc__ + + +def add_parser_to(subparsers): + sp = subparsers.add_parser('initial-merge', + help = 'git merge to join branches with identical trees', + formatter_class=argparse.RawDescriptionHelpFormatter, + description = _module_doc) + sp.add_argument('repo_path', metavar='repo-path', + nargs='*', default=['.'], + action=DirListAction, + help = 'Path to a repo to initial-merge') + sp.add_argument('-n', '--dry-run', + default=False, const=True, action='store_const', + help="Whether to run without actually merging") + sp.set_defaults(command = fedpkg_command) + return sp -- cgit From 18ea3c13dc5bcda0c086efe11809795773f5167b Mon Sep 17 00:00:00 2001 From: Hans Ulrich Niedermann Date: Tue, 10 Aug 2010 13:13:15 +0200 Subject: Add "fedpkg clone --initial-merge" argument --- src/fedpkg.bash | 2 +- src/fedpkg.py | 14 +++++++++++--- src/pyfedpkg/__init__.py | 7 +++++-- src/pyfedpkg/initial_merge.py | 10 ++++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/fedpkg.bash b/src/fedpkg.bash index 5a20446..da37740 100644 --- a/src/fedpkg.bash +++ b/src/fedpkg.bash @@ -110,7 +110,7 @@ _fedpkg() options="--dry-run -x" ;; clone|co) - options="--branches --anonymous" + options="--branches --anonymous --initial-merge" options_branch="-b" after="package" ;; diff --git a/src/fedpkg.py b/src/fedpkg.py index 90c3be7..75770f5 100755 --- a/src/fedpkg.py +++ b/src/fedpkg.py @@ -446,6 +446,8 @@ def clone(args): pyfedpkg.clone_with_dirs(args.module[0], user) else: pyfedpkg.clone(args.module[0], user, args.path, args.branch) + if args.initial_merge: + pyfedpkg.initial_merge.handle_repo(args.module[0]) except pyfedpkg.FedpkgError, e: log.error('Could not clone: %s' % e) sys.exit(1) @@ -947,9 +949,15 @@ packages will be built sequentially. parser_clone = subparsers.add_parser('clone', help = 'Clone and checkout a module') # Allow an old style clone with subdirs for branches - parser_clone.add_argument('--branches', '-B', - action = 'store_true', - help = 'Do an old style checkout with subdirs for branches') + parser_clone_branches_group = parser_clone.add_mutually_exclusive_group() + parser_clone_branches_group.add_argument( + '--branches', '-B', + action = 'store_true', + help = 'Do an old style checkout with subdirs for branches') + parser_clone_branches_group.add_argument( + '--initial-merge', '-i', + action = 'store_true', + help = 'Run initial-merge on the cloned repo immediately') # provide a convenient way to get to a specific branch parser_clone.add_argument('--branch', '-b', help = 'Check out a specific branch') diff --git a/src/pyfedpkg/__init__.py b/src/pyfedpkg/__init__.py index 83b54f7..401f738 100644 --- a/src/pyfedpkg/__init__.py +++ b/src/pyfedpkg/__init__.py @@ -31,8 +31,6 @@ import OpenSSL import fnmatch import offtrac -from . import initial_merge - # Define some global variables, put them here to make it easy to change LOOKASIDE = 'http://pkgs.fedoraproject.org/repo/pkgs' @@ -50,6 +48,11 @@ BRANCHFILTER = 'f\d\d\/master|master|el\d\/master|olpc\d\/master' class FedpkgError(Exception): pass + +# This module needs FedpkgError to be defined +from . import initial_merge + + # Setup our logger # Null logger to avoid spurrious messages, add a handler in app code class NullHandler(logging.Handler): diff --git a/src/pyfedpkg/initial_merge.py b/src/pyfedpkg/initial_merge.py index 4964745..f22453f 100755 --- a/src/pyfedpkg/initial_merge.py +++ b/src/pyfedpkg/initial_merge.py @@ -169,7 +169,17 @@ class Filter(object): self.branch_list = [item] +class UnknownRepoTypeError(pyfedpkg.FedpkgError): + pass + + def handle_repo(repo, dry_run=False): + if type(repo) == str: + repo = git.Repo(repo) + elif isinstance(repo, git.Repo): + pass + else: + raise UnknownRepoTypeError("%s" % repo) log.info("######## initial-merge %s ########" % repo.working_tree_dir) _locals, remotes = pyfedpkg._list_branches(repo=repo) aa = [ Branch(repo.git.rev_parse('%s^{tree}' % b), b) for b in remotes ] -- cgit