Rename fedpkg-make-pull to fedpkg-vcs

It's now subcommand based, with e.g. "pull" and "cherrypick" actions.

2 files changed, 214 insertions, 101 deletions

diff --git a/fedpkg-pull-build-chain b/fedpkg-pull-build-chain
index 7a0339c..9c1d35c 100755
--- a/fedpkg-pull-build-chain
+++ b/fedpkg-pull-build-chain
@@ -140,9 +140,9 @@ def main():
         else:
             last_build_succeeded = False
                 
-        print "Running fedpkg-make-pull"
+        print "Running fedpkg-vcs"
         notify_state('upstream-vcs', { 'module': arg })
-        args = ['fedpkg-make-pull', '--status-file=' + os.path.abspath('pull-status')]
+        args = ['fedpkg-vcs', 'pull-update', '--status-file=' + os.path.abspath('pull-status')]
         if force or not last_build_succeeded:
             args.append('--force')
         try:
diff --git a/fedpkg-make-pull b/fedpkg-vcs
index 39f0f15..3b73fff 100755
--- a/fedpkg-make-pull
+++ b/fedpkg-vcs
@@ -37,6 +37,9 @@ class Vcs(object):
                                                             '', '', ''))
         self._branch = self._parsed_url.fragment
         
+    def get_url(self):
+        return self._parsed_url
+        
     def checkout(self, destdir):
         """Retrieve a new copy of the source tree, saving as destdir"""
         pass
@@ -53,8 +56,10 @@ class Vcs(object):
         
     def _vcs_exec(self, *args, **kwargs):
         print "Running: %r" % (args[0], )
-        kwargs['stdout'] = sys.stdout
-        kwargs['stderr'] = subprocess.STDOUT
+        if not 'stdout' in kwargs:
+            kwargs['stdout'] = sys.stdout
+        if not 'stderr' in kwargs:
+            kwargs['stderr'] = sys.stderr
         subprocess.check_call(*args, **kwargs)
         
     @classmethod
@@ -79,10 +84,20 @@ class GitVcs(Vcs):
             self._vcs_exec(['git', 'checkout', self._branch], cwd=directory)
         self._vcs_exec(['git', 'pull', '-r'], cwd=directory)
         
+    def get_commit_as_patch(self, directory, commitid, destfile):
+        f = open(destfile, 'w')
+        self._vcs_exec(['git', 'format-patch', '--stdout', commitid + '^..' + commitid],
+                        cwd=directory, stdout=f, stderr=sys.stderr)
+        f.close()   
+        
     def get_id(self, directory):
         output = subprocess.Popen(['git', 'show', '--format=%H'], stdout=subprocess.PIPE, cwd=directory).communicate()[0]
         return output.split('\n')[0]
         
+    def get_commit_summary_as_filename(self, directory, commitid):
+        output = subprocess.Popen(['git', 'show', '--format=%f', commitid], stdout=subprocess.PIPE, cwd=directory).communicate()[0]
+        return output.split('\n')[0]
+        
 class BuildSystem(object):
     def __init__(self, directory):
         self._directory = directory
@@ -125,19 +140,26 @@ class Spec(object):
         f = open(filename)
         self._lines = f.readlines()
         f.close()
+        self._saved = False
         
         self._append_buildrequires = []
         self._new_release = None
         self._source_dirname = None
         self._source_archivename = None
         self._substitutions = []
+        self._added_patches = []
+
+    def get_name(self):
+        return self._filename[:-5]
         
     def add_buildrequires(self, new_buildrequires):
+        assert not self._saved
         current_buildrequires = self.get_key_allvalues('BuildRequires')
         new_buildrequires = filter(lambda x: x not in current_buildrequires, new_buildrequires)
         self._append_buildrequires = new_buildrequires
         
     def increment_release_snapshot(self, identifier):
+        assert not self._saved
         cur_release = self.get_key('Release')
         release_has_dist = cur_release.endswith('%{?dist}')
         if release_has_dist:
@@ -160,29 +182,85 @@ class Spec(object):
         self._new_release = new_release
         
     def set_source(self, dirname, archivename):
+        assert not self._saved
         self._source_dirname = dirname
         self._source_archivename = archivename
         
     def substitute(self, substitutions):
+        assert not self._saved
         self._substitutions = substitutions
         
-    def save(self, new_filename):
+    def add_patch(self, filename):
+        patches = self.get_patches()
+        if len(patches) == 0:
+            patchnum = 0
+        else:
+            patchnums = map(lambda a: a[0], patches)
+            patchnum = max(patchnums)
+        self._added_patches.append(filename)
+        
+    def save(self):
+        self._saved = True
+        tmpname = self._filename + '.tmp'
+        self.save_as(tmpname)
+        os.rename(tmpname, self._filename)
+        
+    def save_as(self, new_filename):
         wrote_buildrequires = False
         output = open(new_filename, 'w')
-        for line in self._lines:
+        
+        apply_patchmeta_at_line = -1
+        apply_patch_apply_at_line = -1
+        source_re = re.compile(r'^Source([0-9]*):')
+        patch_re = re.compile(r'^Patch([0-9]+):')
+        apply_re = re.compile(r'^%patch')
+        highest_patchnum = -1
+        for i,line in enumerate(self._lines):
+            match = patch_re.search(line)
+            if match:
+                apply_patchmeta_at_line = i
+                highest_patchnum = int(match.group(1))
+                continue
+            match = source_re.search(line)
+            if match:
+                apply_patchmeta_at_line = i
+                if highest_patchnum == -1:
+                    highest_patchnum = 0
+                continue
+            if line.startswith('%setup'):
+                apply_patch_apply_at_line = i + 1
+                continue
+            match = apply_re.search(line)
+            if match:
+                apply_patch_apply_at_line = i + 1
+                continue
+        if apply_patchmeta_at_line == -1:
+            print "Error: Couldn't determine where to add Patch:"
+            sys.exit(1)
+        if apply_patch_apply_at_line == -1:
+            print "Error: Couldn't determine where to add %patch"
+            sys.exit(1)
+        for i,line in enumerate(self._lines):
+            if i == apply_patchmeta_at_line:
+                for pnum,patch in enumerate(self._added_patches):
+                    output.write('Patch%d: %s\n' % (highest_patchnum + pnum + 1, patch))
+            elif i == apply_patch_apply_at_line:
+                for pnum,patch in enumerate(self._added_patches):
+                    output.write('%%patch%d -p1\n' % (highest_patchnum + pnum + 1, ))
+        
             replacement_matched = False
             for sub_re, replacement in self._substitutions:
                 (line, subcount) = sub_re.subn(replacement, line)
                 if subcount > 0:
                     replacement_matched = True
-                    break
+                    break                
             if replacement_matched:
                 output.write(line)
-            elif line.startswith('Release:'):
+            elif line.startswith('Release:') and self._new_release:
                 output.write('Release: %s\n' % self._new_release)
-            elif line.startswith('Source0:'):
+            elif line.startswith('Source0:') and self._source_archivename:
                 output.write('Source0: %s\n' % self._source_archivename)
-            elif line.startswith('%setup'):  # This is dumb, need to automate this in RPM
+            elif line.startswith('%setup') and self._source_dirname:  # This is dumb, need to automate this in RPM
                 output.write('%%setup -q -n %s\n' % self._source_dirname)
             elif line.startswith('BuildRequires:') and not wrote_buildrequires:
                 output.write(line)
@@ -193,6 +271,16 @@ class Spec(object):
                 output.write(line)
                 
         output.close()
+        
+    def get_patches(self):
+        patchre = re.compile(r'^Patch([0-9]+):')
+        patches = []
+        for line in self._lines:
+            match = patchre.search(line)
+            if not match:
+                continue
+            patches.append((int(match.group(1)), line.split(':', 1)[1].strip()))
+        return patches
 
     def get_version(self):
         return self.get_key('Version')
@@ -218,14 +306,119 @@ class Spec(object):
                 result.append(line[len(key):].strip())
         return result
         
+def command_checkout(spec, vcs, vcsdir, args=[], opts={}):
+    if os.path.exists(vcsdir):
+        print "VCS directory %r already exists" % (vcsdir, )
+        return
+    print "Checking out from %r into new directory %r" % (vcs.get_url(), vcsdir)
+    vcs.checkout(vcsdir)
+    
+def command_pull(spec, vcs, vcsdir, args=[], opts={}):
+    if not os.path.exists(vcsdir):
+        command_checkout(spec, vcs, vcsdir, args=args, opts=opts)
+    print "Updating from %r existing directory %r" % (vcs.get_url(), vcsdir)
+    oldid = vcs.get_id(vcsdir)
+    vcs.update(vcsdir)
+    newid = vcs.get_id(vcsdir)
+    if oldid == newid and not opts['force']:
+        print "No changes upstream"
+        if opts['statusfile'] is not None:
+            f = open(opt_statusfile, 'w')
+            f.write('unchanged')
+            f.close()
+        sys.exit(0)
+        
+def command_pull_update(spec, vcs, vcsdir, args=[], opts={}):
+    command_pull(spec, vcs, vcsdir, args=args, opts=opts)
+    
+    name = spec.get_name()
+    newid = vcs.get_id(vcsdir)
+
+    snapshot_dirname = '%s-%s%s%s' % (name, version, vcs.get_scheme(), newid)
+    snapshot_archivename = snapshot_dirname + '.tar.bz2'
+    subprocess.check_call(['tar', '-cj', r'--transform=s,^\.,' + snapshot_dirname + ',', '-f', '../' + snapshot_archivename, '.'], cwd=vcsdir)
+    
+    buildsys = BuildSystem.new_from_directory(vcsdir)
+    if buildsys is None:
+        print "WARNING: Unrecognized buildsystem in directory %r" % (vcsdir, )
+    else:
+        spec.add_buildrequires(buildsys.get_bootstrap_buildrequires())
+        spec.substitute(buildsys.get_substitutions())
+
+    spec.set_source(snapshot_dirname, snapshot_archivename)
+    spec.increment_release_snapshot(newid)
+    spec.save()
+    
+    snapshot_md5 = hashlib.md5()
+    f = open(snapshot_archivename)
+    b = f.read(8192)
+    while b != '':
+        snapshot_md5.update(b)
+        b = f.read(8192)
+    f.close()
+        
+    snapshot_md5 = snapshot_md5.hexdigest()
+    
+    f = open('sources', 'w')
+    f.write(snapshot_md5)
+    f.write(' ')
+    f.write(snapshot_archivename)
+    f.write('\n')
+    f.close()
+            
+    print "Updated %s and sources file" % (targetspec, )
+    print "If you want to upload to Fedora, you'll need to run:"
+    print "  make upload FILE=%s" % (snapshot_archivename, )
+    print "  cvs commit && make tag build"
+    if opt_statusfile is not None:
+        f = open(opt_statusfile, 'w')
+        f.write('updated')
+        f.close()
+        
+def command_cherrypick(spec, vcs, vcsdir, args=[], opts={}):
+    if len(args) != 1:
+        print "Usage: git-vcs cherrypick COMMITID"
+        sys.exit(1)
+        
+    commitid = args[0]
+
+    filename = vcs.get_commit_summary_as_filename(vcsdir, commitid)
+    filename += '.patch'
+    if os.path.exists(filename):
+        print "Error: File %r already exists" % (filename, )
+        sys.exit(1)
+    vcs.get_commit_as_patch(vcsdir, commitid, filename)
+    
+    spec.add_patch(filename)
+    spec.save()
+    subprocess.check_call(['cvs', 'add', filename])
+    print "Successfully added patch %r" % (filename, )
+    
 def main():
+    valid_commands = { 'checkout': (command_checkout, "Perform an initial checkout of upstream revision control"),
+                       'pull': (command_pull, "Pull the latest upstream code"),
+                       'pull-update': (command_pull_update, "Pull the latest upstream, modify spec file to use it"),
+                       'cherrypick': (command_cherrypick, "Apply a specific commit id as a patch to specfile") }
+    def usage(ecode):
+        print ""
+        print "Usage: fedpkg-vcs COMMAND [-f]"
+        print "Valid commands:"
+        for cmdname in valid_commands:
+            (cmdfunc, description) = valid_commands[cmdname]
+            print "  %s: %s" % (cmdname, description)
+        sys.exit(ecode)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'f', ['force', 'status-file='])
+        opts, args = getopt.getopt(sys.argv[1:], 'f', ['force', 'status-file=', 'help'])
     except getopt.GetoptError, e:
         print unicode(e)
-        print ""
-        print "Usage: make-pull.py [-f]"
-        sys.exit(1)
+        usage(1)
+        
+    if len(args) < 1:
+        usage(1)
+        
+    cmd = args[0]
+    if not cmd in valid_commands:
+        usage(1)
         
     force = False
     opt_statusfile = None
@@ -234,6 +427,8 @@ def main():
             force = True
         elif o in ('--status-file', ):
             opt_statusfile = a
+        elif o in ('--help', ):
+            usage(0)
 
     targetspec = None
     for filename in os.listdir('.'):
@@ -244,8 +439,6 @@ def main():
         sys.stderr.write("Couldn't find spec file\n")
         sys.exit(1)
     spec = Spec(targetspec)
-    name = targetspec[:-5]
-    version = spec.get_version()
 
     f = open('sources')
     lines = f.readlines()
@@ -261,92 +454,12 @@ def main():
         sys.exit(1)
     
     vcs = Vcs.new_from_url(vcsurl)
+    vcsdir = '%s.%s' % (spec.get_name(), vcs.get_scheme())
+ 
+    opts = {'force': force,
+            'statusfile': opt_statusfile}
+    valid_commands[cmd][0](spec, vcs, vcsdir, args=args[1:], opts=opts)
    
-    targetdir = '%s.%s' % (name, vcs.get_scheme())
-    
-    if os.path.exists(targetdir):
-        print "Updating from %r existing directory %r" % (vcsurl, targetdir)
-        oldid = vcs.get_id(targetdir)
-        vcs.update(targetdir)
-        newid = vcs.get_id(targetdir)
-        if oldid == newid and not force:
-            print "No changes upstream"
-            if opt_statusfile is not None:
-                f = open(opt_statusfile, 'w')
-                f.write('unchanged')
-                f.close()
-            sys.exit(0)
-    else:
-        print "Checking out from %r into new directory %r" % (vcsurl, targetdir)
-        vcs.checkout(targetdir)
-        newid = vcs.get_id(targetdir)
-   
-    snapshot_dirname = '%s-%s%s%s' % (name, version, vcs.get_scheme(), newid)
-    snapshot_archivename = snapshot_dirname + '.tar.bz2'
-    subprocess.check_call(['tar', '-cj', r'--transform=s,^\.,' + snapshot_dirname + ',', '-f', '../' + snapshot_archivename, '.'], cwd=targetdir)
-    
-    new_specname = '%s-%s%s.spec.tmp' % (name, vcs.get_scheme(), newid)
-    
-    buildsys = BuildSystem.new_from_directory(targetdir)
-    if buildsys is None:
-        print "WARNING: Unrecognized buildsystem in directory %r" % (targetdir, )
-    else:
-        spec.add_buildrequires(buildsys.get_bootstrap_buildrequires())
-        spec.substitute(buildsys.get_substitutions())
-
-    spec.set_source(snapshot_dirname, snapshot_archivename)
-    spec.increment_release_snapshot(newid)
-    spec.save(new_specname)
-
-    if False:  # This stuff is buggy...
-        tempdir_name = new_specname + '.dir'
-        os.mkdir(tempdir_name)
-        # Ok, this is a gross hack...parse Source/Patch? from .spec?
-        for filename in os.listdir('.'):
-            if filename.endswith('.patch'):
-                shutil.copy(filename, os.path.join(tempdir_name, filename))
-        os.rename(snapshot_archivename, os.path.join(tempdir_name, snapshot_archivename))
-        os.rename(new_specname, os.path.join(tempdir_name, targetspec))
-        subprocess.check_call(['rpmbuild', '--nodeps', 
-                               '--define', '_sourcedir .',
-                               '--define', '_specdir .',
-                               '--define', '_builddir .',
-                               '--define', '_rpmdir .',
-                               '--define', '_srcrpmdir .',
-                               '-bs', targetspec], cwd=tempdir_name)
-        srpm_name = filter(lambda x: x.endswith('.src.rpm'), os.listdir(tempdir_name))[0]
-        os.rename(os.path.join(tempdir_name, srpm_name), srpm_name)
-        shutil.rmtree(tempdir_name)
-        print "Created SRPM: %s" % (srpm_name, )
-    elif True:
-        os.rename(new_specname, targetspec)
-        
-        snapshot_md5 = hashlib.md5()
-        f = open(snapshot_archivename)
-        b = f.read(8192)
-        while b != '':
-            snapshot_md5.update(b)
-            b = f.read(8192)
-        f.close()
-        
-        snapshot_md5 = snapshot_md5.hexdigest()
-        
-        f = open('sources', 'w')
-        f.write(snapshot_md5)
-        f.write(' ')
-        f.write(snapshot_archivename)
-        f.write('\n')
-        f.close()
-            
-        print "Updated %s and sources file" % (targetspec, )
-        print "If you want to upload to Fedora, you'll need to run:"
-        print "  make upload FILE=%s" % (snapshot_archivename, )
-        print "  cvs commit && make tag build"
-        if opt_statusfile is not None:
-            f = open(opt_statusfile, 'w')
-            f.write('updated')
-            f.close()   
-
     sys.exit(0)
         
 if __name__ == '__main__':