rpmci-srpm-builder: Start of code

6 files changed, 203 insertions, 87 deletions

diff --git a/rpmci-srpm-builder b/rpmci-srpm-builder
new file mode 100755
index 0000000..18a3622
--- /dev/null
+++ b/rpmci-srpm-builder
@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+# rpmci-srpm-builder:
+# Create source RPMs after VCS changes
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+
+if os.path.isdir('.git'):
+    sys.path.insert(0, os.getcwd())
+
+import rpmci
+from rpmci.rpmci_srpm_builder_main import main
+
+if __name__ == '__main__':
+    main()
diff --git a/rpmci/artifact.py b/rpmci/artifact.py
index 4be1e76..1685074 100644
--- a/rpmci/artifact.py
+++ b/rpmci/artifact.py
@@ -7,6 +7,10 @@
 # Copyright (C) 2010 Red Hat, Inc.
 # Written by Colin Walters <walters@verbum.org>
 
+import os
+import sys
+import urllib
+
 FEDORA_ANONGIT_URL = 'git://pkgs.fedoraproject.org'
 
 class BuildTarget(object):
@@ -80,7 +84,8 @@ def fedora_git_url_for_build_target(config, buildtarget):
 
 def upstream_git_url_for_build_target(config, buildtarget):
     fedora_url = fedora_git_url_for_build_target(config, buildtarget)
-    escaped_url = urllib.quote(url, '')
+    escaped_url = urllib.quote(fedora_url, '')
+    mirror_dir = config.get('VCS', 'mirror_dir')
     vcsdir = os.path.join(mirror_dir, escaped_url)
     if not os.path.isdir(vcsdir):
         raise SystemExit("Not a directory: %r" % (vcsdir, ))
diff --git a/rpmci/dynrepo.py b/rpmci/dynrepo.py
index e6cdc90..505775d 100644
--- a/rpmci/dynrepo.py
+++ b/rpmci/dynrepo.py
@@ -25,80 +25,80 @@ class Repo(object):
         self._monitor = self._dir_gfile.monitor(gio.FILE_MONITOR_NONE)
         self._monitor.connect('changed', self._on_dir_changed)
 
-   def get_rpms(self):
-       return self._rpms
+    def get_rpms(self):
+        return self._rpms
 
-   def get_rpms_for_name(self, name, arch):
-       result = []
-       for filename,(n,a,e,v,r) in self._rpms.iteritems():
-           if n == name and (arch is None or a == arch):
-               result.append((filename, n, a, e, v, r))
-       return result
+    def get_rpms_for_name(self, name, arch):
+        result = []
+        for filename,(n,a,e,v,r) in self._rpms.iteritems():
+            if n == name and (arch is None or a == arch):
+                result.append((filename, n, a, e, v, r))
+        return result
 
-   def get_latest_srpm_for_name(self, name):
-       rpms = self.get_rpms_for_name(name, 'src')
-       cmpevr=rpmUtils.miscutils.compareEVR
-       if rpms:
-           rpms.sort(lambda a,b: cmpevr(a[3:6], b[3:6]))
-           return rpms[-1]
-       return None
+    def get_latest_srpm_for_name(self, name):
+        rpms = self.get_rpms_for_name(name, 'src')
+        cmpevr=rpmUtils.miscutils.compareEVR
+        if rpms:
+            rpms.sort(lambda a,b: cmpevr(a[3:6], b[3:6]))
+            return rpms[-1]
+        return None
+ 
+    def add_srpm(self, srpm_path):
+        basename = os.path.basename(srpm_path)
+        os.link(srpm_path, os.path.join(self._dir, basename))
 
-   def add_srpm(self, srpm_path):
-       basename = os.path.basename(srpm_path)
-       os.link(srpm_path, os.path.join(self._dir, basename))
-
-   def _delete_old_rpms_in_dir(self, dirpath):
-       proc = popen_verbose(['repomanage', '-o', '.'], stdout=subprocess.PIPE,
-                            stderr=sys.stderr,
-                            cwd=dirpath)
-       output = proc.communicate()[0]
-       for line in output.split('\n'):
-           if line.endswith('.rpm') and os.path.exists(line):
+    def _delete_old_rpms_in_dir(self, dirpath):
+        proc = popen_verbose(['repomanage', '-o', '.'], stdout=subprocess.PIPE,
+                             stderr=sys.stderr,
+                             cwd=dirpath)
+        output = proc.communicate()[0]
+        for line in output.split('\n'):
+             if line.endswith('.rpm') and os.path.exists(line):
                os.unlink(line)
 
-   def update_repo_sync(self):
-       self._delete_old_rpms_in_dir(dirpath)
-       subprocess.check_call(['createrepo', '.'], cwd=self._dir)
+    def update_repo_sync(self):
+        self._delete_old_rpms_in_dir(dirpath)
+        subprocess.check_call(['createrepo', '.'], cwd=self._dir)
 
-   def _headers_from_packages(self, rpmlist):
-       result = {}
-       ts = rpm.TransactionSet()
-       ts.setVSFlags(~(rpm._RPMVSF_NOPAYLOAD))
-       for pkg in rpmlist:
-           pkg_path = os.path.join(self._dir, pkg)
-           try:
+    def _headers_from_packages(self, rpmlist):
+        result = {}
+        ts = rpm.TransactionSet()
+        ts.setVSFlags(~(rpm._RPMVSF_NOPAYLOAD))
+        for pkg in rpmlist:
+            pkg_path = os.path.join(self._dir, pkg)
+            try:
                header = rpmUtils.miscutils.hdrFromPackage(ts, pkg_path)
-           except rpmUtils.RpmUtilsError, e:
-               logging.exception(e)
-               continue
-           (n,a,e,v,r) = rpmUtils.miscutils.pkgTupleFromHeader(header)
-           del header
-           result[pkg] = (n,a,e,v,r)
-       return result
+            except rpmUtils.RpmUtilsError, e:
+                logging.exception(e)
+                continue
+            (n,a,e,v,r) = rpmUtils.miscutils.pkgTupleFromHeader(header)
+            del header
+            result[pkg] = (n,a,e,v,r)
+        return result
 
-   def _on_dir_changed(self, mon, gfile, other, event):
-       self._reload()
+    def _on_dir_changed(self, mon, gfile, other, event):
+        self._reload()
 
-   def _reload(self):
-       dir_contents = os.listdir(self._dirpath)
-       messages = set()
-       rpmlist = set()
-       for filename in dir_contents:
-           if filename.endswith('.tmp'):
-               continue
-           if not filename.endswith('.rpm'):
-               continue
-           rpmlist.add(filename)
+    def _reload(self):
+        dir_contents = os.listdir(self._dirpath)
+        messages = set()
+        rpmlist = set()
+        for filename in dir_contents:
+            if filename.endswith('.tmp'):
+                continue
+            if not filename.endswith('.rpm'):
+                continue
+            rpmlist.add(filename)
         
-       deleted = []
-       for rpm in self._rpms:
-           if rpm not in rpmlist:
-               deleted.add(rpm)
-       for rpm in deleted:
-           del self._rpms[rpm]
-       new = []
-       for rpm in self._rpms:
-           if rpm not in rpmlist:
-               new.append(rpm)
-       for rpm,data in self._headers_from_packages(new).iteritems():
-           self._rpms[rpm] = data
+        deleted = []
+        for rpm in self._rpms:
+            if rpm not in rpmlist:
+                deleted.add(rpm)
+        for rpm in deleted:
+            del self._rpms[rpm]
+        new = []
+        for rpm in self._rpms:
+            if rpm not in rpmlist:
+                new.append(rpm)
+        for rpm,data in self._headers_from_packages(new).iteritems():
+            self._rpms[rpm] = data
diff --git a/rpmci/lame_vcs_abstraction.py b/rpmci/lame_vcs_abstraction.py
index d1ca557..f603809 100644
--- a/rpmci/lame_vcs_abstraction.py
+++ b/rpmci/lame_vcs_abstraction.py
@@ -120,6 +120,20 @@ class GitVcs(Vcs):
         assert self._dir is not None
         return self._vcs_exec_async(['git', 'pull', '-r'], logfile, on_exited)
 
+    def export_directory(self, commit_id, target_directory, log_output_stream):
+        assert self._dir is not None
+        basename = os.path.basename(target_directory)
+        if not basename.endswith('/'):
+            basename += '/'
+        basedir = os.path.basename(target_directory)
+        args = ['git', 'archive', '--format=tar', '--prefix=%s' % (basename,), commit_id]
+        logging.info("Synchronously executing: %r" % (args, ))
+        gitproc = subprocess.Popen(args, cwd=self._dir, stdout=subprocess.PIPE, stderr=log_output_stream)
+        args = ['tar', 'xf', '-']
+        logging.info("Synchronously executing: %r" % (args, ))
+        tarproc = subprocess.Popen(args, cwd=basedir, stdout=None, stdin=gitproc.stdout, stderr=log_output_stream)
+        tarproc.wait()
+
     def export_archive(self, commit_id, prefix, target_filename, log_output_stream):
         assert self._dir is not None
         if not prefix.endswith('/'):
diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py
index 2bfbef4..3b44168 100644
--- a/rpmci/rpmci_srpm_builder_main.py
+++ b/rpmci/rpmci_srpm_builder_main.py
@@ -22,38 +22,110 @@ import gio
 
 from . import msgqueue
 from . import dynrepo
+from . import artifact
 from . import lame_vcs_abstraction
 
 class SRPMBuilder(object):
     def __init__(self, options, config):
         self.config = config
-        self.urls = urls
 
         self._options = options
 
         vcs_msgqueue_dir = config.get('VCS', 'msgqueue')
         self._vcs_msgqueue = msgqueue.MessageQueue(vcs_msgqueue_dir)
 
+        srpm_msgqueue_dir = config.get('SRPM', 'msgqueue')
+        self._srpm_msgqueue = msgqueue.MessageQueue(srpm_msgqueue_dir)
+
         self._artifactset = artifact.ArtifactSet.from_config(config)
-        self._vcs_by_url = {}
+        self._target_vcs = {}
         for target in self._artifactset.get_build_targets():
-            fedora_url = artifact.fedora_git_url_for_build_target(target)
-            upstream_url = artifact.upstream_git_url_for_build_target(target)
-            self._vcs_by_url[fedora_url] = lame_vcs_abstraction.VCS.new_from_spec(fedora_url)
-            self._vcs_by_url[upstream_url] = lame_vcs_abstraction.VCS.new_from_spec(upstream_url)
-
-        self._srcrepo = dynrepo.Repo(config.get('SRPM', 'srpm_dir'))
+            fedora_url = artifact.fedora_git_url_for_build_target(self.config, target)
+            upstream_url = artifact.upstream_git_url_for_build_target(self.config, target)
+            fedora_vcs = lame_vcs_abstraction.VCS.new_from_spec(fedora_url)
+            upstream_vcs = lame_vcs_abstraction.VCS.new_from_spec(upstream_url)
+            self._target_vcs[target] = (fedora_vcs, upstream_vcs)
+        
+        self._srcdir = config.get('SRPM', 'srpm_dir')
+        self._srcrepo = dynrepo.Repo(self._srcdir)
 
     def start(self):
         self._vcs_msgqueue.connect(self._on_vcs_message)
+
+    def _get_base_rpmbuild_args(self, src_dirpath):
+        return list(["--define '_sourcedir %s'" % src_dirpath,
+                     "--define '_specdir %s'" % src_dirpath,
+                     "--define '_builddir %s'" % src_dirpath,
+                     "--define '_srcrpmdir %s'" % src_dirpath,
+                     "--define '_rpmdir %s'" % src_dirpath])
         
     def _on_vcs_message(self, q, messages):
         for msg in messages:
             url = msg.payload.url
-            q.consume(msg)
-            vcs = self._vcs_by_url[url]
-            
+            self._handle_vcs_message_url(q, url)
+
+    def _handle_vcs_message_url(self, q, url):
+        q.consume(msg)
+        is_fedora = False
+        target = None
+        for iter_target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems():
+            fedora_url = fedora_vcs.get_url_string()
+            is_fedora = (url == fedora_url)
+            if not is_fedora:
+                upstream_url = upstream_vcs.get_url_string()
+                if not url == upstream_url:
+                    continue
+            target = iter_target
+            break
+        assert target is not None, ("Couldn't find target for url %r" % (url, ))
+        (fedora_vcs, upstream_vcs) = self._target[target]
+
+        work_dir = tempfile.mkdtemp('.tmp', 'srpm-builder')
+        logging.debug("Creating new SRPM for %r in %r" % (target.module, work_dir))
+        fedora_dir = os.path.join(work_dir, target.module)
+        fedora_vcs.export_directory('HEAD', fedora_dir, sys.stderr) 
+
+        specname = target.module + '.spec'
+        target_spec_path = os.path.join(fedora_dir, specname)
+        target_spec_obj = spec.Spec(target_spec_path)
+
+        orig_spec_path = os.path.join(self._srcdir, specname)
+        if os.path.isfile(orig_spec_path):
+            # Okay, we have a previous build, so now we "merge" by taking everything
+            # from the new spec file, except we use the Version/Release from the old
+            # one.  This is necessary to pacify RPM's insistence on globally
+            # ordering versions, rather than simply having repository build versions.
+            orig_spec_obj = spec.Spec(orig_spec_path)
+            orig_version = orig_spec_obj.get_key('Version')
+            orig_release = orig_spec_obj.get_key('Release')
+
+            target_spec_obj.set_key('Version', orig_version)
+            target_spec_obj.set_key('Release', orig_release)
+            target_spec_obj.save()
             
+        exec_basedir = os.path.dirname(sys.argv[0])
+        subprocess.check_call([os.path.join(exec_basedir, 'rpmci-spec-vcs'),
+                               '--vcsdir=' + upstream_vcs.get_directory(),
+                               'set-revision', 'HEAD'],
+                              stdout=sys.stdout, stderr=sys.stderr,
+                              cwd=fedora_dir)
+        args = ['rpmbuild']
+        args.extend(self._get_base_rpmbuild_args(fedora_dir))
+        args.extend(['-bs', specname])
+        subprocess.check_call(args, cwd=fedora_dir, stdout=sys.stdout,
+                              stderr=sys.stderr)
+        srpm_path = None
+        srpm_basename = None
+        for filename in os.listdir(fedora_dir):
+            if filename.endswith('.src.rpm'):
+                srpm_basename = filename
+                srpm_path = os.path.join(fedora_dir, filename)
+                break
+        assert srpm_path is not None
+        os.rename(srpm_path, os.path.join(self._srcdir, srpm_basename))
+        self._srcrepo.update_repo_sync()
+        
+        self._srpm_msgqueue.append(msgqueue.Message({'type': 'srpm'}, {'filename': srpm_basename}))
 
 def main():
     glib.threads_init()
@@ -73,7 +145,7 @@ def main():
     level = logging.DEBUG if options.debug else logging.INFO
     logging.basicConfig(stream=sys.stderr, level=level)
 
-    builder = SRPMBuilder(options, config, urls)
+    builder = SRPMBuilder(options, config)
     builder.start()
     
     loop = glib.MainLoop()
diff --git a/rpmci/spec.py b/rpmci/spec.py
index aa804d8..6d3bba2 100644
--- a/rpmci/spec.py
+++ b/rpmci/spec.py
@@ -30,10 +30,10 @@ class Spec(object):
         f.close()
         self._saved = False
         self._append_buildrequires = []
-        self._new_release = None
         self._source_dirname = None
         self._source_archivename = None
         self._substitutions = []
+        self._set_keys = {} # name -> value
         # Map from section name (e.g. '%build') -> (list of functions)
         self._section_filters = {}
         self._added_patches = []
@@ -87,7 +87,7 @@ class Spec(object):
         if release_has_dist:
             new_release += '%{?dist}'
             
-        self._new_release = new_release
+        self.set_key('Release', new_release)
         
     def set_source(self, dirname, archivename):
         assert not self._saved
@@ -126,7 +126,7 @@ class Spec(object):
             section_end = len(self._lines) - 1
         return (section_start, section_end)
 
-    def replace_key_line(self, key, new_value, line):
+    def _replace_key_line(self, key, new_value, line):
         """Takes a line of the form "Release: 42  # foo" and replaces
 the 42 with new_value, preserving the comment # foo."""
         comment = line.rfind('#')
@@ -209,15 +209,16 @@ the 42 with new_value, preserving the comment # foo."""
                 output.write('%%setup -q -n %s\n' % self._source_dirname)
             elif ':' in line:
                 key, value = line.split(':', 1)
-                if key == 'Release' and self._new_release:
-                    output.write(self.replace_key_line(key, self._new_release, line))
-                elif (line.startswith('Source0:') or line.startswith('Source:')) and self._source_archivename:
-                    output.write(self.replace_key_line(key, self._source_archivename, line))
+                if (line.startswith('Source0:') or line.startswith('Source:')) and self._source_archivename:
+                    output.write("""# Snapshot created from upstream version control by rpmci-spec-vcs.\n"""
+                    output.write(self._replace_key_line(key, self._source_archivename, line))
                 elif key == 'BuildRequires' and not wrote_buildrequires:
                     output.write(line)
                     for req in self._append_buildrequires:
                         output.write('BuildRequires: %s\n' % req)
                     wrote_buildrequires = True
+                elif key in self._set_keys:
+                    output.write(self._replace_key_line(key, self._set_keys[key], line))
                 else:
                     output.write(line)
             else:
@@ -259,6 +260,10 @@ the 42 with new_value, preserving the comment # foo."""
                 result.append(line[len(key):].strip())
         return result
 
+    def set_key(self, key, value):
+        """Only supports one key-value pair."""
+        self._set_keys[key] = value
+
     def __str__(self):
         return self._filename