rpmci-binrpm-builder: New file

Initial hacking on binary builder.

3 files changed, 248 insertions, 0 deletions

diff --git a/rpmci-binrpm-builder b/rpmci-binrpm-builder
new file mode 100755
index 0000000..464e799
--- /dev/null
+++ b/rpmci-binrpm-builder
@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+# rpmci-binrpm-builder:
+# Create binary RPMs after SRPM changes
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+
+if os.path.isdir('.git'):
+    sys.path.insert(0, os.getcwd())
+
+import rpmci
+from rpmci.rpmci_binrpm_builder_main import main
+
+if __name__ == '__main__':
+    main()
diff --git a/rpmci/rpmci_binrpm_builder_main.py b/rpmci/rpmci_binrpm_builder_main.py
new file mode 100644
index 0000000..d8dd02b
--- /dev/null
+++ b/rpmci/rpmci_binrpm_builder_main.py
@@ -0,0 +1,129 @@
+#!/usr/bin/python
+
+# rpmci_binrpm_builder_main.py:
+# Implementation of rpm-binrpm-builder
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+import time
+import shutil
+import optparse
+from ConfigParser import SafeConfigParser
+import logging
+import tempfile
+import subprocess
+import urllib
+
+import glib
+import gobject
+import gio
+
+from . import msgqueue
+from . import subtask
+from . import dynrepo
+from . import artifact
+from . import spec
+from . import lame_vcs_abstraction
+
+class BinRPMBuilder(object):
+    def __init__(self, options, config):
+        self.config = config
+
+        self._options = options
+
+        srpm_msgqueue_dir = config.get('SRPM', 'msgqueue')
+        self._srpm_msgqueue = msgqueue.MessageQueue(srpm_msgqueue_dir)
+
+        self._artifactset = artifact.ArtifactSet.from_config(config)
+        self._artifact_to_repo = {}
+        artifact_basedir = config.get('build', 'artifactdir')
+        for artifact in self._artifactset.artifacts:
+            artifact_repopath = os.path.join(artifact_basedir, artifact.name)
+            self._artifact_to_repo[artifact] = dynrepo.Repo(artifact_repopath)
+
+        self._srcrepo = dynrepo.Repo(config.get('SRPM', 'spm_dir'))
+
+    def start(self):
+        self._srpm_msgqueue.connect(self._on_srpm_message)
+
+    def _target_for_srpm(self, srpm_name):
+        (name, ver, rest) = srpm_name.rsplit('-', 2)
+        for target in self._artifactset.get_build_targets():
+            if target.name == name:
+                return target
+        return None
+
+    def _on_srpm_message(self, q, messages):
+        msg_list = list(messages)
+        num_msgs = len(msg_list)
+        logging.debug("Starting processing of %d messages" % (num_msgs, ))
+        for msg in msg_list:
+            srpm_name = msg.payload['srpm']
+            logging.debug("Processing update for SRPM %r" % (srpm_name, ))
+            target = self._target_for_srpm(srpm_name)
+            if target is None:
+                logging.warn("Skipping unknown srpm %r" % (srpm_name, ))
+                continue
+            affected_artifacts = []
+            for artifact in self._artifactset.artifacts:
+                if target in artifact.targets:
+                    affected_artifacts.append(artifact)
+            
+            self._handle_srpm_message_name(q, srpm_name)
+            q.consume(msg)
+        logging.debug("Processed %d messages successfully" % (num_msgs, ))
+
+    def _handle_srpm_message_name(self, q, srpm_name):
+        exec_basedir = os.path.dirname(sys.argv[0])
+        
+        subtask.spawn_sync('mock-many-srpms' % (target.module, ),
+                           args, cwd=fedora_dir)
+
+        args = ['rpmbuild']
+        args.extend(self._get_base_rpmbuild_args(fedora_dir))
+        args.extend(['-bs', specname])
+        subtask.spawn_sync('rpmbuild-srpm-%s' % (target.module, ),
+                           args, cwd=fedora_dir)
+        srpm_path = None
+        srpm_basename = None
+        for filename in os.listdir(fedora_dir):
+            if filename.endswith('.src.rpm'):
+                srpm_basename = filename
+                srpm_path = os.path.join(fedora_dir, filename)
+                break
+        assert srpm_path is not None
+        os.rename(srpm_path, os.path.join(self._srcdir, srpm_basename))
+        self._srcrepo.update_repo_sync()
+        
+        self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'filename': srpm_basename}))
+
+def main():
+    glib.threads_init()
+
+    opts = optparse.OptionParser("usage: %prog [options]")
+    opts.add_option('-c', '--config', dest='config', help="Path to configuration file")
+    opts.add_option('', '--debug', action='store_true', help="Print verbose debugging")
+
+    (options, args) = opts.parse_args()
+
+    if options.config is None:
+        print "Must specify --config"
+        sys.exit(1)
+
+    config = SafeConfigParser({'home': os.environ['HOME']})
+    config.read(options.config)
+    level = logging.DEBUG if options.debug else logging.INFO
+    logging.basicConfig(stream=sys.stderr, level=level)
+
+    subtask.global_configure(config)
+
+    builder = SRPMBuilder(options, config)
+    builder.start()
+    
+    logging.info("Awaiting events")
+    loop = glib.MainLoop()
+    loop.run()
diff --git a/rpmci/versioned_repos.py b/rpmci/versioned_repos.py
new file mode 100644
index 0000000..6de753c
--- /dev/null
+++ b/rpmci/versioned_repos.py
@@ -0,0 +1,99 @@
+#!/usr/bin/python
+
+# versioned_repos.py:
+# Directory of "versioned" repositories, sharing hard-linked files
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+import logging
+
+import glib
+import gio
+import rpm
+import rpmUtils
+import rpmUtils.miscutils
+
+from . import subtask
+
+class Repo(object):
+    def __init__(self, basedir, name, version):
+        self.name = name
+        self.version = version
+        self._dirname = '%d' % (version, )
+        self._dir = os.path.join(basedir, self._dirname)
+        self._task_basename = 'repo-%s' % (name, )
+        if not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+
+    def get_dir(self):
+        return self._dir
+
+    def add_rpm(self, rpm_path):
+        basename = os.path.basename(rpm_path)
+        os.link(rpm_path, os.path.join(self._dir, basename))
+
+    def iter_rpms(self):
+        for filename in os.listdir(self._dir):
+            if filename.endswith('.rpm'):
+                yield os.path.join(self._dir, filename)
+
+    def _delete_old_rpms_in_dir(self, dirpath):
+        output = subtask.spawn_sync_get_output('repomanage-' + self._task_basename,
+                                               ['repomanage', '-o', '.'], cwd=dirpath)
+        for line in output.split('\n'):
+             if line.endswith('.rpm') and os.path.exists(line):
+               os.unlink(line)
+
+    def update_sync(self):
+        self._delete_old_rpms_in_dir(self._dir)
+        subtask.spawn_sync('createrepo' + self._task_basename,
+                           ['createrepo', '.'], cwd=self._dir)
+
+class VersionedRepo(object):
+    def __init__(self, dirpath):
+        self._dir = dirpath
+        if not os.path.isdir(self._dir):
+            os.makedirs(self._dir)
+        self._name = os.path.basename(dirpath)
+        self._versions = {}
+        self._last_version = -1
+        self._reload()
+
+    def get_versions(self):
+        return self._versions
+
+    def _reload(self):
+        self._versions = {}
+        for filename in os.listdir(self._dir):
+            path = os.path.join(self,_dir, filename)
+            if not os.path.isdir(path):
+                continue
+            try:
+                revision = int(filename)
+            except ValueError, e:
+                continue
+            self._versions[revision] = Repo(self._dir, self._name, revision)
+            if revision > self._last_version:
+                self._last_version = revision
+ 
+    def get_latest_path(self):
+        if self._last_version == -1:
+            return None
+        return self._versions[self._last_version].get_dir()
+        
+    def commit_sync(self, new_rpms):
+        rev = self._last_version + 1
+        new_repo = Repo(self._dir, self._name, rev)
+        if rev > 0:
+            prev_repo = self._versions[self._last_version]
+            for rpm in prev_repo.iter_rpms():
+                new_repo.add_rpm(rpm)
+        for rpm in new_rpms:
+            new_repo.add_rpm(rpm)
+        new_repo.update_sync()
+        return rev
+