From 4dbd6dbea57d79b0d2a06ec051d0ac160df1d414 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 7 Oct 2010 22:08:12 -0400 Subject: rpmci-binrpm-builder: New file Initial hacking on binary builder. --- rpmci-binrpm-builder | 20 ++++++ rpmci/rpmci_binrpm_builder_main.py | 129 +++++++++++++++++++++++++++++++++++++ rpmci/versioned_repos.py | 99 ++++++++++++++++++++++++++++ 3 files changed, 248 insertions(+) create mode 100755 rpmci-binrpm-builder create mode 100644 rpmci/rpmci_binrpm_builder_main.py create mode 100644 rpmci/versioned_repos.py diff --git a/rpmci-binrpm-builder b/rpmci-binrpm-builder new file mode 100755 index 0000000..464e799 --- /dev/null +++ b/rpmci-binrpm-builder @@ -0,0 +1,20 @@ +#!/usr/bin/python + +# rpmci-binrpm-builder: +# Create binary RPMs after SRPM changes +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys + +if os.path.isdir('.git'): + sys.path.insert(0, os.getcwd()) + +import rpmci +from rpmci.rpmci_binrpm_builder_main import main + +if __name__ == '__main__': + main() diff --git a/rpmci/rpmci_binrpm_builder_main.py b/rpmci/rpmci_binrpm_builder_main.py new file mode 100644 index 0000000..d8dd02b --- /dev/null +++ b/rpmci/rpmci_binrpm_builder_main.py @@ -0,0 +1,129 @@ +#!/usr/bin/python + +# rpmci_binrpm_builder_main.py: +# Implementation of rpm-binrpm-builder +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys +import time +import shutil +import optparse +from ConfigParser import SafeConfigParser +import logging +import tempfile +import subprocess +import urllib + +import glib +import gobject +import gio + +from . import msgqueue +from . import subtask +from . import dynrepo +from . import artifact +from . import spec +from . import lame_vcs_abstraction + +class BinRPMBuilder(object): + def __init__(self, options, config): + self.config = config + + self._options = options + + srpm_msgqueue_dir = config.get('SRPM', 'msgqueue') + self._srpm_msgqueue = msgqueue.MessageQueue(srpm_msgqueue_dir) + + self._artifactset = artifact.ArtifactSet.from_config(config) + self._artifact_to_repo = {} + artifact_basedir = config.get('build', 'artifactdir') + for artifact in self._artifactset.artifacts: + artifact_repopath = os.path.join(artifact_basedir, artifact.name) + self._artifact_to_repo[artifact] = dynrepo.Repo(artifact_repopath) + + self._srcrepo = dynrepo.Repo(config.get('SRPM', 'spm_dir')) + + def start(self): + self._srpm_msgqueue.connect(self._on_srpm_message) + + def _target_for_srpm(self, srpm_name): + (name, ver, rest) = srpm_name.rsplit('-', 2) + for target in self._artifactset.get_build_targets(): + if target.name == name: + return target + return None + + def _on_srpm_message(self, q, messages): + msg_list = list(messages) + num_msgs = len(msg_list) + logging.debug("Starting processing of %d messages" % (num_msgs, )) + for msg in msg_list: + srpm_name = msg.payload['srpm'] + logging.debug("Processing update for SRPM %r" % (srpm_name, )) + target = self._target_for_srpm(srpm_name) + if target is None: + logging.warn("Skipping unknown srpm %r" % (srpm_name, )) + continue + affected_artifacts = [] + for artifact in self._artifactset.artifacts: + if target in artifact.targets: + affected_artifacts.append(artifact) + + self._handle_srpm_message_name(q, srpm_name) + q.consume(msg) + logging.debug("Processed %d messages successfully" % (num_msgs, )) + + def _handle_srpm_message_name(self, q, srpm_name): + exec_basedir = os.path.dirname(sys.argv[0]) + + subtask.spawn_sync('mock-many-srpms' % (target.module, ), + args, cwd=fedora_dir) + + args = ['rpmbuild'] + args.extend(self._get_base_rpmbuild_args(fedora_dir)) + args.extend(['-bs', specname]) + subtask.spawn_sync('rpmbuild-srpm-%s' % (target.module, ), + args, cwd=fedora_dir) + srpm_path = None + srpm_basename = None + for filename in os.listdir(fedora_dir): + if filename.endswith('.src.rpm'): + srpm_basename = filename + srpm_path = os.path.join(fedora_dir, filename) + break + assert srpm_path is not None + os.rename(srpm_path, os.path.join(self._srcdir, srpm_basename)) + self._srcrepo.update_repo_sync() + + self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'filename': srpm_basename})) + +def main(): + glib.threads_init() + + opts = optparse.OptionParser("usage: %prog [options]") + opts.add_option('-c', '--config', dest='config', help="Path to configuration file") + opts.add_option('', '--debug', action='store_true', help="Print verbose debugging") + + (options, args) = opts.parse_args() + + if options.config is None: + print "Must specify --config" + sys.exit(1) + + config = SafeConfigParser({'home': os.environ['HOME']}) + config.read(options.config) + level = logging.DEBUG if options.debug else logging.INFO + logging.basicConfig(stream=sys.stderr, level=level) + + subtask.global_configure(config) + + builder = SRPMBuilder(options, config) + builder.start() + + logging.info("Awaiting events") + loop = glib.MainLoop() + loop.run() diff --git a/rpmci/versioned_repos.py b/rpmci/versioned_repos.py new file mode 100644 index 0000000..6de753c --- /dev/null +++ b/rpmci/versioned_repos.py @@ -0,0 +1,99 @@ +#!/usr/bin/python + +# versioned_repos.py: +# Directory of "versioned" repositories, sharing hard-linked files +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys +import logging + +import glib +import gio +import rpm +import rpmUtils +import rpmUtils.miscutils + +from . import subtask + +class Repo(object): + def __init__(self, basedir, name, version): + self.name = name + self.version = version + self._dirname = '%d' % (version, ) + self._dir = os.path.join(basedir, self._dirname) + self._task_basename = 'repo-%s' % (name, ) + if not os.path.isdir(self._dir): + os.makedirs(self._dir) + + def get_dir(self): + return self._dir + + def add_rpm(self, rpm_path): + basename = os.path.basename(rpm_path) + os.link(rpm_path, os.path.join(self._dir, basename)) + + def iter_rpms(self): + for filename in os.listdir(self._dir): + if filename.endswith('.rpm'): + yield os.path.join(self._dir, filename) + + def _delete_old_rpms_in_dir(self, dirpath): + output = subtask.spawn_sync_get_output('repomanage-' + self._task_basename, + ['repomanage', '-o', '.'], cwd=dirpath) + for line in output.split('\n'): + if line.endswith('.rpm') and os.path.exists(line): + os.unlink(line) + + def update_sync(self): + self._delete_old_rpms_in_dir(self._dir) + subtask.spawn_sync('createrepo' + self._task_basename, + ['createrepo', '.'], cwd=self._dir) + +class VersionedRepo(object): + def __init__(self, dirpath): + self._dir = dirpath + if not os.path.isdir(self._dir): + os.makedirs(self._dir) + self._name = os.path.basename(dirpath) + self._versions = {} + self._last_version = -1 + self._reload() + + def get_versions(self): + return self._versions + + def _reload(self): + self._versions = {} + for filename in os.listdir(self._dir): + path = os.path.join(self,_dir, filename) + if not os.path.isdir(path): + continue + try: + revision = int(filename) + except ValueError, e: + continue + self._versions[revision] = Repo(self._dir, self._name, revision) + if revision > self._last_version: + self._last_version = revision + + def get_latest_path(self): + if self._last_version == -1: + return None + return self._versions[self._last_version].get_dir() + + def commit_sync(self, new_rpms): + rev = self._last_version + 1 + new_repo = Repo(self._dir, self._name, rev) + if rev > 0: + prev_repo = self._versions[self._last_version] + for rpm in prev_repo.iter_rpms(): + new_repo.add_rpm(rpm) + for rpm in new_rpms: + new_repo.add_rpm(rpm) + new_repo.update_sync() + return rev + -- cgit