From 1c907887574b423485b2df0e130673dc09c6aca5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 5 Oct 2010 09:13:32 -0400 Subject: Some hacking on srpm builder --- rpmci/artifact.py | 23 +++++++-- rpmci/dynrepo.py | 104 ++++++++++++++++++++++++++++++++++++++ rpmci/rpmci_srpm_builder_main.py | 80 +++++++++++++++++++++++++++++ rpmci/rpmci_update_config_main.py | 12 ----- 4 files changed, 204 insertions(+), 15 deletions(-) create mode 100644 rpmci/dynrepo.py create mode 100644 rpmci/rpmci_srpm_builder_main.py (limited to 'rpmci') diff --git a/rpmci/artifact.py b/rpmci/artifact.py index 6a4a5cb..4be1e76 100644 --- a/rpmci/artifact.py +++ b/rpmci/artifact.py @@ -45,8 +45,9 @@ class ArtifactSet(object): return targets @classmethod - def from_config(cls, config, section): - artifacts_str = config.get('releases', 'artifacts') + def from_config(cls, config): + section = 'releases' + artifacts_str = config.get(section, 'artifacts') artifact_list = map(str.strip, artifacts_str.split(' ')) artifacts = [] @@ -68,7 +69,7 @@ class ArtifactSet(object): artifacts.append(Artifact(artifact_build_targets)) return cls(artifacts) - + def fedora_git_url_for_build_target(config, buildtarget): fedora_os_master = config.get('fedora', 'master') base = '%s/%s.git' % (FEDORA_ANONGIT_URL, buildtarget.module) @@ -76,3 +77,19 @@ def fedora_git_url_for_build_target(config, buildtarget): return base else: return '%s#%s/master' % (base, buildtarget.os) + +def upstream_git_url_for_build_target(config, buildtarget): + fedora_url = fedora_git_url_for_build_target(config, buildtarget) + escaped_url = urllib.quote(url, '') + vcsdir = os.path.join(mirror_dir, escaped_url) + if not os.path.isdir(vcsdir): + raise SystemExit("Not a directory: %r" % (vcsdir, )) + specpath = None + for filename in os.listdir(vcsdir): + if filename.endswith('.spec'): + specpath = os.path.join(vcsdir, filename) + break + assert specpath is not None + spec_obj = spec.Spec(specpath) + return spec_obj.get_vcs() + diff --git a/rpmci/dynrepo.py b/rpmci/dynrepo.py new file mode 100644 index 0000000..e6cdc90 --- /dev/null +++ b/rpmci/dynrepo.py @@ -0,0 +1,104 @@ +#!/usr/bin/python + +# dynrepo.py: +# Monitor set of RPMs in a directory. +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys +import logging + +import glib +import rpm +import rpmUtils +import rpmUtils.miscutils + +from . import async_subprocess + +class Repo(object): + def __init__(self, dirpath): + self._dir = dirpath + self._dir_gfile = gio.File(path=dirpath) + self._monitor = self._dir_gfile.monitor(gio.FILE_MONITOR_NONE) + self._monitor.connect('changed', self._on_dir_changed) + + def get_rpms(self): + return self._rpms + + def get_rpms_for_name(self, name, arch): + result = [] + for filename,(n,a,e,v,r) in self._rpms.iteritems(): + if n == name and (arch is None or a == arch): + result.append((filename, n, a, e, v, r)) + return result + + def get_latest_srpm_for_name(self, name): + rpms = self.get_rpms_for_name(name, 'src') + cmpevr=rpmUtils.miscutils.compareEVR + if rpms: + rpms.sort(lambda a,b: cmpevr(a[3:6], b[3:6])) + return rpms[-1] + return None + + def add_srpm(self, srpm_path): + basename = os.path.basename(srpm_path) + os.link(srpm_path, os.path.join(self._dir, basename)) + + def _delete_old_rpms_in_dir(self, dirpath): + proc = popen_verbose(['repomanage', '-o', '.'], stdout=subprocess.PIPE, + stderr=sys.stderr, + cwd=dirpath) + output = proc.communicate()[0] + for line in output.split('\n'): + if line.endswith('.rpm') and os.path.exists(line): + os.unlink(line) + + def update_repo_sync(self): + self._delete_old_rpms_in_dir(dirpath) + subprocess.check_call(['createrepo', '.'], cwd=self._dir) + + def _headers_from_packages(self, rpmlist): + result = {} + ts = rpm.TransactionSet() + ts.setVSFlags(~(rpm._RPMVSF_NOPAYLOAD)) + for pkg in rpmlist: + pkg_path = os.path.join(self._dir, pkg) + try: + header = rpmUtils.miscutils.hdrFromPackage(ts, pkg_path) + except rpmUtils.RpmUtilsError, e: + logging.exception(e) + continue + (n,a,e,v,r) = rpmUtils.miscutils.pkgTupleFromHeader(header) + del header + result[pkg] = (n,a,e,v,r) + return result + + def _on_dir_changed(self, mon, gfile, other, event): + self._reload() + + def _reload(self): + dir_contents = os.listdir(self._dirpath) + messages = set() + rpmlist = set() + for filename in dir_contents: + if filename.endswith('.tmp'): + continue + if not filename.endswith('.rpm'): + continue + rpmlist.add(filename) + + deleted = [] + for rpm in self._rpms: + if rpm not in rpmlist: + deleted.add(rpm) + for rpm in deleted: + del self._rpms[rpm] + new = [] + for rpm in self._rpms: + if rpm not in rpmlist: + new.append(rpm) + for rpm,data in self._headers_from_packages(new).iteritems(): + self._rpms[rpm] = data diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py new file mode 100644 index 0000000..2bfbef4 --- /dev/null +++ b/rpmci/rpmci_srpm_builder_main.py @@ -0,0 +1,80 @@ +#!/usr/bin/python + +# rpmci_srpm_builder_main.py: +# Implementation of rpm-vcs-mirror +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys +import time +import shutil +import optparse +from ConfigParser import SafeConfigParser +import logging +import urllib + +import glib +import gobject +import gio + +from . import msgqueue +from . import dynrepo +from . import lame_vcs_abstraction + +class SRPMBuilder(object): + def __init__(self, options, config): + self.config = config + self.urls = urls + + self._options = options + + vcs_msgqueue_dir = config.get('VCS', 'msgqueue') + self._vcs_msgqueue = msgqueue.MessageQueue(vcs_msgqueue_dir) + + self._artifactset = artifact.ArtifactSet.from_config(config) + self._vcs_by_url = {} + for target in self._artifactset.get_build_targets(): + fedora_url = artifact.fedora_git_url_for_build_target(target) + upstream_url = artifact.upstream_git_url_for_build_target(target) + self._vcs_by_url[fedora_url] = lame_vcs_abstraction.VCS.new_from_spec(fedora_url) + self._vcs_by_url[upstream_url] = lame_vcs_abstraction.VCS.new_from_spec(upstream_url) + + self._srcrepo = dynrepo.Repo(config.get('SRPM', 'srpm_dir')) + + def start(self): + self._vcs_msgqueue.connect(self._on_vcs_message) + + def _on_vcs_message(self, q, messages): + for msg in messages: + url = msg.payload.url + q.consume(msg) + vcs = self._vcs_by_url[url] + + + +def main(): + glib.threads_init() + + opts = optparse.OptionParser("usage: %prog [options]") + opts.add_option('-c', '--config', dest='config', help="Path to configuration file") + opts.add_option('', '--debug', action='store_true', help="Print verbose debugging") + + (options, args) = opts.parse_args() + + if options.config is None: + print "Must specify --config" + sys.exit(1) + + config = SafeConfigParser({'home': os.environ['HOME']}) + config.read(options.config) + level = logging.DEBUG if options.debug else logging.INFO + logging.basicConfig(stream=sys.stderr, level=level) + + builder = SRPMBuilder(options, config, urls) + builder.start() + + loop = glib.MainLoop() + loop.run() diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py index 98355c1..8e36d36 100644 --- a/rpmci/rpmci_update_config_main.py +++ b/rpmci/rpmci_update_config_main.py @@ -61,18 +61,6 @@ def update_config(options, config): all_urls = set(fedora_git_urls) for url in fedora_git_urls: - escaped_url = urllib.quote(url, '') - vcsdir = os.path.join(mirror_dir, escaped_url) - if not os.path.isdir(vcsdir): - raise SystemExit("Not a directory: %r" % (vcsdir, )) - specpath = None - for filename in os.listdir(vcsdir): - if filename.endswith('.spec'): - specpath = os.path.join(vcsdir, filename) - break - assert specpath is not None - spec_obj = spec.Spec(specpath) - upstream_vcs_url = spec_obj.get_vcs() all_urls.add(upstream_vcs_url) _write_vcs_urls(options, config, all_urls) -- cgit