From aa471079451dc5f54c36ff341830e66bc36f1a97 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 3 Oct 2010 16:56:42 -0400 Subject: Add rpmci-update-config, other bits --- .gitignore | 1 + DESIGN | 3 +- rpmci-update-config | 20 ++++++++ rpmci/artifact.py | 78 +++++++++++++++++++++++++++++ rpmci/lame_vcs_abstraction.py | 7 +++ rpmci/rpmci_update_config_main.py | 100 ++++++++++++++++++++++++++++++++++++++ rpmci/rpmci_vcs_mirror_main.py | 38 ++++++++++++--- rpmci/spec.py | 2 +- sample.config | 26 ++++++++++ 9 files changed, 264 insertions(+), 11 deletions(-) create mode 100755 rpmci-update-config create mode 100644 rpmci/artifact.py create mode 100644 rpmci/rpmci_update_config_main.py diff --git a/.gitignore b/.gitignore index 0d20b64..f3d74a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.pyc +*~ diff --git a/DESIGN b/DESIGN index a6bebfa..06bbf3f 100644 --- a/DESIGN +++ b/DESIGN @@ -1,4 +1,4 @@ -rpmci-generate-config: +rpmci-update-config: takes master build configuration, generates config files for individual components rpmci-vcs-mirror: @@ -12,4 +12,3 @@ rpmci-srpm-builder: rpmci-builder: takes updated srpms calls mock-many-srpms for configured releases - diff --git a/rpmci-update-config b/rpmci-update-config new file mode 100755 index 0000000..fd53e9d --- /dev/null +++ b/rpmci-update-config @@ -0,0 +1,20 @@ +#!/usr/bin/python + +# rpmci-vcs-mirror: +# Poll set of VCS URLs, caching local repositories +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys + +if os.path.isdir('.git'): + sys.path.insert(0, os.getcwd()) + +import rpmci +from rpmci.rpmci_update_config_main import main + +if __name__ == '__main__': + main() diff --git a/rpmci/artifact.py b/rpmci/artifact.py new file mode 100644 index 0000000..6a4a5cb --- /dev/null +++ b/rpmci/artifact.py @@ -0,0 +1,78 @@ +#!/usr/bin/python + +# artifact.py: +# Set of RPMs built into a repository. +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +FEDORA_ANONGIT_URL = 'git://pkgs.fedoraproject.org' + +class BuildTarget(object): + def __init__(self, module, os, architecture): + self.module = module + self.os = os + self.architecture = architecture + + def __cmp__(self, other): + i = cmp(self.module, other.module) + if i != 0: + return i + i = cmp(self.os, other.os) + if i != 0: + return i + i = cmp(self.architecture, other.architecture) + if i != 0: + return i + return 0 + +class Artifact(object): + def __init__(self, targets): + self.targets = targets + +class ArtifactSet(object): + def __init__(self, artifacts): + self.artifacts = artifacts + + def get_build_targets(self): + """Returns unordered set of build targets.""" + + targets = set() + for artifact in self.artifacts: + for target in artifact.targets: + targets.add(target) + return targets + + @classmethod + def from_config(cls, config, section): + artifacts_str = config.get('releases', 'artifacts') + artifact_list = map(str.strip, artifacts_str.split(' ')) + artifacts = [] + + for artifact_name in artifact_list: + artifact_build_targets = [] + + config_name = artifact_name.replace('-', '_') + + modules = config.get(section, 'artifact_%s_modules' % (config_name, )).split(' ') + modules = map(str.strip, modules) + os = config.get(section, 'artifact_%s_os' % (config_name, )) + architectures = config.get(section, 'artifact_%s_architectures' % (config_name, )).split(' ') + architectures = map(str.strip, architectures) + + for module in modules: + for architecture in architectures: + target = BuildTarget(module, os, architecture) + artifact_build_targets.append(target) + + artifacts.append(Artifact(artifact_build_targets)) + return cls(artifacts) + +def fedora_git_url_for_build_target(config, buildtarget): + fedora_os_master = config.get('fedora', 'master') + base = '%s/%s.git' % (FEDORA_ANONGIT_URL, buildtarget.module) + if buildtarget.os == fedora_os_master: + return base + else: + return '%s#%s/master' % (base, buildtarget.os) diff --git a/rpmci/lame_vcs_abstraction.py b/rpmci/lame_vcs_abstraction.py index a96f23d..2d5314e 100644 --- a/rpmci/lame_vcs_abstraction.py +++ b/rpmci/lame_vcs_abstraction.py @@ -22,6 +22,7 @@ from . import async_subprocess class Vcs(object): def __init__(self, parsedurl, directory=None): self._parsed_url = parsedurl + self._url_string = urlparse.urlunsplit(parsedurl) # Deliberately drop params/query self._nonfragment_url_string = urlparse.urlunparse((parsedurl.scheme, parsedurl.netloc, @@ -32,11 +33,17 @@ class Vcs(object): self._dir = directory def get_url(self): + """Retrieve the SplitResult (broken up components tuple) URL for this repository.""" return self._parsed_url def get_base_url_string(self): + """Retrieve the non-branched URL for this repository.""" return self._nonfragment_url_string + def get_url_string(self): + """Retrieve the stringified form of the repository URL.""" + return self._url_string + def checkout_async(self, destdir): """Retrieve a new copy of the source tree, saving as destdir""" raise Exception("not implemented") diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py new file mode 100644 index 0000000..98355c1 --- /dev/null +++ b/rpmci/rpmci_update_config_main.py @@ -0,0 +1,100 @@ +#!/usr/bin/python + +# rpmci_update_config_main.py: +# Implementation of rpmci-update-config +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +import os +import sys +import time +import shutil +import optparse +from ConfigParser import SafeConfigParser +import logging +import urllib +import urlparse +import subprocess + +import glib +import gobject +import gio + +from . import msgqueue +from . import artifact +from . import spec + +def _write_vcs_urls(options, config, urls): + mirror_dir = config.get('VCS', 'mirror_dir') + f = open(os.path.join(mirror_dir, 'vcs.txt'), 'w') + for url in urls: + f.write(url) + f.write('\n') + f.close() + +def _run_vcsmirror(options, config): + exec_basedir = os.path.dirname(sys.argv[0]) + + args = [os.path.join(exec_basedir, 'rpmci-vcs-mirror'), + '--config', options.config, + '--clone-then-exit'] + print "Running: %r" % (args, ) + subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr) + +def update_config(options, config): + mirror_dir = config.get('VCS', 'mirror_dir') + + artifact_set = artifact.ArtifactSet.from_config(config, 'releases') + + fedora_git_urls = set() + + unique_buildtargets = artifact_set.get_build_targets() + + for target in unique_buildtargets: + url = artifact.fedora_git_url_for_build_target(config, target) + fedora_git_urls.add(url) + + _write_vcs_urls(options, config, fedora_git_urls) + _run_vcsmirror(options, config) + + all_urls = set(fedora_git_urls) + for url in fedora_git_urls: + escaped_url = urllib.quote(url, '') + vcsdir = os.path.join(mirror_dir, escaped_url) + if not os.path.isdir(vcsdir): + raise SystemExit("Not a directory: %r" % (vcsdir, )) + specpath = None + for filename in os.listdir(vcsdir): + if filename.endswith('.spec'): + specpath = os.path.join(vcsdir, filename) + break + assert specpath is not None + spec_obj = spec.Spec(specpath) + upstream_vcs_url = spec_obj.get_vcs() + all_urls.add(upstream_vcs_url) + + _write_vcs_urls(options, config, all_urls) + _run_vcsmirror(options, config) + +def main(): + glib.threads_init() + + opts = optparse.OptionParser("usage: %prog [options]") + opts.add_option('-c', '--config', dest='config', help="Path to configuration file") + opts.add_option('', '--debug', action='store_true', help="Print verbose debugging") + + (options, args) = opts.parse_args() + + if options.config is None: + print "Must specify --config" + sys.exit(1) + + config = SafeConfigParser({'home': os.environ['HOME']}) + config.read(options.config) + level = logging.DEBUG if options.debug else logging.INFO + logging.basicConfig(stream=sys.stderr, level=level) + + update_config(options, config) + sys.exit(0) diff --git a/rpmci/rpmci_vcs_mirror_main.py b/rpmci/rpmci_vcs_mirror_main.py index 85615a3..f8a78ee 100644 --- a/rpmci/rpmci_vcs_mirror_main.py +++ b/rpmci/rpmci_vcs_mirror_main.py @@ -24,10 +24,12 @@ from . import msgqueue from . import lame_vcs_abstraction class VCSMirror(object): - def __init__(self, config, urls): + def __init__(self, options, config, urls): self.config = config self.urls = urls + self._options = options + self._dir = config.get('VCS', 'mirror_dir') self._process_dir = config.get('VCS', 'process_logdir') if not os.path.isdir(self._process_dir): @@ -59,7 +61,7 @@ class VCSMirror(object): return vcs.get_url().netloc def _escape_vcs_url(self, vcs): - return urllib.quote(vcs.get_base_url_string(), '') + return urllib.quote(vcs.get_url_string(), '') def _cachedir_for_vcs(self, vcs): return os.path.join(self._dir, self._escape_vcs_url(vcs)) @@ -105,13 +107,14 @@ class VCSMirror(object): current_id = vcs.get_id() if current_id != previous_id: logging.info("vcs %r: New commit id %r differs from previous %r" % (vcs, current_id, previous_id)) - msg = msgqueue.Message(None, {'type': 'update'}, {'id': current_id}) + msg = msgqueue.Message(None, {'type': 'update'}, {'url': vcs.get_url_string(), 'id': current_id}) self._msgqueue.append(msg) else: logging.info("No changes in %r from previous commit id %r" % (vcs, previous_id)) - target_time = int(time.time() + self._timeout_seconds) - self._vcs_queue.append((vcs, target_time)) + if not failed and not self._options.clone_then_exit: + target_time = int(time.time() + self._timeout_seconds) + self._vcs_queue.append((vcs, target_time)) self._poll() @@ -144,13 +147,14 @@ class VCSMirror(object): glib.source_remove(self._active_queue_timeout_id) self._active_queue_timeout_seconds = timeout self._active_queue_timeout_id = glib.timeout_add_seconds(timeout, self._poll) - def _poll(self): current_time = int(time.time()) - logging.info("Doing poll (%d active tasks)" % (self._num_active_jobs(), )) + orig_active_jobs = self._num_active_jobs() + logging.debug("Queue: %r" % (self._vcs_queue, )) + processed = [] for vcs, target_time in self._vcs_queue: active = self._job_for_vcs(vcs) @@ -177,7 +181,12 @@ class VCSMirror(object): previous_id = None logging.info("Doing initial checkout for %r" % (vcs.get_base_url_string(), )) vcs_tempdir = vcsdir + '.tmp' + if os.path.isdir(vcs_tempdir): + shutil.rmtree(vcs_tempdir) process = vcs.checkout_async(vcs_tempdir, job_logpath, self._on_job_exited) + elif self._options.clone_then_exit: + processed.append(vcs) + continue else: vcs.set_directory(vcsdir) previous_id = vcs.get_id() @@ -186,6 +195,8 @@ class VCSMirror(object): process.__vcs = vcs self._jobs_by_host[host].append((process, previous_id)) processed.append(vcs) + + added_job_count = len(processed) while processed: vcs = processed[0] del processed[0] @@ -196,8 +207,18 @@ class VCSMirror(object): break assert index >= 0 del self._vcs_queue[index] + + new_active_jobs = self._num_active_jobs() + + if len(self._vcs_queue) == 0 and new_active_jobs == 0: + logging.info("Queue is empty and no active jobs. Exiting.") + sys.exit(0) self._adjust_timeout() + if new_active_jobs == 0: + logging.info("No active jobs; sleeping for %d seconds" % (self._active_queue_timeout_seconds, )) + else: + logging.info("Poll complete, started %d jobs (%d total)" % (new_active_jobs - orig_active_jobs, new_active_jobs)) return False @@ -208,6 +229,7 @@ def main(): opts = optparse.OptionParser("usage: %prog [options]") opts.add_option('-c', '--config', dest='config', help="Path to configuration file") opts.add_option('', '--debug', action='store_true', help="Print verbose debugging") + opts.add_option('', '--clone-then-exit', action='store_true', help="If true, perform any necessary clones, then exit") (options, args) = opts.parse_args() @@ -232,7 +254,7 @@ def main(): urls = f.readlines() f.close() - mirror = VCSMirror(config, urls) + mirror = VCSMirror(options, config, urls) mirror.start() loop = glib.MainLoop() diff --git a/rpmci/spec.py b/rpmci/spec.py index 40ef6af..aa804d8 100644 --- a/rpmci/spec.py +++ b/rpmci/spec.py @@ -242,7 +242,7 @@ the 42 with new_value, preserving the comment # foo.""" for line in self._lines: if line.startswith('#VCS:'): return line[5:].strip() - raise ValueError("No such key #VCS in file %r" % (self._filename, )) + return self.get_key('VCS') def get_key(self, key): key = key + ':' diff --git a/sample.config b/sample.config index afeeda0..8c87f3a 100644 --- a/sample.config +++ b/sample.config @@ -1,3 +1,12 @@ +[fedora] +master=f15 + +[releases] +artifacts=gnome-3-f15 +artifact_gnome_3_f15_modules=glib2 pixman cairo gobject-introspection libgee vala dconf pango atk gdk-pixbuf2 gtk2 gtk3 json-glib clutter mutter gjs gnome-shell +artifact_gnome_3_f15_os=f15 +artifact_gnome_3_f15_architectures=i386 x86_64 + [VCS] # Standard stuff basedir=%(home)s/rpmci @@ -12,3 +21,20 @@ poll_seconds=60 max_host_concurrency=4 # Hard limit on maximum concurrent operations max_concurrency=16 + +[SRPM] +basedir=%(home)s/rpmci +msg_basedir=%(basedir)s/msgqueue +process_logdir=%(basedir)s/process-logs + +msgqueue=%(msg_basedir)s/srpm-queue +fedpkg_dir=%(basedir)/fedpkg +srpm_dir=%(basedir)/srpms + +[build] +basedir=%(home)s/rpmci +msg_basedir=%(basedir)s/msgqueue +process_logdir=%(basedir)s/process-logs + +msgqueue=%(msg_basedir)s/build +artifactdir=%(basedir)/repos -- cgit