diff options
Diffstat (limited to 'rpmci')
| -rw-r--r-- | rpmci/artifact.py | 24 | ||||
| -rw-r--r-- | rpmci/dynrepo.py | 13 | ||||
| -rw-r--r-- | rpmci/lame_vcs_abstraction.py | 4 | ||||
| -rw-r--r-- | rpmci/msgqueue.py | 5 | ||||
| -rw-r--r-- | rpmci/rpmci_inject_rebuild_main.py | 71 | ||||
| -rw-r--r-- | rpmci/rpmci_srpm_builder_main.py | 50 | ||||
| -rw-r--r-- | rpmci/rpmci_update_config_main.py | 8 | ||||
| -rw-r--r-- | rpmci/rpmci_vcs_mirror_main.py | 11 | ||||
| -rw-r--r-- | rpmci/spec.py | 2 |
9 files changed, 157 insertions, 31 deletions
diff --git a/rpmci/artifact.py b/rpmci/artifact.py index 1685074..13841ad 100644 --- a/rpmci/artifact.py +++ b/rpmci/artifact.py @@ -10,8 +10,23 @@ import os import sys import urllib +import ConfigParser -FEDORA_ANONGIT_URL = 'git://pkgs.fedoraproject.org' +from . import spec + +FEDORA_ANONGIT_URL = 'git:git://pkgs.fedoraproject.org' + +def get_default_architectures(): + # There's probably some RPM thingy to find this + uname = os.uname() + arch = uname[-1] + if arch == 'x86_64': + # On x86_64, we can build both + return ('i686', 'x86_64') + elif arch == 'i686': + # 32 bit can only build itself + return 'i686' + raise SystemExit("Unhandled architecture %r from os.uname(), known=[x86_64, i686]" % (arch, )) class BuildTarget(object): def __init__(self, module, os, architecture): @@ -63,7 +78,10 @@ class ArtifactSet(object): modules = config.get(section, 'artifact_%s_modules' % (config_name, )).split(' ') modules = map(str.strip, modules) os = config.get(section, 'artifact_%s_os' % (config_name, )) - architectures = config.get(section, 'artifact_%s_architectures' % (config_name, )).split(' ') + try: + architectures = config.get(section, 'artifact_%s_architectures' % (config_name, )).split(' ') + except ConfigParser.NoOptionError, e: + architectures = get_default_architectures() architectures = map(str.strip, architectures) for module in modules: @@ -82,7 +100,7 @@ def fedora_git_url_for_build_target(config, buildtarget): else: return '%s#%s/master' % (base, buildtarget.os) -def upstream_git_url_for_build_target(config, buildtarget): +def upstream_vcs_url_for_build_target(config, buildtarget): fedora_url = fedora_git_url_for_build_target(config, buildtarget) escaped_url = urllib.quote(fedora_url, '') mirror_dir = config.get('VCS', 'mirror_dir') diff --git a/rpmci/dynrepo.py b/rpmci/dynrepo.py index 505775d..8db1a8d 100644 --- a/rpmci/dynrepo.py +++ b/rpmci/dynrepo.py @@ -10,8 +10,10 @@ import os import sys import logging +import subprocess import glib +import gio import rpm import rpmUtils import rpmUtils.miscutils @@ -24,6 +26,7 @@ class Repo(object): self._dir_gfile = gio.File(path=dirpath) self._monitor = self._dir_gfile.monitor(gio.FILE_MONITOR_NONE) self._monitor.connect('changed', self._on_dir_changed) + self._rpms = {} def get_rpms(self): return self._rpms @@ -48,16 +51,16 @@ class Repo(object): os.link(srpm_path, os.path.join(self._dir, basename)) def _delete_old_rpms_in_dir(self, dirpath): - proc = popen_verbose(['repomanage', '-o', '.'], stdout=subprocess.PIPE, - stderr=sys.stderr, - cwd=dirpath) + proc = subprocess.Popen(['repomanage', '-o', '.'], stdout=subprocess.PIPE, + stderr=sys.stderr, + cwd=dirpath) output = proc.communicate()[0] for line in output.split('\n'): if line.endswith('.rpm') and os.path.exists(line): os.unlink(line) def update_repo_sync(self): - self._delete_old_rpms_in_dir(dirpath) + self._delete_old_rpms_in_dir(self._dir) subprocess.check_call(['createrepo', '.'], cwd=self._dir) def _headers_from_packages(self, rpmlist): @@ -80,7 +83,7 @@ class Repo(object): self._reload() def _reload(self): - dir_contents = os.listdir(self._dirpath) + dir_contents = os.listdir(self._dir) messages = set() rpmlist = set() for filename in dir_contents: diff --git a/rpmci/lame_vcs_abstraction.py b/rpmci/lame_vcs_abstraction.py index f603809..02c6149 100644 --- a/rpmci/lame_vcs_abstraction.py +++ b/rpmci/lame_vcs_abstraction.py @@ -42,7 +42,7 @@ class Vcs(object): def get_url_string(self): """Retrieve the stringified form of the repository URL.""" - return self._url_string + return '%s:%s' % (self.vcstype, self._url_string) def checkout_async(self, destdir): """Retrieve a new copy of the source tree, saving as destdir""" @@ -125,7 +125,7 @@ class GitVcs(Vcs): basename = os.path.basename(target_directory) if not basename.endswith('/'): basename += '/' - basedir = os.path.basename(target_directory) + basedir = os.path.dirname(target_directory) args = ['git', 'archive', '--format=tar', '--prefix=%s' % (basename,), commit_id] logging.info("Synchronously executing: %r" % (args, )) gitproc = subprocess.Popen(args, cwd=self._dir, stdout=subprocess.PIPE, stderr=log_output_stream) diff --git a/rpmci/msgqueue.py b/rpmci/msgqueue.py index 7148773..d1432a9 100644 --- a/rpmci/msgqueue.py +++ b/rpmci/msgqueue.py @@ -10,6 +10,7 @@ import sys import hashlib import json +import glib import gio class Message(object): @@ -58,6 +59,7 @@ class MessageQueue(object): def connect(self, callback): self._subscribers.append(callback) + glib.idle_add(self._reprocess_queue) def consume(self, message): self._consumed.append(message) @@ -72,6 +74,9 @@ class MessageQueue(object): os.rename(temp_filename, filename) def _on_dir_changed(self, mon, gfile, other, event): + self._reprocess_queue() + + def _reprocess_queue(self): dir_contents = os.listdir(self._dirpath) messages = set() for filename in dir_contents: diff --git a/rpmci/rpmci_inject_rebuild_main.py b/rpmci/rpmci_inject_rebuild_main.py new file mode 100644 index 0000000..eb1373f --- /dev/null +++ b/rpmci/rpmci_inject_rebuild_main.py @@ -0,0 +1,71 @@ +#!/usr/bin/python + +# rpmci_inject_rebuild_main.py: +# Implementation of rpmci-inject-rebuild +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters <walters@verbum.org> + +import os +import sys +import time +import shutil +import optparse +import logging +from ConfigParser import SafeConfigParser +import subprocess +import datetime + +import glib +import gobject +import gio + +from . import artifact +from . import msgqueue + +def inject_rebuild(options, config, module): + artifact_set = artifact.ArtifactSet.from_config(config) + target = None + for target_iter in artifact_set.get_build_targets(): + if target_iter.module == module: + target = target_iter + break + if target is None: + raise SystemExit("Couldn't find module %r in configuration" % (module, )) + upstream_url = artifact.upstream_vcs_url_for_build_target(config, target) + + vcs_msgqueue_dir = config.get('VCS', 'msgqueue') + vcs_msgqueue = msgqueue.MessageQueue(vcs_msgqueue_dir) + msg = msgqueue.Message(None, {'type': 'update'}, {'url': upstream_url}) + vcs_msgqueue.append(msg) + print "Wrote %s" % (msg.ident, ) + +def main(): + glib.threads_init() + + opts = optparse.OptionParser("usage: %prog [options] ACTION [args]") + opts.add_option('-c', '--config', dest='config', help="Path to configuration file") + opts.add_option('', '--debug', action='store_true', help="Print verbose debugging") + + (options, args) = opts.parse_args() + + if options.config is None: + print "Must specify --config" + sys.exit(1) + + config = SafeConfigParser({'home': os.environ['HOME']}) + config.read(options.config) + level = logging.DEBUG if options.debug else logging.INFO + logging.basicConfig(stream=sys.stderr, level=level) + + if len(args) < 1: + print "Must specify module to be rebuilt" + opts.print_usage() + sys.exit(1) + + module = args[0] + + inject_rebuild(options, config, module) + + sys.exit(0) diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py index 3b44168..1796d8b 100644 --- a/rpmci/rpmci_srpm_builder_main.py +++ b/rpmci/rpmci_srpm_builder_main.py @@ -14,6 +14,8 @@ import shutil import optparse from ConfigParser import SafeConfigParser import logging +import tempfile +import subprocess import urllib import glib @@ -23,6 +25,7 @@ import gio from . import msgqueue from . import dynrepo from . import artifact +from . import spec from . import lame_vcs_abstraction class SRPMBuilder(object): @@ -31,6 +34,8 @@ class SRPMBuilder(object): self._options = options + mirror_dir = config.get('VCS', 'mirror_dir') + vcs_msgqueue_dir = config.get('VCS', 'msgqueue') self._vcs_msgqueue = msgqueue.MessageQueue(vcs_msgqueue_dir) @@ -41,44 +46,54 @@ class SRPMBuilder(object): self._target_vcs = {} for target in self._artifactset.get_build_targets(): fedora_url = artifact.fedora_git_url_for_build_target(self.config, target) - upstream_url = artifact.upstream_git_url_for_build_target(self.config, target) - fedora_vcs = lame_vcs_abstraction.VCS.new_from_spec(fedora_url) - upstream_vcs = lame_vcs_abstraction.VCS.new_from_spec(upstream_url) + upstream_url = artifact.upstream_vcs_url_for_build_target(self.config, target) + fedora_vcs = lame_vcs_abstraction.Vcs.new_from_spec(fedora_url) + fedora_vcs.set_directory(os.path.join(mirror_dir, urllib.quote(fedora_vcs.get_url_string(), ''))) + upstream_vcs = lame_vcs_abstraction.Vcs.new_from_spec(upstream_url) + upstream_vcs.set_directory(os.path.join(mirror_dir, urllib.quote(upstream_vcs.get_url_string(), ''))) self._target_vcs[target] = (fedora_vcs, upstream_vcs) self._srcdir = config.get('SRPM', 'srpm_dir') + if not os.path.isdir(self._srcdir): + os.makedirs(self._srcdir) self._srcrepo = dynrepo.Repo(self._srcdir) def start(self): self._vcs_msgqueue.connect(self._on_vcs_message) def _get_base_rpmbuild_args(self, src_dirpath): - return list(["--define '_sourcedir %s'" % src_dirpath, - "--define '_specdir %s'" % src_dirpath, - "--define '_builddir %s'" % src_dirpath, - "--define '_srcrpmdir %s'" % src_dirpath, - "--define '_rpmdir %s'" % src_dirpath]) + return ['--define', '_sourcedir ' + src_dirpath, + '--define', '_specdir ' + src_dirpath, + '--define', '_builddir ' + src_dirpath, + '--define', '_srcrpmdir ' + src_dirpath, + '--define', '_rpmdir ' + src_dirpath] def _on_vcs_message(self, q, messages): + num_msgs = len(messages) + logging.debug("Starting processing of %d messages" % (num_msgs, )) for msg in messages: - url = msg.payload.url + url = msg.payload['url'] + logging.debug("Processing update for VCS url %r" % (url, )) self._handle_vcs_message_url(q, url) + q.consume(msg) + logging.debug("Processed %d messages successfully" % (num_msgs, )) def _handle_vcs_message_url(self, q, url): - q.consume(msg) is_fedora = False target = None + valid_urls = [] for iter_target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems(): fedora_url = fedora_vcs.get_url_string() is_fedora = (url == fedora_url) if not is_fedora: upstream_url = upstream_vcs.get_url_string() if not url == upstream_url: + valid_urls.append(upstream_url) continue target = iter_target break - assert target is not None, ("Couldn't find target for url %r" % (url, )) - (fedora_vcs, upstream_vcs) = self._target[target] + assert target is not None, ("Couldn't find target for url %r; valid=%r" % (url, valid_urls)) + (fedora_vcs, upstream_vcs) = self._target_vcs[target] work_dir = tempfile.mkdtemp('.tmp', 'srpm-builder') logging.debug("Creating new SRPM for %r in %r" % (target.module, work_dir)) @@ -104,14 +119,17 @@ class SRPMBuilder(object): target_spec_obj.save() exec_basedir = os.path.dirname(sys.argv[0]) - subprocess.check_call([os.path.join(exec_basedir, 'rpmci-spec-vcs'), - '--vcsdir=' + upstream_vcs.get_directory(), - 'set-revision', 'HEAD'], + args = [os.path.abspath(os.path.join(exec_basedir, 'rpmci-spec-vcs')), + '--vcsdir=' + upstream_vcs.get_directory(), + 'set-revision', 'HEAD'] + print "Synchronously executing: %r" % (args, ) + subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr, cwd=fedora_dir) args = ['rpmbuild'] args.extend(self._get_base_rpmbuild_args(fedora_dir)) args.extend(['-bs', specname]) + print "Synchronously executing: %r" % (args, ) subprocess.check_call(args, cwd=fedora_dir, stdout=sys.stdout, stderr=sys.stderr) srpm_path = None @@ -125,7 +143,7 @@ class SRPMBuilder(object): os.rename(srpm_path, os.path.join(self._srcdir, srpm_basename)) self._srcrepo.update_repo_sync() - self._srpm_msgqueue.append(msgqueue.Message({'type': 'srpm'}, {'filename': srpm_basename})) + self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'filename': srpm_basename})) def main(): glib.threads_init() diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py index 8e36d36..dac658e 100644 --- a/rpmci/rpmci_update_config_main.py +++ b/rpmci/rpmci_update_config_main.py @@ -45,8 +45,11 @@ def _run_vcsmirror(options, config): def update_config(options, config): mirror_dir = config.get('VCS', 'mirror_dir') + + if not os.path.isdir(mirror_dir): + os.makedirs(mirror_dir) - artifact_set = artifact.ArtifactSet.from_config(config, 'releases') + artifact_set = artifact.ArtifactSet.from_config(config) fedora_git_urls = set() @@ -60,7 +63,8 @@ def update_config(options, config): _run_vcsmirror(options, config) all_urls = set(fedora_git_urls) - for url in fedora_git_urls: + for target in unique_buildtargets: + upstream_vcs_url = artifact.upstream_vcs_url_for_build_target(target) all_urls.add(upstream_vcs_url) _write_vcs_urls(options, config, all_urls) diff --git a/rpmci/rpmci_vcs_mirror_main.py b/rpmci/rpmci_vcs_mirror_main.py index f8a78ee..51c706b 100644 --- a/rpmci/rpmci_vcs_mirror_main.py +++ b/rpmci/rpmci_vcs_mirror_main.py @@ -67,8 +67,14 @@ class VCSMirror(object): return os.path.join(self._dir, self._escape_vcs_url(vcs)) def start(self): - logging.info("Starting poll of %d repositories on %d unique hosts" - % (len(self._vcslist), len(list(self._vcs_by_host.iterkeys())))) + num_vcs = len(self._vcslist) + num_hosts = len(list(self._vcs_by_host.iterkeys())) + if self._options.clone_then_exit: + logging.info("Doing clone of %d repositories on %d unique hosts, then will terminate" + % (num_vcs, num_hosts)) + else: + logging.info("Starting poll of %d repositories on %d unique hosts" + % (num_vcs, num_hosts)) # Schedule jobs for now for vcs in self._vcslist: @@ -185,6 +191,7 @@ class VCSMirror(object): shutil.rmtree(vcs_tempdir) process = vcs.checkout_async(vcs_tempdir, job_logpath, self._on_job_exited) elif self._options.clone_then_exit: + logging.info("Already have clone directory %s" % (vcsdir, )) processed.append(vcs) continue else: diff --git a/rpmci/spec.py b/rpmci/spec.py index 6d3bba2..14a94c0 100644 --- a/rpmci/spec.py +++ b/rpmci/spec.py @@ -210,7 +210,7 @@ the 42 with new_value, preserving the comment # foo.""" elif ':' in line: key, value = line.split(':', 1) if (line.startswith('Source0:') or line.startswith('Source:')) and self._source_archivename: - output.write("""# Snapshot created from upstream version control by rpmci-spec-vcs.\n""" + output.write("""# Snapshot created from upstream version control by rpmci-spec-vcs.\n""") output.write(self._replace_key_line(key, self._source_archivename, line)) elif key == 'BuildRequires' and not wrote_buildrequires: output.write(line) |