summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorColin Walters <walters@verbum.org>2010-10-03 16:56:42 -0400
committerColin Walters <walters@verbum.org>2010-10-03 16:56:42 -0400
commitaa471079451dc5f54c36ff341830e66bc36f1a97 (patch)
treec76e34538304c65ee66064f3dfe6ac02e329ed2b
parentdb8df4a040883b631ff3c719c453246df9085776 (diff)
downloadrpmci-aa471079451dc5f54c36ff341830e66bc36f1a97.tar.gz
rpmci-aa471079451dc5f54c36ff341830e66bc36f1a97.tar.xz
rpmci-aa471079451dc5f54c36ff341830e66bc36f1a97.zip
Add rpmci-update-config, other bits
-rw-r--r--.gitignore1
-rw-r--r--DESIGN3
-rwxr-xr-xrpmci-update-config20
-rw-r--r--rpmci/artifact.py78
-rw-r--r--rpmci/lame_vcs_abstraction.py7
-rw-r--r--rpmci/rpmci_update_config_main.py100
-rw-r--r--rpmci/rpmci_vcs_mirror_main.py38
-rw-r--r--rpmci/spec.py2
-rw-r--r--sample.config26
9 files changed, 264 insertions, 11 deletions
diff --git a/.gitignore b/.gitignore
index 0d20b64..f3d74a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
*.pyc
+*~
diff --git a/DESIGN b/DESIGN
index a6bebfa..06bbf3f 100644
--- a/DESIGN
+++ b/DESIGN
@@ -1,4 +1,4 @@
-rpmci-generate-config:
+rpmci-update-config:
takes master build configuration, generates config files for individual components
rpmci-vcs-mirror:
@@ -12,4 +12,3 @@ rpmci-srpm-builder:
rpmci-builder:
takes updated srpms
calls mock-many-srpms for configured releases
-
diff --git a/rpmci-update-config b/rpmci-update-config
new file mode 100755
index 0000000..fd53e9d
--- /dev/null
+++ b/rpmci-update-config
@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+# rpmci-vcs-mirror:
+# Poll set of VCS URLs, caching local repositories
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+
+if os.path.isdir('.git'):
+ sys.path.insert(0, os.getcwd())
+
+import rpmci
+from rpmci.rpmci_update_config_main import main
+
+if __name__ == '__main__':
+ main()
diff --git a/rpmci/artifact.py b/rpmci/artifact.py
new file mode 100644
index 0000000..6a4a5cb
--- /dev/null
+++ b/rpmci/artifact.py
@@ -0,0 +1,78 @@
+#!/usr/bin/python
+
+# artifact.py:
+# Set of RPMs built into a repository.
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+FEDORA_ANONGIT_URL = 'git://pkgs.fedoraproject.org'
+
+class BuildTarget(object):
+ def __init__(self, module, os, architecture):
+ self.module = module
+ self.os = os
+ self.architecture = architecture
+
+ def __cmp__(self, other):
+ i = cmp(self.module, other.module)
+ if i != 0:
+ return i
+ i = cmp(self.os, other.os)
+ if i != 0:
+ return i
+ i = cmp(self.architecture, other.architecture)
+ if i != 0:
+ return i
+ return 0
+
+class Artifact(object):
+ def __init__(self, targets):
+ self.targets = targets
+
+class ArtifactSet(object):
+ def __init__(self, artifacts):
+ self.artifacts = artifacts
+
+ def get_build_targets(self):
+ """Returns unordered set of build targets."""
+
+ targets = set()
+ for artifact in self.artifacts:
+ for target in artifact.targets:
+ targets.add(target)
+ return targets
+
+ @classmethod
+ def from_config(cls, config, section):
+ artifacts_str = config.get('releases', 'artifacts')
+ artifact_list = map(str.strip, artifacts_str.split(' '))
+ artifacts = []
+
+ for artifact_name in artifact_list:
+ artifact_build_targets = []
+
+ config_name = artifact_name.replace('-', '_')
+
+ modules = config.get(section, 'artifact_%s_modules' % (config_name, )).split(' ')
+ modules = map(str.strip, modules)
+ os = config.get(section, 'artifact_%s_os' % (config_name, ))
+ architectures = config.get(section, 'artifact_%s_architectures' % (config_name, )).split(' ')
+ architectures = map(str.strip, architectures)
+
+ for module in modules:
+ for architecture in architectures:
+ target = BuildTarget(module, os, architecture)
+ artifact_build_targets.append(target)
+
+ artifacts.append(Artifact(artifact_build_targets))
+ return cls(artifacts)
+
+def fedora_git_url_for_build_target(config, buildtarget):
+ fedora_os_master = config.get('fedora', 'master')
+ base = '%s/%s.git' % (FEDORA_ANONGIT_URL, buildtarget.module)
+ if buildtarget.os == fedora_os_master:
+ return base
+ else:
+ return '%s#%s/master' % (base, buildtarget.os)
diff --git a/rpmci/lame_vcs_abstraction.py b/rpmci/lame_vcs_abstraction.py
index a96f23d..2d5314e 100644
--- a/rpmci/lame_vcs_abstraction.py
+++ b/rpmci/lame_vcs_abstraction.py
@@ -22,6 +22,7 @@ from . import async_subprocess
class Vcs(object):
def __init__(self, parsedurl, directory=None):
self._parsed_url = parsedurl
+ self._url_string = urlparse.urlunsplit(parsedurl)
# Deliberately drop params/query
self._nonfragment_url_string = urlparse.urlunparse((parsedurl.scheme,
parsedurl.netloc,
@@ -32,11 +33,17 @@ class Vcs(object):
self._dir = directory
def get_url(self):
+ """Retrieve the SplitResult (broken up components tuple) URL for this repository."""
return self._parsed_url
def get_base_url_string(self):
+ """Retrieve the non-branched URL for this repository."""
return self._nonfragment_url_string
+ def get_url_string(self):
+ """Retrieve the stringified form of the repository URL."""
+ return self._url_string
+
def checkout_async(self, destdir):
"""Retrieve a new copy of the source tree, saving as destdir"""
raise Exception("not implemented")
diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py
new file mode 100644
index 0000000..98355c1
--- /dev/null
+++ b/rpmci/rpmci_update_config_main.py
@@ -0,0 +1,100 @@
+#!/usr/bin/python
+
+# rpmci_update_config_main.py:
+# Implementation of rpmci-update-config
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+import time
+import shutil
+import optparse
+from ConfigParser import SafeConfigParser
+import logging
+import urllib
+import urlparse
+import subprocess
+
+import glib
+import gobject
+import gio
+
+from . import msgqueue
+from . import artifact
+from . import spec
+
+def _write_vcs_urls(options, config, urls):
+ mirror_dir = config.get('VCS', 'mirror_dir')
+ f = open(os.path.join(mirror_dir, 'vcs.txt'), 'w')
+ for url in urls:
+ f.write(url)
+ f.write('\n')
+ f.close()
+
+def _run_vcsmirror(options, config):
+ exec_basedir = os.path.dirname(sys.argv[0])
+
+ args = [os.path.join(exec_basedir, 'rpmci-vcs-mirror'),
+ '--config', options.config,
+ '--clone-then-exit']
+ print "Running: %r" % (args, )
+ subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr)
+
+def update_config(options, config):
+ mirror_dir = config.get('VCS', 'mirror_dir')
+
+ artifact_set = artifact.ArtifactSet.from_config(config, 'releases')
+
+ fedora_git_urls = set()
+
+ unique_buildtargets = artifact_set.get_build_targets()
+
+ for target in unique_buildtargets:
+ url = artifact.fedora_git_url_for_build_target(config, target)
+ fedora_git_urls.add(url)
+
+ _write_vcs_urls(options, config, fedora_git_urls)
+ _run_vcsmirror(options, config)
+
+ all_urls = set(fedora_git_urls)
+ for url in fedora_git_urls:
+ escaped_url = urllib.quote(url, '')
+ vcsdir = os.path.join(mirror_dir, escaped_url)
+ if not os.path.isdir(vcsdir):
+ raise SystemExit("Not a directory: %r" % (vcsdir, ))
+ specpath = None
+ for filename in os.listdir(vcsdir):
+ if filename.endswith('.spec'):
+ specpath = os.path.join(vcsdir, filename)
+ break
+ assert specpath is not None
+ spec_obj = spec.Spec(specpath)
+ upstream_vcs_url = spec_obj.get_vcs()
+ all_urls.add(upstream_vcs_url)
+
+ _write_vcs_urls(options, config, all_urls)
+ _run_vcsmirror(options, config)
+
+def main():
+ glib.threads_init()
+
+ opts = optparse.OptionParser("usage: %prog [options]")
+ opts.add_option('-c', '--config', dest='config', help="Path to configuration file")
+ opts.add_option('', '--debug', action='store_true', help="Print verbose debugging")
+
+ (options, args) = opts.parse_args()
+
+ if options.config is None:
+ print "Must specify --config"
+ sys.exit(1)
+
+ config = SafeConfigParser({'home': os.environ['HOME']})
+ config.read(options.config)
+ level = logging.DEBUG if options.debug else logging.INFO
+ logging.basicConfig(stream=sys.stderr, level=level)
+
+ update_config(options, config)
+ sys.exit(0)
diff --git a/rpmci/rpmci_vcs_mirror_main.py b/rpmci/rpmci_vcs_mirror_main.py
index 85615a3..f8a78ee 100644
--- a/rpmci/rpmci_vcs_mirror_main.py
+++ b/rpmci/rpmci_vcs_mirror_main.py
@@ -24,10 +24,12 @@ from . import msgqueue
from . import lame_vcs_abstraction
class VCSMirror(object):
- def __init__(self, config, urls):
+ def __init__(self, options, config, urls):
self.config = config
self.urls = urls
+ self._options = options
+
self._dir = config.get('VCS', 'mirror_dir')
self._process_dir = config.get('VCS', 'process_logdir')
if not os.path.isdir(self._process_dir):
@@ -59,7 +61,7 @@ class VCSMirror(object):
return vcs.get_url().netloc
def _escape_vcs_url(self, vcs):
- return urllib.quote(vcs.get_base_url_string(), '')
+ return urllib.quote(vcs.get_url_string(), '')
def _cachedir_for_vcs(self, vcs):
return os.path.join(self._dir, self._escape_vcs_url(vcs))
@@ -105,13 +107,14 @@ class VCSMirror(object):
current_id = vcs.get_id()
if current_id != previous_id:
logging.info("vcs %r: New commit id %r differs from previous %r" % (vcs, current_id, previous_id))
- msg = msgqueue.Message(None, {'type': 'update'}, {'id': current_id})
+ msg = msgqueue.Message(None, {'type': 'update'}, {'url': vcs.get_url_string(), 'id': current_id})
self._msgqueue.append(msg)
else:
logging.info("No changes in %r from previous commit id %r" % (vcs, previous_id))
- target_time = int(time.time() + self._timeout_seconds)
- self._vcs_queue.append((vcs, target_time))
+ if not failed and not self._options.clone_then_exit:
+ target_time = int(time.time() + self._timeout_seconds)
+ self._vcs_queue.append((vcs, target_time))
self._poll()
@@ -144,13 +147,14 @@ class VCSMirror(object):
glib.source_remove(self._active_queue_timeout_id)
self._active_queue_timeout_seconds = timeout
self._active_queue_timeout_id = glib.timeout_add_seconds(timeout, self._poll)
-
def _poll(self):
current_time = int(time.time())
- logging.info("Doing poll (%d active tasks)" % (self._num_active_jobs(), ))
+ orig_active_jobs = self._num_active_jobs()
+
logging.debug("Queue: %r" % (self._vcs_queue, ))
+
processed = []
for vcs, target_time in self._vcs_queue:
active = self._job_for_vcs(vcs)
@@ -177,7 +181,12 @@ class VCSMirror(object):
previous_id = None
logging.info("Doing initial checkout for %r" % (vcs.get_base_url_string(), ))
vcs_tempdir = vcsdir + '.tmp'
+ if os.path.isdir(vcs_tempdir):
+ shutil.rmtree(vcs_tempdir)
process = vcs.checkout_async(vcs_tempdir, job_logpath, self._on_job_exited)
+ elif self._options.clone_then_exit:
+ processed.append(vcs)
+ continue
else:
vcs.set_directory(vcsdir)
previous_id = vcs.get_id()
@@ -186,6 +195,8 @@ class VCSMirror(object):
process.__vcs = vcs
self._jobs_by_host[host].append((process, previous_id))
processed.append(vcs)
+
+ added_job_count = len(processed)
while processed:
vcs = processed[0]
del processed[0]
@@ -196,8 +207,18 @@ class VCSMirror(object):
break
assert index >= 0
del self._vcs_queue[index]
+
+ new_active_jobs = self._num_active_jobs()
+
+ if len(self._vcs_queue) == 0 and new_active_jobs == 0:
+ logging.info("Queue is empty and no active jobs. Exiting.")
+ sys.exit(0)
self._adjust_timeout()
+ if new_active_jobs == 0:
+ logging.info("No active jobs; sleeping for %d seconds" % (self._active_queue_timeout_seconds, ))
+ else:
+ logging.info("Poll complete, started %d jobs (%d total)" % (new_active_jobs - orig_active_jobs, new_active_jobs))
return False
@@ -208,6 +229,7 @@ def main():
opts = optparse.OptionParser("usage: %prog [options]")
opts.add_option('-c', '--config', dest='config', help="Path to configuration file")
opts.add_option('', '--debug', action='store_true', help="Print verbose debugging")
+ opts.add_option('', '--clone-then-exit', action='store_true', help="If true, perform any necessary clones, then exit")
(options, args) = opts.parse_args()
@@ -232,7 +254,7 @@ def main():
urls = f.readlines()
f.close()
- mirror = VCSMirror(config, urls)
+ mirror = VCSMirror(options, config, urls)
mirror.start()
loop = glib.MainLoop()
diff --git a/rpmci/spec.py b/rpmci/spec.py
index 40ef6af..aa804d8 100644
--- a/rpmci/spec.py
+++ b/rpmci/spec.py
@@ -242,7 +242,7 @@ the 42 with new_value, preserving the comment # foo."""
for line in self._lines:
if line.startswith('#VCS:'):
return line[5:].strip()
- raise ValueError("No such key #VCS in file %r" % (self._filename, ))
+ return self.get_key('VCS')
def get_key(self, key):
key = key + ':'
diff --git a/sample.config b/sample.config
index afeeda0..8c87f3a 100644
--- a/sample.config
+++ b/sample.config
@@ -1,3 +1,12 @@
+[fedora]
+master=f15
+
+[releases]
+artifacts=gnome-3-f15
+artifact_gnome_3_f15_modules=glib2 pixman cairo gobject-introspection libgee vala dconf pango atk gdk-pixbuf2 gtk2 gtk3 json-glib clutter mutter gjs gnome-shell
+artifact_gnome_3_f15_os=f15
+artifact_gnome_3_f15_architectures=i386 x86_64
+
[VCS]
# Standard stuff
basedir=%(home)s/rpmci
@@ -12,3 +21,20 @@ poll_seconds=60
max_host_concurrency=4
# Hard limit on maximum concurrent operations
max_concurrency=16
+
+[SRPM]
+basedir=%(home)s/rpmci
+msg_basedir=%(basedir)s/msgqueue
+process_logdir=%(basedir)s/process-logs
+
+msgqueue=%(msg_basedir)s/srpm-queue
+fedpkg_dir=%(basedir)/fedpkg
+srpm_dir=%(basedir)/srpms
+
+[build]
+basedir=%(home)s/rpmci
+msg_basedir=%(basedir)s/msgqueue
+process_logdir=%(basedir)s/process-logs
+
+msgqueue=%(msg_basedir)s/build
+artifactdir=%(basedir)/repos