From dc107b408cd5de88deffe491671e4f435789ac45 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 8 Oct 2010 14:33:11 -0400 Subject: Closer to building binary RPMs --- rpmci/dynrepo.py | 103 ------------------------------------- rpmci/rpmci_binrpm_builder_main.py | 50 +++++++++--------- rpmci/rpmci_srpm_builder_main.py | 77 ++++++++++++++++----------- rpmci/subtask.py | 42 +++++++-------- rpmci/versioned_repos.py | 15 ++++-- sample.config | 4 +- 6 files changed, 105 insertions(+), 186 deletions(-) delete mode 100644 rpmci/dynrepo.py diff --git a/rpmci/dynrepo.py b/rpmci/dynrepo.py deleted file mode 100644 index 2b8199a..0000000 --- a/rpmci/dynrepo.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/python - -# dynrepo.py: -# Monitor set of RPMs in a directory. -# -# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) -# Copyright (C) 2010 Red Hat, Inc. -# Written by Colin Walters - -import os -import sys -import logging - -import glib -import gio -import rpm -import rpmUtils -import rpmUtils.miscutils - -from . import subtask - -class Repo(object): - def __init__(self, dirpath): - self._dir = dirpath - self._dir_gfile = gio.File(path=dirpath) - self._monitor = self._dir_gfile.monitor(gio.FILE_MONITOR_NONE) - self._monitor.connect('changed', self._on_dir_changed) - self._rpms = {} - - def get_rpms(self): - return self._rpms - - def get_rpms_for_name(self, name, arch): - result = [] - for filename,(n,a,e,v,r) in self._rpms.iteritems(): - if n == name and (arch is None or a == arch): - result.append((filename, n, a, e, v, r)) - return result - - def get_latest_srpm_for_name(self, name): - rpms = self.get_rpms_for_name(name, 'src') - cmpevr=rpmUtils.miscutils.compareEVR - if rpms: - rpms.sort(lambda a,b: cmpevr(a[3:6], b[3:6])) - return rpms[-1] - return None - - def add_srpm(self, srpm_path): - basename = os.path.basename(srpm_path) - os.link(srpm_path, os.path.join(self._dir, basename)) - - def _delete_old_rpms_in_dir(self, dirpath): - output = subtask.spawn_sync_get_output('srpm-repomanage', ['repomanage', '-o', '.'], cwd=dirpath) - for line in output.split('\n'): - if line.endswith('.rpm') and os.path.exists(line): - os.unlink(line) - - def update_repo_sync(self): - self._delete_old_rpms_in_dir(self._dir) - subtask.spawn_sync('srpm-createrepo', ['createrepo', '.'], cwd=self._dir) - - def _headers_from_packages(self, rpmlist): - result = {} - ts = rpm.TransactionSet() - ts.setVSFlags(~(rpm._RPMVSF_NOPAYLOAD)) - for pkg in rpmlist: - pkg_path = os.path.join(self._dir, pkg) - try: - header = rpmUtils.miscutils.hdrFromPackage(ts, pkg_path) - except rpmUtils.RpmUtilsError, e: - logging.exception(e) - continue - (n,a,e,v,r) = rpmUtils.miscutils.pkgTupleFromHeader(header) - del header - result[pkg] = (n,a,e,v,r) - return result - - def _on_dir_changed(self, mon, gfile, other, event): - self._reload() - - def _reload(self): - dir_contents = os.listdir(self._dir) - messages = set() - rpmlist = set() - for filename in dir_contents: - if filename.endswith('.tmp'): - continue - if not filename.endswith('.rpm'): - continue - rpmlist.add(filename) - - deleted = [] - for rpm in self._rpms: - if rpm not in rpmlist: - deleted.add(rpm) - for rpm in deleted: - del self._rpms[rpm] - new = [] - for rpm in self._rpms: - if rpm not in rpmlist: - new.append(rpm) - for rpm,data in self._headers_from_packages(new).iteritems(): - self._rpms[rpm] = data diff --git a/rpmci/rpmci_binrpm_builder_main.py b/rpmci/rpmci_binrpm_builder_main.py index 23ed99c..3854434 100644 --- a/rpmci/rpmci_binrpm_builder_main.py +++ b/rpmci/rpmci_binrpm_builder_main.py @@ -24,15 +24,13 @@ import gio from . import msgqueue from . import subtask -from . import dynrepo +from . import rpmutils +from . import versioned_repos from . import artifact -from . import spec -from . import lame_vcs_abstraction class BinRPMBuilder(object): def __init__(self, options, config): self.config = config - self._options = options srpm_msgqueue_dir = config.get('SRPM', 'msgqueue') @@ -41,13 +39,17 @@ class BinRPMBuilder(object): self._artifactset = artifact.ArtifactSet.from_config(config) self._artifact_to_repo = {} artifact_basedir = config.get('build', 'artifactdir') - for artifact in self._artifactset.artifacts: - artifact_repopath = os.path.join(artifact_basedir, artifact.name) - self._artifact_to_repo[artifact] = dynrepo.Repo(artifact_repopath) + for artifact_iter in self._artifactset.artifacts: + artifact_repopath = os.path.join(artifact_basedir, artifact_iter.name) + artifact_repo = versioned_repos.BaseRepo(artifact_repopath, 'repo-' + artifact_iter.name) + self._artifact_to_repo[artifact] = artifact_repo - self._srcrepo = dynrepo.Repo(config.get('SRPM', 'spm_dir')) + self._srcrepo = versioned_repos.VersionedRepo(config.get('SRPM', 'srpm_dir')) def start(self): + latest = self._srcrepo.get_latest_version() + if latest is not None: + self._rebuild_to_repoversion(latest.version) self._srpm_msgqueue.connect(self._on_srpm_message) def _target_for_srpm(self, srpm_name): @@ -57,25 +59,26 @@ class BinRPMBuilder(object): return target return None - def _srpm_for_target(self, repo, target): - for rpm in repo.iter_rpms(): - basename = os.path.basename(rpm): + def _srpm_for_target(self, target): + latest = self._srcrepo.get_latest_version() + for rpm in latest.iter_rpms(): + basename = os.path.basename(rpm) srpm_name = rpmutils.get_rpm_name(basename) if srpm_name == target.module: return rpm return None - def _mock_root_from_os_arch(self, os, architecture): - fedora_os_master = config.get('fedora', 'master') - if buildtarget.os == fedora_os_master: - os = 'devel': + def _mock_root_from_os_arch(self, target_os, architecture): + fedora_os_master = self.config.get('fedora', 'master') + if target_os == fedora_os_master: + mock_os = 'fedora-devel' else: - os = buildtarget.os - if target.architecture == 'i686': + mock_os = buildtarget.os + if architecture == 'i686': arch = 'i386' else: - arch = target.architecture - return '%s-%s' % (os, arch) + arch = architecture + return '%s-%s' % (mock_os, arch) def _on_srpm_message(self, q, messages): msg_list = list(messages) @@ -112,17 +115,18 @@ class BinRPMBuilder(object): arch_roots = [] for arch in artifact.architectures: arch_roots.append(self._mock_root_from_os_arch(artifact.os, arch)) - base_artifactdir = self._config.get('build', 'artifactdir') + base_artifactdir = self.config.get('build', 'artifactdir') artifactdir = os.path.join(base_artifactdir, artifact.name) - base_logdir = self._config.get('build', 'logdir') + base_logdir = self.config.get('build', 'logdir') logdir = os.path.join(base_logdir, artifact.name) args = [os.path.join(exec_basedir, 'mock-many-srpms'), '--skip-have-build', - '--resultdir=' + resultdir, + '--resultdir=' + artifactdir, '--logdir=' + logdir] for root in arch_roots: args.append('--root=' + root) + args.extend(artifact_srpms) subtask.spawn_sync(taskid, args) def main(): @@ -146,7 +150,7 @@ def main(): subtask.global_configure(config) - builder = SRPMBuilder(options, config) + builder = BinRPMBuilder(options, config) builder.start() logging.info("Awaiting events") diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py index 8e8c1ee..6ca7195 100644 --- a/rpmci/rpmci_srpm_builder_main.py +++ b/rpmci/rpmci_srpm_builder_main.py @@ -89,6 +89,18 @@ class SRPMBuilder(object): return True shutil.rmtree(tmpdir) return False + + def _targets_for_vcs_url(self, url): + targets = [] + for iter_target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems(): + fedora_url = fedora_vcs.get_url_string() + is_fedora = (url == fedora_url) + if not is_fedora: + upstream_url = upstream_vcs.get_url_string() + if not url == upstream_url: + continue + targets.append(iter_target) + return targets def _on_vcs_message(self, q, messages): msg_list = list(messages) @@ -101,43 +113,46 @@ class SRPMBuilder(object): q.consume(msg) logging.debug("Creating SRPMs for VCS urls: %r" % (updated_vcs_urls, )) + + # This is a bit gross; we find the module-os pairs we need to rebuild, + # disregarding architecture. Really, we should change BuildTarget to + # just be module name, and have it be scoped to artifact. + non_arch_os_modules = set() + for url in updated_vcs_urls: + targets = self._targets_for_vcs_url(url) + if len(targets) == 0: + logging.warn("Couldn't find target for VCS url %r" % (url, )) + continue + non_arch_os_modules.add((targets[0].module, targets[0].os)) + work_dir = tempfile.mkdtemp('.tmp', 'srpm-builder') updated_srpms = set() updated_srpm_names = set() - for url in updated_vcs_urls: - if srpm in updated_srpms: - continue - srpm = self._create_srpm_for_updated_vcs_url(work_dir, url) - updated_srpms.add(srpm) - updated_srpm_names.add(rpmutils.get_rpm_name(os.path.basename(srpm))) + for (iter_module, iter_os) in non_arch_os_modules: + found = False + # Now find a target (disregarding arch) for each one, which gives us a ref + # info the respective VCSes. + for target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems(): + if target.module == iter_module and target.os == iter_os: + srpm = self._create_srpm(work_dir, target.module, target.os, fedora_vcs, upstream_vcs) + updated_srpms.add(srpm) + updated_srpm_names.add(rpmutils.get_rpm_name(os.path.basename(srpm))) + found = True + break + assert found + commitid = self._srcrepo.commit_sync(updated_srpms) logging.info("New repository revision %d updates SRPMs=%r" % (commitid, updated_srpm_names)) shutil.rmtree(work_dir) self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'version': commitid})) logging.debug("Processed %d messages successfully" % (num_msgs, )) - def _create_srpm_for_updated_vcs_url(self, work_dir, url): - is_fedora = False - target = None - valid_urls = [] - for iter_target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems(): - fedora_url = fedora_vcs.get_url_string() - is_fedora = (url == fedora_url) - if not is_fedora: - upstream_url = upstream_vcs.get_url_string() - if not url == upstream_url: - valid_urls.append(upstream_url) - continue - target = iter_target - break - assert target is not None, ("Couldn't find target for url %r; valid=%r" % (url, valid_urls)) - (fedora_vcs, upstream_vcs) = self._target_vcs[target] - - logging.debug("Creating new SRPM for %r in %r" % (target.module, work_dir)) - fedora_dir = os.path.join(work_dir, target.module) + def _create_srpm(self, work_dir, module, os_name, fedora_vcs, upstream_vcs): + logging.debug("Creating new SRPM for %s-%s in %r" % (module, os_name, work_dir)) + fedora_dir = os.path.join(work_dir, module) fedora_vcs.export_directory('HEAD', fedora_dir, sys.stderr) - specname = target.module + '.spec' + specname = module + '.spec' target_spec_path = os.path.join(fedora_dir, specname) target_spec_obj = spec.Spec(target_spec_path) @@ -149,7 +164,7 @@ class SRPMBuilder(object): filename = os.path.basename(rpmpath) assert filename.endswith('.src.rpm') name = rpmutils.get_rpm_name(filename) - if name == target.module: + if name == module: orig_spec_path = os.path.join(work_dir, specname) if not self._extract_specfile_from_srpm(rpmpath, orig_spec_path): raise ValueError("Failed to extract specfile from %r" % (rpmpath, )) @@ -163,7 +178,7 @@ class SRPMBuilder(object): orig_version = orig_spec_obj.get_key('Version') orig_release = orig_spec_obj.get_key('Release') - logging.info("Will update %r from version=%r release=%r" % (target.module, orig_version, orig_release)) + logging.info("Will update %r from version=%r release=%r" % (module, orig_version, orig_release)) target_spec_obj.set_key('Version', orig_version) target_spec_obj.set_key('Release', orig_release) @@ -174,18 +189,18 @@ class SRPMBuilder(object): args = [os.path.abspath(os.path.join(exec_basedir, 'rpmci-spec-vcs')), '--vcsdir=' + upstream_vcs.get_directory(), 'set-revision', 'HEAD'] - subtask.spawn_sync('rpmci-spec-vcs-%s' % (target.module, ), + subtask.spawn_sync('rpmci-spec-vcs-%s' % (module, ), args, cwd=fedora_dir) target_spec_obj = spec.Spec(target_spec_path) new_version = target_spec_obj.get_key('Version') new_release = target_spec_obj.get_key('Release') - logging.info("Updated %r to version=%r release=%r" % (target.module, new_version, new_release)) + logging.info("Updated %r to version=%r release=%r" % (module, new_version, new_release)) args = ['rpmbuild'] args.extend(self._get_base_rpmbuild_args(fedora_dir)) args.extend(['-bs', specname]) - subtask.spawn_sync('rpmbuild-srpm-%s' % (target.module, ), + subtask.spawn_sync('rpmbuild-srpm-%s' % (module, ), args, cwd=fedora_dir) srpm_path = None srpm_basename = None diff --git a/rpmci/subtask.py b/rpmci/subtask.py index a3902be..a95831a 100644 --- a/rpmci/subtask.py +++ b/rpmci/subtask.py @@ -45,33 +45,31 @@ def _init_task_run(taskid, argv, cwd): f.flush() return (log_path, f) +def _handle_task_exit(taskid, log_path, logf, proc): + ecode = proc.wait() + if ecode == 0: + msg = "Subtask %s completed succesfully" % (taskid, ) + logging.info(msg) + else: + msg = "Subtask %s exited with code %d" % (taskid, ecode) + logging.warn(msg) + logf.write(msg) + logf.close() + if ecode != 0: + shutil.move(log_path, _failed_path) + def spawn_sync(taskid, argv, cwd=None): (log_path, f) = _init_task_run(taskid, argv, cwd) nullf = open(os.devnull, 'w') - try: - proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f) - logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid)) - proc.wait() - except subprocess.CalledProcessError, e: - logging.exception(e) - f.write("Failed: %r" % (e, )) - f.close() - shutil.move(log_path, _failed_path) - raise e - f.close() + proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f) + logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid)) + _handle_task_exit(taskid, log_path, f, proc) def spawn_sync_get_output(taskid, argv, cwd=None): (log_path, f) = _init_task_run(taskid, argv, cwd) nullf = open(os.devnull, 'w') - try: - proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=subprocess.PIPE, stderr=f) - logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid)) - output = proc.communicate()[0] - except subprocess.CalledProcessError, e: - logging.exception(e) - f.write("Failed: %r" % (e, )) - f.close() - shutil.move(log_path, _failed_path) - raise e - f.close() + proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=subprocess.PIPE, stderr=f) + logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid)) + output = proc.communicate()[0] + _handle_task_exit(taskid, log_path, f, proc) return output diff --git a/rpmci/versioned_repos.py b/rpmci/versioned_repos.py index 6ee4b6c..4275a2b 100644 --- a/rpmci/versioned_repos.py +++ b/rpmci/versioned_repos.py @@ -19,12 +19,10 @@ import rpmUtils.miscutils from . import subtask -class Repo(object): - def __init__(self, basedir, name, version): +class BaseRepo(object): + def __init__(self, dirpath, name): + self._dir = dirpath self.name = name - self.version = version - self._dirname = '%d' % (version, ) - self._dir = os.path.join(basedir, self._dirname) self._task_basename = 'repo-%s' % (name, ) if not os.path.isdir(self._dir): os.makedirs(self._dir) @@ -56,6 +54,13 @@ class Repo(object): subtask.spawn_sync('createrepo' + self._task_basename, ['createrepo', '.'], cwd=self._dir) +class Repo(BaseRepo): + def __init__(self, basedir, name, version): + dirname = '%d' % (version, ) + dirpath = os.path.join(basedir, dirname) + BaseRepo.__init__(self, dirpath, name) + self.version = version + class VersionedRepo(object): def __init__(self, dirpath): self._dir = dirpath diff --git a/sample.config b/sample.config index 164fcef..5683f29 100644 --- a/sample.config +++ b/sample.config @@ -40,5 +40,5 @@ basedir=%(home)s/rpmci msg_basedir=%(basedir)s/msgqueue msgqueue=%(msg_basedir)s/build -logdir=%(basedir)/build-logs -artifactdir=%(basedir)/repos +logdir=%(basedir)s/build-logs +artifactdir=%(basedir)s/repos -- cgit