Closer to building binary RPMs

6 files changed, 105 insertions, 186 deletions

diff --git a/rpmci/dynrepo.py b/rpmci/dynrepo.py
deleted file mode 100644
index 2b8199a..0000000
--- a/rpmci/dynrepo.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/python
-
-# dynrepo.py:
-# Monitor set of RPMs in a directory.
-#
-# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
-# Copyright (C) 2010 Red Hat, Inc.
-# Written by Colin Walters <walters@verbum.org>
-
-import os
-import sys
-import logging
-
-import glib
-import gio
-import rpm
-import rpmUtils
-import rpmUtils.miscutils
-
-from . import subtask
-
-class Repo(object):
-    def __init__(self, dirpath):
-        self._dir = dirpath
-        self._dir_gfile = gio.File(path=dirpath)
-        self._monitor = self._dir_gfile.monitor(gio.FILE_MONITOR_NONE)
-        self._monitor.connect('changed', self._on_dir_changed)
-        self._rpms = {}
-
-    def get_rpms(self):
-        return self._rpms
-
-    def get_rpms_for_name(self, name, arch):
-        result = []
-        for filename,(n,a,e,v,r) in self._rpms.iteritems():
-            if n == name and (arch is None or a == arch):
-                result.append((filename, n, a, e, v, r))
-        return result
-
-    def get_latest_srpm_for_name(self, name):
-        rpms = self.get_rpms_for_name(name, 'src')
-        cmpevr=rpmUtils.miscutils.compareEVR
-        if rpms:
-            rpms.sort(lambda a,b: cmpevr(a[3:6], b[3:6]))
-            return rpms[-1]
-        return None
- 
-    def add_srpm(self, srpm_path):
-        basename = os.path.basename(srpm_path)
-        os.link(srpm_path, os.path.join(self._dir, basename))
-
-    def _delete_old_rpms_in_dir(self, dirpath):
-        output = subtask.spawn_sync_get_output('srpm-repomanage', ['repomanage', '-o', '.'], cwd=dirpath)
-        for line in output.split('\n'):
-             if line.endswith('.rpm') and os.path.exists(line):
-               os.unlink(line)
-
-    def update_repo_sync(self):
-        self._delete_old_rpms_in_dir(self._dir)
-        subtask.spawn_sync('srpm-createrepo', ['createrepo', '.'], cwd=self._dir)
-
-    def _headers_from_packages(self, rpmlist):
-        result = {}
-        ts = rpm.TransactionSet()
-        ts.setVSFlags(~(rpm._RPMVSF_NOPAYLOAD))
-        for pkg in rpmlist:
-            pkg_path = os.path.join(self._dir, pkg)
-            try:
-               header = rpmUtils.miscutils.hdrFromPackage(ts, pkg_path)
-            except rpmUtils.RpmUtilsError, e:
-                logging.exception(e)
-                continue
-            (n,a,e,v,r) = rpmUtils.miscutils.pkgTupleFromHeader(header)
-            del header
-            result[pkg] = (n,a,e,v,r)
-        return result
-
-    def _on_dir_changed(self, mon, gfile, other, event):
-        self._reload()
-
-    def _reload(self):
-        dir_contents = os.listdir(self._dir)
-        messages = set()
-        rpmlist = set()
-        for filename in dir_contents:
-            if filename.endswith('.tmp'):
-                continue
-            if not filename.endswith('.rpm'):
-                continue
-            rpmlist.add(filename)
-        
-        deleted = []
-        for rpm in self._rpms:
-            if rpm not in rpmlist:
-                deleted.add(rpm)
-        for rpm in deleted:
-            del self._rpms[rpm]
-        new = []
-        for rpm in self._rpms:
-            if rpm not in rpmlist:
-                new.append(rpm)
-        for rpm,data in self._headers_from_packages(new).iteritems():
-            self._rpms[rpm] = data
diff --git a/rpmci/rpmci_binrpm_builder_main.py b/rpmci/rpmci_binrpm_builder_main.py
index 23ed99c..3854434 100644
--- a/rpmci/rpmci_binrpm_builder_main.py
+++ b/rpmci/rpmci_binrpm_builder_main.py
@@ -24,15 +24,13 @@ import gio
 
 from . import msgqueue
 from . import subtask
-from . import dynrepo
+from . import rpmutils
+from . import versioned_repos
 from . import artifact
-from . import spec
-from . import lame_vcs_abstraction
 
 class BinRPMBuilder(object):
     def __init__(self, options, config):
         self.config = config
-
         self._options = options
 
         srpm_msgqueue_dir = config.get('SRPM', 'msgqueue')
@@ -41,13 +39,17 @@ class BinRPMBuilder(object):
         self._artifactset = artifact.ArtifactSet.from_config(config)
         self._artifact_to_repo = {}
         artifact_basedir = config.get('build', 'artifactdir')
-        for artifact in self._artifactset.artifacts:
-            artifact_repopath = os.path.join(artifact_basedir, artifact.name)
-            self._artifact_to_repo[artifact] = dynrepo.Repo(artifact_repopath)
+        for artifact_iter in self._artifactset.artifacts:
+            artifact_repopath = os.path.join(artifact_basedir, artifact_iter.name)
+            artifact_repo = versioned_repos.BaseRepo(artifact_repopath, 'repo-' + artifact_iter.name)
+            self._artifact_to_repo[artifact] = artifact_repo
 
-        self._srcrepo = dynrepo.Repo(config.get('SRPM', 'spm_dir'))
+        self._srcrepo = versioned_repos.VersionedRepo(config.get('SRPM', 'srpm_dir'))
 
     def start(self):
+        latest = self._srcrepo.get_latest_version()
+        if latest is not None:
+            self._rebuild_to_repoversion(latest.version)
         self._srpm_msgqueue.connect(self._on_srpm_message)
 
     def _target_for_srpm(self, srpm_name):
@@ -57,25 +59,26 @@ class BinRPMBuilder(object):
                 return target
         return None
 
-    def _srpm_for_target(self, repo, target):
-        for rpm in repo.iter_rpms():
-            basename = os.path.basename(rpm):
+    def _srpm_for_target(self, target):
+        latest = self._srcrepo.get_latest_version()
+        for rpm in latest.iter_rpms():
+            basename = os.path.basename(rpm)
             srpm_name = rpmutils.get_rpm_name(basename)
             if srpm_name == target.module:
                 return rpm
         return None
 
-    def _mock_root_from_os_arch(self, os, architecture):
-        fedora_os_master = config.get('fedora', 'master')
-        if buildtarget.os == fedora_os_master:
-            os = 'devel':
+    def _mock_root_from_os_arch(self, target_os, architecture):
+        fedora_os_master = self.config.get('fedora', 'master')
+        if target_os == fedora_os_master:
+            mock_os = 'fedora-devel'
         else:
-            os = buildtarget.os
-        if target.architecture == 'i686':
+            mock_os = buildtarget.os
+        if architecture == 'i686':
             arch = 'i386'
         else:
-            arch = target.architecture
-        return '%s-%s' % (os, arch)
+            arch = architecture
+        return '%s-%s' % (mock_os, arch)
 
     def _on_srpm_message(self, q, messages):
         msg_list = list(messages)
@@ -112,17 +115,18 @@ class BinRPMBuilder(object):
             arch_roots = []
             for arch in artifact.architectures:
                 arch_roots.append(self._mock_root_from_os_arch(artifact.os, arch))
-            base_artifactdir = self._config.get('build', 'artifactdir')
+            base_artifactdir = self.config.get('build', 'artifactdir')
             artifactdir = os.path.join(base_artifactdir, artifact.name)
-            base_logdir = self._config.get('build', 'logdir')
+            base_logdir = self.config.get('build', 'logdir')
             logdir = os.path.join(base_logdir, artifact.name)
 
             args = [os.path.join(exec_basedir, 'mock-many-srpms'),
                     '--skip-have-build',
-                    '--resultdir=' + resultdir,
+                    '--resultdir=' + artifactdir,
                     '--logdir=' + logdir]
             for root in arch_roots:
                 args.append('--root=' + root)
+            args.extend(artifact_srpms)
             subtask.spawn_sync(taskid, args)
 
 def main():
@@ -146,7 +150,7 @@ def main():
 
     subtask.global_configure(config)
 
-    builder = SRPMBuilder(options, config)
+    builder = BinRPMBuilder(options, config)
     builder.start()
     
     logging.info("Awaiting events")
diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py
index 8e8c1ee..6ca7195 100644
--- a/rpmci/rpmci_srpm_builder_main.py
+++ b/rpmci/rpmci_srpm_builder_main.py
@@ -89,6 +89,18 @@ class SRPMBuilder(object):
                 return True
         shutil.rmtree(tmpdir)
         return False
+
+    def _targets_for_vcs_url(self, url):
+        targets = []
+        for iter_target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems():
+            fedora_url = fedora_vcs.get_url_string()
+            is_fedora = (url == fedora_url)
+            if not is_fedora:
+                upstream_url = upstream_vcs.get_url_string()
+                if not url == upstream_url:
+                    continue
+            targets.append(iter_target)
+        return targets
         
     def _on_vcs_message(self, q, messages):
         msg_list = list(messages)
@@ -101,43 +113,46 @@ class SRPMBuilder(object):
             q.consume(msg)
 
         logging.debug("Creating SRPMs for VCS urls: %r" % (updated_vcs_urls, ))
+
+        # This is a bit gross; we find the module-os pairs we need to rebuild,
+        # disregarding architecture.  Really, we should change BuildTarget to
+        # just be module name, and have it be scoped to artifact.
+        non_arch_os_modules = set()
+        for url in updated_vcs_urls:
+            targets = self._targets_for_vcs_url(url)
+            if len(targets) == 0:
+                logging.warn("Couldn't find target for VCS url %r" % (url, ))
+                continue
+            non_arch_os_modules.add((targets[0].module, targets[0].os))
+
         work_dir = tempfile.mkdtemp('.tmp', 'srpm-builder')
         updated_srpms = set()
         updated_srpm_names = set()
-        for url in updated_vcs_urls:
-            if srpm in updated_srpms:
-                continue
-            srpm = self._create_srpm_for_updated_vcs_url(work_dir, url)
-            updated_srpms.add(srpm)
-            updated_srpm_names.add(rpmutils.get_rpm_name(os.path.basename(srpm)))
+        for (iter_module, iter_os) in non_arch_os_modules:
+            found = False
+            # Now find a target (disregarding arch) for each one, which gives us a ref
+            # info the respective VCSes.
+            for target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems():
+                if target.module == iter_module and target.os == iter_os:
+                    srpm = self._create_srpm(work_dir, target.module, target.os, fedora_vcs, upstream_vcs)
+                    updated_srpms.add(srpm)
+                    updated_srpm_names.add(rpmutils.get_rpm_name(os.path.basename(srpm)))
+                    found = True
+                    break
+            assert found
+
         commitid = self._srcrepo.commit_sync(updated_srpms)
         logging.info("New repository revision %d updates SRPMs=%r" % (commitid, updated_srpm_names))
         shutil.rmtree(work_dir)
         self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'version': commitid}))
         logging.debug("Processed %d messages successfully" % (num_msgs, ))
 
-    def _create_srpm_for_updated_vcs_url(self, work_dir, url):
-        is_fedora = False
-        target = None
-        valid_urls = []
-        for iter_target,(fedora_vcs, upstream_vcs) in self._target_vcs.iteritems():
-            fedora_url = fedora_vcs.get_url_string()
-            is_fedora = (url == fedora_url)
-            if not is_fedora:
-                upstream_url = upstream_vcs.get_url_string()
-                if not url == upstream_url:
-                    valid_urls.append(upstream_url)
-                    continue
-            target = iter_target
-            break
-        assert target is not None, ("Couldn't find target for url %r; valid=%r" % (url, valid_urls))
-        (fedora_vcs, upstream_vcs) = self._target_vcs[target]
-
-        logging.debug("Creating new SRPM for %r in %r" % (target.module, work_dir))
-        fedora_dir = os.path.join(work_dir, target.module)
+    def _create_srpm(self, work_dir, module, os_name, fedora_vcs, upstream_vcs):
+        logging.debug("Creating new SRPM for %s-%s in %r" % (module, os_name, work_dir))
+        fedora_dir = os.path.join(work_dir, module)
         fedora_vcs.export_directory('HEAD', fedora_dir, sys.stderr) 
 
-        specname = target.module + '.spec'
+        specname = module + '.spec'
         target_spec_path = os.path.join(fedora_dir, specname)
         target_spec_obj = spec.Spec(target_spec_path)
 
@@ -149,7 +164,7 @@ class SRPMBuilder(object):
                 filename = os.path.basename(rpmpath)
                 assert filename.endswith('.src.rpm')
                 name = rpmutils.get_rpm_name(filename)
-                if name == target.module:
+                if name == module:
                     orig_spec_path = os.path.join(work_dir, specname)
                     if not self._extract_specfile_from_srpm(rpmpath, orig_spec_path):
                         raise ValueError("Failed to extract specfile from %r" % (rpmpath, ))
@@ -163,7 +178,7 @@ class SRPMBuilder(object):
             orig_version = orig_spec_obj.get_key('Version')
             orig_release = orig_spec_obj.get_key('Release')
 
-            logging.info("Will update %r from version=%r release=%r" % (target.module, orig_version, orig_release))
+            logging.info("Will update %r from version=%r release=%r" % (module, orig_version, orig_release))
 
             target_spec_obj.set_key('Version', orig_version)
             target_spec_obj.set_key('Release', orig_release)
@@ -174,18 +189,18 @@ class SRPMBuilder(object):
         args = [os.path.abspath(os.path.join(exec_basedir, 'rpmci-spec-vcs')),
                 '--vcsdir=' + upstream_vcs.get_directory(),
                 'set-revision', 'HEAD']
-        subtask.spawn_sync('rpmci-spec-vcs-%s' % (target.module, ),
+        subtask.spawn_sync('rpmci-spec-vcs-%s' % (module, ),
                            args, cwd=fedora_dir)
         target_spec_obj = spec.Spec(target_spec_path)
         new_version = target_spec_obj.get_key('Version')
         new_release = target_spec_obj.get_key('Release')
         
-        logging.info("Updated %r to version=%r release=%r" % (target.module, new_version, new_release))
+        logging.info("Updated %r to version=%r release=%r" % (module, new_version, new_release))
 
         args = ['rpmbuild']
         args.extend(self._get_base_rpmbuild_args(fedora_dir))
         args.extend(['-bs', specname])
-        subtask.spawn_sync('rpmbuild-srpm-%s' % (target.module, ),
+        subtask.spawn_sync('rpmbuild-srpm-%s' % (module, ),
                            args, cwd=fedora_dir)
         srpm_path = None
         srpm_basename = None
diff --git a/rpmci/subtask.py b/rpmci/subtask.py
index a3902be..a95831a 100644
--- a/rpmci/subtask.py
+++ b/rpmci/subtask.py
@@ -45,33 +45,31 @@ def _init_task_run(taskid, argv, cwd):
     f.flush()
     return (log_path, f)
 
+def _handle_task_exit(taskid, log_path, logf, proc):
+    ecode = proc.wait()
+    if ecode == 0:
+        msg = "Subtask %s completed succesfully" % (taskid, )
+        logging.info(msg)
+    else:
+        msg = "Subtask %s exited with code %d" % (taskid, ecode)
+        logging.warn(msg)
+    logf.write(msg)
+    logf.close()
+    if ecode != 0:
+        shutil.move(log_path, _failed_path)
+
 def spawn_sync(taskid, argv, cwd=None):
     (log_path, f) = _init_task_run(taskid, argv, cwd)
     nullf = open(os.devnull, 'w')
-    try:
-        proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f)
-        logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid))
-        proc.wait()
-    except subprocess.CalledProcessError, e:
-        logging.exception(e)
-        f.write("Failed: %r" % (e, ))
-        f.close()
-        shutil.move(log_path, _failed_path)
-        raise e
-    f.close()
+    proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f)
+    logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid))
+    _handle_task_exit(taskid, log_path, f, proc)
 
 def spawn_sync_get_output(taskid, argv, cwd=None):
     (log_path, f) = _init_task_run(taskid, argv, cwd)
     nullf = open(os.devnull, 'w')
-    try:
-        proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=subprocess.PIPE, stderr=f)
-        logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid))
-        output = proc.communicate()[0]
-    except subprocess.CalledProcessError, e:
-        logging.exception(e)
-        f.write("Failed: %r" % (e, ))
-        f.close()
-        shutil.move(log_path, _failed_path)
-        raise e
-    f.close()
+    proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=subprocess.PIPE, stderr=f)
+    logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid))
+    output = proc.communicate()[0]
+    _handle_task_exit(taskid, log_path, f, proc)
     return output
diff --git a/rpmci/versioned_repos.py b/rpmci/versioned_repos.py
index 6ee4b6c..4275a2b 100644
--- a/rpmci/versioned_repos.py
+++ b/rpmci/versioned_repos.py
@@ -19,12 +19,10 @@ import rpmUtils.miscutils
 
 from . import subtask
 
-class Repo(object):
-    def __init__(self, basedir, name, version):
+class BaseRepo(object):
+    def __init__(self, dirpath, name):
+        self._dir = dirpath
         self.name = name
-        self.version = version
-        self._dirname = '%d' % (version, )
-        self._dir = os.path.join(basedir, self._dirname)
         self._task_basename = 'repo-%s' % (name, )
         if not os.path.isdir(self._dir):
             os.makedirs(self._dir)
@@ -56,6 +54,13 @@ class Repo(object):
         subtask.spawn_sync('createrepo' + self._task_basename,
                            ['createrepo', '.'], cwd=self._dir)
 
+class Repo(BaseRepo):
+    def __init__(self, basedir, name, version):
+        dirname = '%d' % (version, )
+        dirpath = os.path.join(basedir, dirname)
+        BaseRepo.__init__(self, dirpath, name)
+        self.version = version
+
 class VersionedRepo(object):
     def __init__(self, dirpath):
         self._dir = dirpath
diff --git a/sample.config b/sample.config
index 164fcef..5683f29 100644
--- a/sample.config
+++ b/sample.config
@@ -40,5 +40,5 @@ basedir=%(home)s/rpmci
 msg_basedir=%(basedir)s/msgqueue
 
 msgqueue=%(msg_basedir)s/build
-logdir=%(basedir)/build-logs
-artifactdir=%(basedir)/repos
+logdir=%(basedir)s/build-logs
+artifactdir=%(basedir)s/repos