rpmci-srpm-builder: More hacking

4 files changed, 76 insertions, 20 deletions

diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py
index 5efd903..32cbbd8 100644
--- a/rpmci/rpmci_srpm_builder_main.py
+++ b/rpmci/rpmci_srpm_builder_main.py
@@ -27,6 +27,7 @@ from . import subtask
 from . import versioned_repos
 from . import artifact
 from . import spec
+from . import rpmutils
 from . import lame_vcs_abstraction
 
 class SRPMBuilder(object):
@@ -68,6 +69,26 @@ class SRPMBuilder(object):
                 '--define', '_builddir ' + src_dirpath,
                 '--define', '_srcrpmdir ' + src_dirpath,
                 '--define', '_rpmdir ' + src_dirpath]
+
+    def _extract_specfile_from_srpm(self, srpm_path, destpath):
+        tmpdir = tempfile.mkdtemp('', 'extract-srpm-spec-tmp')
+        name = rpmutils.get_rpm_name(os.path.basename(srpm_path))
+        task_logpath = subtask.prepare_task_logfile('extract-srpm-%s' % (name, ))
+        logf = open(task_logpath, 'w')
+        devnull = open(os.devnull, 'w')
+        rpm2cpio = subprocess.Popen(['rpm2cpio', srpm_path], stdout=subprocess.PIPE,
+                                    stderr=logf)
+        cpio = subprocess.Popen(['cpio', '--quiet', '-i', '*.spec'], cwd=tmpdir,
+                                stdin=rpm2cpio.stdout, stdout=devnull,
+                                stderr=devnull)
+        cpio.wait()
+        for filename in os.listdir(tmpdir):
+            if filename.endswith('.spec'):
+                shutil.move(os.path.join(tmpdir, filename), destpath)
+                shutil.rmtree(tmpdir)
+                return True
+        shutil.rmtree(tmpdir)
+        return False
         
     def _on_vcs_message(self, q, messages):
         msg_list = list(messages)
@@ -107,11 +128,19 @@ class SRPMBuilder(object):
         target_spec_obj = spec.Spec(target_spec_path)
 
         latest_repopath = self._srcrepo.get_latest_path()
-        if latest_repopath:
-            orig_spec_path = os.path.join(latest_repopath, specname)
-        else:
-            orig_spec_path = None
-        if orig_spec_path is not None and os.path.isfile(orig_spec_path):
+        latest_repo = self._srcrepo.get_latest_version()
+        orig_spec_path = None
+        if latest_repo:
+            for rpmpath in latest_repo.iter_rpms():
+                filename = os.path.basename(rpmpath)
+                assert filename.endswith('.src.rpm')
+                name = rpmutils.get_rpm_name(filename)
+                if name == target.module:
+                    orig_spec_path = os.path.join(work_dir, specname)
+                    if not self._extract_specfile_from_srpm(rpmpath, orig_spec_path):
+                        raise ValueError("Failed to extract specfile from %r" % (rpmpath, ))
+                    break
+        if orig_spec_path is not None:
             # Okay, we have a previous build, so now we "merge" by taking everything
             # from the new spec file, except we use the Version/Release from the old
             # one.  This is necessary to pacify RPM's insistence on globally
@@ -120,6 +149,8 @@ class SRPMBuilder(object):
             orig_version = orig_spec_obj.get_key('Version')
             orig_release = orig_spec_obj.get_key('Release')
 
+            logging.info("Will update %r from version=%r release=%r" % (target.module, orig_version, orig_release))
+
             target_spec_obj.set_key('Version', orig_version)
             target_spec_obj.set_key('Release', orig_release)
             target_spec_obj.save()
@@ -131,6 +162,11 @@ class SRPMBuilder(object):
                 'set-revision', 'HEAD']
         subtask.spawn_sync('rpmci-spec-vcs-%s' % (target.module, ),
                            args, cwd=fedora_dir)
+        target_spec_obj = spec.Spec(target_spec_path)
+        new_version = target_spec_obj.get_key('Version')
+        new_release = target_spec_obj.get_key('Release')
+        
+        logging.info("Updated %r to version=%r release=%r" % (target.module, new_version, new_release))
 
         args = ['rpmbuild']
         args.extend(self._get_base_rpmbuild_args(fedora_dir))
@@ -145,7 +181,8 @@ class SRPMBuilder(object):
                 srpm_path = os.path.join(fedora_dir, filename)
                 break
         assert srpm_path is not None
-        self._srcrepo.commit([srpm_path])
+        self._srcrepo.commit_sync([srpm_path])
+        shutil.rmtree(work_dir)
         self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'version': srpm_basename}))
 
 def main():
diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py
index dac658e..d7b2265 100644
--- a/rpmci/rpmci_update_config_main.py
+++ b/rpmci/rpmci_update_config_main.py
@@ -64,7 +64,7 @@ def update_config(options, config):
                                         
     all_urls = set(fedora_git_urls)
     for target in unique_buildtargets:
-        upstream_vcs_url = artifact.upstream_vcs_url_for_build_target(target)
+        upstream_vcs_url = artifact.upstream_vcs_url_for_build_target(config, target)
         all_urls.add(upstream_vcs_url)
             
     _write_vcs_urls(options, config, all_urls)
diff --git a/rpmci/subtask.py b/rpmci/subtask.py
index e97279e..a3902be 100644
--- a/rpmci/subtask.py
+++ b/rpmci/subtask.py
@@ -39,29 +39,37 @@ def prepare_task_logfile(taskid):
 
 def _init_task_run(taskid, argv, cwd):
     log_path = prepare_task_logfile(taskid)
-    logging.info("Running task %r synchronously, cwd=%r args=%r" % (taskid, cwd, argv))
-    return log_path
+    logging.info("Calling task %r synchronously, cwd=%r args=%r" % (taskid, cwd, argv))
+    f = open(log_path, 'w')
+    f.write("Logfile for process cwd=%r args=%r\n" % (cwd, argv))
+    f.flush()
+    return (log_path, f)
 
 def spawn_sync(taskid, argv, cwd=None):
-    log_path = _init_task_run(taskid, argv, cwd)
-    f = open(log_path, 'w')
+    (log_path, f) = _init_task_run(taskid, argv, cwd)
     nullf = open(os.devnull, 'w')
     try:
-        subprocess.check_call(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f)
+        proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f)
+        logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid))
+        proc.wait()
     except subprocess.CalledProcessError, e:
+        logging.exception(e)
+        f.write("Failed: %r" % (e, ))
         f.close()
         shutil.move(log_path, _failed_path)
         raise e
     f.close()
 
 def spawn_sync_get_output(taskid, argv, cwd=None):
-    log_path = _init_task_run(taskid, argv, cwd)
-    f = open(log_path, 'w')
+    (log_path, f) = _init_task_run(taskid, argv, cwd)
     nullf = open(os.devnull, 'w')
     try:
         proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=subprocess.PIPE, stderr=f)
+        logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid))
         output = proc.communicate()[0]
     except subprocess.CalledProcessError, e:
+        logging.exception(e)
+        f.write("Failed: %r" % (e, ))
         f.close()
         shutil.move(log_path, _failed_path)
         raise e
diff --git a/rpmci/versioned_repos.py b/rpmci/versioned_repos.py
index 6de753c..6ee4b6c 100644
--- a/rpmci/versioned_repos.py
+++ b/rpmci/versioned_repos.py
@@ -34,7 +34,9 @@ class Repo(object):
 
     def add_rpm(self, rpm_path):
         basename = os.path.basename(rpm_path)
-        os.link(rpm_path, os.path.join(self._dir, basename))
+        dest = os.path.join(self._dir, basename)
+        logging.debug("Linking %r to %r" % (rpm_path, dest))
+        os.link(rpm_path, dest)
 
     def iter_rpms(self):
         for filename in os.listdir(self._dir):
@@ -45,8 +47,9 @@ class Repo(object):
         output = subtask.spawn_sync_get_output('repomanage-' + self._task_basename,
                                                ['repomanage', '-o', '.'], cwd=dirpath)
         for line in output.split('\n'):
-             if line.endswith('.rpm') and os.path.exists(line):
-               os.unlink(line)
+            path = os.path.join(self._dir, line)
+            if line.endswith('.rpm') and os.path.exists(path):
+                os.unlink(path)
 
     def update_sync(self):
         self._delete_old_rpms_in_dir(self._dir)
@@ -69,7 +72,7 @@ class VersionedRepo(object):
     def _reload(self):
         self._versions = {}
         for filename in os.listdir(self._dir):
-            path = os.path.join(self,_dir, filename)
+            path = os.path.join(self._dir, filename)
             if not os.path.isdir(path):
                 continue
             try:
@@ -80,11 +83,17 @@ class VersionedRepo(object):
             if revision > self._last_version:
                 self._last_version = revision
  
-    def get_latest_path(self):
+    def get_latest_version(self):
         if self._last_version == -1:
             return None
-        return self._versions[self._last_version].get_dir()
+        return self._versions[self._last_version]
         
+    def get_latest_path(self):
+        latest = self.get_latest_version()
+        if latest is None:
+            return None
+        return latest.get_dir()
+
     def commit_sync(self, new_rpms):
         rev = self._last_version + 1
         new_repo = Repo(self._dir, self._name, rev)
@@ -95,5 +104,7 @@ class VersionedRepo(object):
         for rpm in new_rpms:
             new_repo.add_rpm(rpm)
         new_repo.update_sync()
+        self._versions[rev] = new_repo
+        self._last_version = rev
         return rev