From 8f7ff8ae7db6f1c3a8b5a0c4eb83c32d795790af Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 7 Oct 2010 23:51:01 -0400 Subject: rpmci-srpm-builder: More hacking --- rpmci/rpmci_srpm_builder_main.py | 49 ++++++++++++++++++++++++++++++++++----- rpmci/rpmci_update_config_main.py | 2 +- rpmci/subtask.py | 22 ++++++++++++------ rpmci/versioned_repos.py | 23 +++++++++++++----- 4 files changed, 76 insertions(+), 20 deletions(-) diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py index 5efd903..32cbbd8 100644 --- a/rpmci/rpmci_srpm_builder_main.py +++ b/rpmci/rpmci_srpm_builder_main.py @@ -27,6 +27,7 @@ from . import subtask from . import versioned_repos from . import artifact from . import spec +from . import rpmutils from . import lame_vcs_abstraction class SRPMBuilder(object): @@ -68,6 +69,26 @@ class SRPMBuilder(object): '--define', '_builddir ' + src_dirpath, '--define', '_srcrpmdir ' + src_dirpath, '--define', '_rpmdir ' + src_dirpath] + + def _extract_specfile_from_srpm(self, srpm_path, destpath): + tmpdir = tempfile.mkdtemp('', 'extract-srpm-spec-tmp') + name = rpmutils.get_rpm_name(os.path.basename(srpm_path)) + task_logpath = subtask.prepare_task_logfile('extract-srpm-%s' % (name, )) + logf = open(task_logpath, 'w') + devnull = open(os.devnull, 'w') + rpm2cpio = subprocess.Popen(['rpm2cpio', srpm_path], stdout=subprocess.PIPE, + stderr=logf) + cpio = subprocess.Popen(['cpio', '--quiet', '-i', '*.spec'], cwd=tmpdir, + stdin=rpm2cpio.stdout, stdout=devnull, + stderr=devnull) + cpio.wait() + for filename in os.listdir(tmpdir): + if filename.endswith('.spec'): + shutil.move(os.path.join(tmpdir, filename), destpath) + shutil.rmtree(tmpdir) + return True + shutil.rmtree(tmpdir) + return False def _on_vcs_message(self, q, messages): msg_list = list(messages) @@ -107,11 +128,19 @@ class SRPMBuilder(object): target_spec_obj = spec.Spec(target_spec_path) latest_repopath = self._srcrepo.get_latest_path() - if latest_repopath: - orig_spec_path = os.path.join(latest_repopath, specname) - else: - orig_spec_path = None - if orig_spec_path is not None and os.path.isfile(orig_spec_path): + latest_repo = self._srcrepo.get_latest_version() + orig_spec_path = None + if latest_repo: + for rpmpath in latest_repo.iter_rpms(): + filename = os.path.basename(rpmpath) + assert filename.endswith('.src.rpm') + name = rpmutils.get_rpm_name(filename) + if name == target.module: + orig_spec_path = os.path.join(work_dir, specname) + if not self._extract_specfile_from_srpm(rpmpath, orig_spec_path): + raise ValueError("Failed to extract specfile from %r" % (rpmpath, )) + break + if orig_spec_path is not None: # Okay, we have a previous build, so now we "merge" by taking everything # from the new spec file, except we use the Version/Release from the old # one. This is necessary to pacify RPM's insistence on globally @@ -120,6 +149,8 @@ class SRPMBuilder(object): orig_version = orig_spec_obj.get_key('Version') orig_release = orig_spec_obj.get_key('Release') + logging.info("Will update %r from version=%r release=%r" % (target.module, orig_version, orig_release)) + target_spec_obj.set_key('Version', orig_version) target_spec_obj.set_key('Release', orig_release) target_spec_obj.save() @@ -131,6 +162,11 @@ class SRPMBuilder(object): 'set-revision', 'HEAD'] subtask.spawn_sync('rpmci-spec-vcs-%s' % (target.module, ), args, cwd=fedora_dir) + target_spec_obj = spec.Spec(target_spec_path) + new_version = target_spec_obj.get_key('Version') + new_release = target_spec_obj.get_key('Release') + + logging.info("Updated %r to version=%r release=%r" % (target.module, new_version, new_release)) args = ['rpmbuild'] args.extend(self._get_base_rpmbuild_args(fedora_dir)) @@ -145,7 +181,8 @@ class SRPMBuilder(object): srpm_path = os.path.join(fedora_dir, filename) break assert srpm_path is not None - self._srcrepo.commit([srpm_path]) + self._srcrepo.commit_sync([srpm_path]) + shutil.rmtree(work_dir) self._srpm_msgqueue.append(msgqueue.Message(None, {'type': 'srpm'}, {'version': srpm_basename})) def main(): diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py index dac658e..d7b2265 100644 --- a/rpmci/rpmci_update_config_main.py +++ b/rpmci/rpmci_update_config_main.py @@ -64,7 +64,7 @@ def update_config(options, config): all_urls = set(fedora_git_urls) for target in unique_buildtargets: - upstream_vcs_url = artifact.upstream_vcs_url_for_build_target(target) + upstream_vcs_url = artifact.upstream_vcs_url_for_build_target(config, target) all_urls.add(upstream_vcs_url) _write_vcs_urls(options, config, all_urls) diff --git a/rpmci/subtask.py b/rpmci/subtask.py index e97279e..a3902be 100644 --- a/rpmci/subtask.py +++ b/rpmci/subtask.py @@ -39,29 +39,37 @@ def prepare_task_logfile(taskid): def _init_task_run(taskid, argv, cwd): log_path = prepare_task_logfile(taskid) - logging.info("Running task %r synchronously, cwd=%r args=%r" % (taskid, cwd, argv)) - return log_path + logging.info("Calling task %r synchronously, cwd=%r args=%r" % (taskid, cwd, argv)) + f = open(log_path, 'w') + f.write("Logfile for process cwd=%r args=%r\n" % (cwd, argv)) + f.flush() + return (log_path, f) def spawn_sync(taskid, argv, cwd=None): - log_path = _init_task_run(taskid, argv, cwd) - f = open(log_path, 'w') + (log_path, f) = _init_task_run(taskid, argv, cwd) nullf = open(os.devnull, 'w') try: - subprocess.check_call(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f) + proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=f, stderr=f) + logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid)) + proc.wait() except subprocess.CalledProcessError, e: + logging.exception(e) + f.write("Failed: %r" % (e, )) f.close() shutil.move(log_path, _failed_path) raise e f.close() def spawn_sync_get_output(taskid, argv, cwd=None): - log_path = _init_task_run(taskid, argv, cwd) - f = open(log_path, 'w') + (log_path, f) = _init_task_run(taskid, argv, cwd) nullf = open(os.devnull, 'w') try: proc = subprocess.Popen(argv, cwd=cwd, stdin=nullf, stdout=subprocess.PIPE, stderr=f) + logging.info("Started subtask %s, pid=%d" % (taskid, proc.pid)) output = proc.communicate()[0] except subprocess.CalledProcessError, e: + logging.exception(e) + f.write("Failed: %r" % (e, )) f.close() shutil.move(log_path, _failed_path) raise e diff --git a/rpmci/versioned_repos.py b/rpmci/versioned_repos.py index 6de753c..6ee4b6c 100644 --- a/rpmci/versioned_repos.py +++ b/rpmci/versioned_repos.py @@ -34,7 +34,9 @@ class Repo(object): def add_rpm(self, rpm_path): basename = os.path.basename(rpm_path) - os.link(rpm_path, os.path.join(self._dir, basename)) + dest = os.path.join(self._dir, basename) + logging.debug("Linking %r to %r" % (rpm_path, dest)) + os.link(rpm_path, dest) def iter_rpms(self): for filename in os.listdir(self._dir): @@ -45,8 +47,9 @@ class Repo(object): output = subtask.spawn_sync_get_output('repomanage-' + self._task_basename, ['repomanage', '-o', '.'], cwd=dirpath) for line in output.split('\n'): - if line.endswith('.rpm') and os.path.exists(line): - os.unlink(line) + path = os.path.join(self._dir, line) + if line.endswith('.rpm') and os.path.exists(path): + os.unlink(path) def update_sync(self): self._delete_old_rpms_in_dir(self._dir) @@ -69,7 +72,7 @@ class VersionedRepo(object): def _reload(self): self._versions = {} for filename in os.listdir(self._dir): - path = os.path.join(self,_dir, filename) + path = os.path.join(self._dir, filename) if not os.path.isdir(path): continue try: @@ -80,11 +83,17 @@ class VersionedRepo(object): if revision > self._last_version: self._last_version = revision - def get_latest_path(self): + def get_latest_version(self): if self._last_version == -1: return None - return self._versions[self._last_version].get_dir() + return self._versions[self._last_version] + def get_latest_path(self): + latest = self.get_latest_version() + if latest is None: + return None + return latest.get_dir() + def commit_sync(self, new_rpms): rev = self._last_version + 1 new_repo = Repo(self._dir, self._name, rev) @@ -95,5 +104,7 @@ class VersionedRepo(object): for rpm in new_rpms: new_repo.add_rpm(rpm) new_repo.update_sync() + self._versions[rev] = new_repo + self._last_version = rev return rev -- cgit