diff options
Diffstat (limited to 'rpmci/rpmci_vcs_mirror_main.py')
| -rw-r--r-- | rpmci/rpmci_vcs_mirror_main.py | 38 |
1 files changed, 30 insertions, 8 deletions
diff --git a/rpmci/rpmci_vcs_mirror_main.py b/rpmci/rpmci_vcs_mirror_main.py index 85615a3..f8a78ee 100644 --- a/rpmci/rpmci_vcs_mirror_main.py +++ b/rpmci/rpmci_vcs_mirror_main.py @@ -24,10 +24,12 @@ from . import msgqueue from . import lame_vcs_abstraction class VCSMirror(object): - def __init__(self, config, urls): + def __init__(self, options, config, urls): self.config = config self.urls = urls + self._options = options + self._dir = config.get('VCS', 'mirror_dir') self._process_dir = config.get('VCS', 'process_logdir') if not os.path.isdir(self._process_dir): @@ -59,7 +61,7 @@ class VCSMirror(object): return vcs.get_url().netloc def _escape_vcs_url(self, vcs): - return urllib.quote(vcs.get_base_url_string(), '') + return urllib.quote(vcs.get_url_string(), '') def _cachedir_for_vcs(self, vcs): return os.path.join(self._dir, self._escape_vcs_url(vcs)) @@ -105,13 +107,14 @@ class VCSMirror(object): current_id = vcs.get_id() if current_id != previous_id: logging.info("vcs %r: New commit id %r differs from previous %r" % (vcs, current_id, previous_id)) - msg = msgqueue.Message(None, {'type': 'update'}, {'id': current_id}) + msg = msgqueue.Message(None, {'type': 'update'}, {'url': vcs.get_url_string(), 'id': current_id}) self._msgqueue.append(msg) else: logging.info("No changes in %r from previous commit id %r" % (vcs, previous_id)) - target_time = int(time.time() + self._timeout_seconds) - self._vcs_queue.append((vcs, target_time)) + if not failed and not self._options.clone_then_exit: + target_time = int(time.time() + self._timeout_seconds) + self._vcs_queue.append((vcs, target_time)) self._poll() @@ -144,13 +147,14 @@ class VCSMirror(object): glib.source_remove(self._active_queue_timeout_id) self._active_queue_timeout_seconds = timeout self._active_queue_timeout_id = glib.timeout_add_seconds(timeout, self._poll) - def _poll(self): current_time = int(time.time()) - logging.info("Doing poll (%d active tasks)" % (self._num_active_jobs(), )) + orig_active_jobs = self._num_active_jobs() + logging.debug("Queue: %r" % (self._vcs_queue, )) + processed = [] for vcs, target_time in self._vcs_queue: active = self._job_for_vcs(vcs) @@ -177,7 +181,12 @@ class VCSMirror(object): previous_id = None logging.info("Doing initial checkout for %r" % (vcs.get_base_url_string(), )) vcs_tempdir = vcsdir + '.tmp' + if os.path.isdir(vcs_tempdir): + shutil.rmtree(vcs_tempdir) process = vcs.checkout_async(vcs_tempdir, job_logpath, self._on_job_exited) + elif self._options.clone_then_exit: + processed.append(vcs) + continue else: vcs.set_directory(vcsdir) previous_id = vcs.get_id() @@ -186,6 +195,8 @@ class VCSMirror(object): process.__vcs = vcs self._jobs_by_host[host].append((process, previous_id)) processed.append(vcs) + + added_job_count = len(processed) while processed: vcs = processed[0] del processed[0] @@ -196,8 +207,18 @@ class VCSMirror(object): break assert index >= 0 del self._vcs_queue[index] + + new_active_jobs = self._num_active_jobs() + + if len(self._vcs_queue) == 0 and new_active_jobs == 0: + logging.info("Queue is empty and no active jobs. Exiting.") + sys.exit(0) self._adjust_timeout() + if new_active_jobs == 0: + logging.info("No active jobs; sleeping for %d seconds" % (self._active_queue_timeout_seconds, )) + else: + logging.info("Poll complete, started %d jobs (%d total)" % (new_active_jobs - orig_active_jobs, new_active_jobs)) return False @@ -208,6 +229,7 @@ def main(): opts = optparse.OptionParser("usage: %prog [options]") opts.add_option('-c', '--config', dest='config', help="Path to configuration file") opts.add_option('', '--debug', action='store_true', help="Print verbose debugging") + opts.add_option('', '--clone-then-exit', action='store_true', help="If true, perform any necessary clones, then exit") (options, args) = opts.parse_args() @@ -232,7 +254,7 @@ def main(): urls = f.readlines() f.close() - mirror = VCSMirror(config, urls) + mirror = VCSMirror(options, config, urls) mirror.start() loop = glib.MainLoop() |