retrace server: add more logic to cleanup script

2 files changed, 115 insertions, 29 deletions

diff --git a/src/retrace/abrt-retrace-cleanup b/src/retrace/abrt-retrace-cleanup
index 9286889f..cbe1b818 100755
--- a/src/retrace/abrt-retrace-cleanup
+++ b/src/retrace/abrt-retrace-cleanup
@@ -1,7 +1,6 @@
 #!/usr/bin/python
 
 import os
-import shutil
 import sys
 import time
 from retrace import *
@@ -13,31 +12,49 @@ if __name__ == "__main__":
 
     try:
         log = open(logfile, "a")
-        log.write(time.strftime("[%Y-%m-%d %H:%M:%S] Running cleanup\n"))
-
-        files = os.listdir(CONFIG["WorkDir"])
     except IOError, ex:
         print "Error opening log file: %s" % ex
         sys.exit(1)
+
+    log.write(time.strftime("[%Y-%m-%d %H:%M:%S] Running cleanup\n"))
+
+    # kill tasks running > 1 hour
+    ps_output = run_ps()
+    running_tasks = get_running_tasks(ps_output)
+    for pid, taskid, runtime in running_tasks:
+        # ToDo: 5 = mm:ss, >5 = hh:mm:ss
+        if len(runtime) > 5:
+            log.write("Killing task %d running for %s\n" % (taskid, runtime))
+            kill_process_and_childs(pid, ps_output)
+
+    # kill orphaned tasks
+    running_tasks = get_running_tasks()
+    running_ids = []
+    for pid, taskid, runtime in running_tasks:
+        running_ids.append(taskid)
+
+    for task in get_active_tasks():
+        if not task in running_ids:
+            log.write("Cleaning up orphaned task %d\n" % task)
+            cleanup_task(task)
+
+    # clean up old tasks
+    try:
+        files = os.listdir(CONFIG["SaveDir"])
     except OSError, ex:
-        log.write("Unable to list work directory: %s" % ex)
-        sys.exit(2)
+        files = []
+        log.write("Error listing task directory: %s\n" % ex)
 
     for filename in files:
-        filepath = "%s/%s" % (CONFIG["WorkDir"], filename)
-        if os.path.isdir(filepath):
-            try:
-                if (now - os.path.getctime(filepath)) / 3600 >= CONFIG["DeleteTaskAfter"]:
-                    log.write("Deleting directory '%s'\n" % filepath)
-                    shutil.rmtree(filepath)
-            except OSError, ex:
-                log.write("Error deleting directory: %s\n" % (filepath, ex))
-            except IOError, ex:
-                print "Unable to write to log file: %s" % ex
-                sys.exit(3)
+        try:
+            taskid = int(filename)
+        except:
+            continue
 
-    try:
-        log.close()
-    except IOError, ex:
-        print "Error closing log file: %s" % ex
-        sys.exit(4)
+        dirpath = "%s/%s" % (CONFIG["SaveDir"], filename)
+        if os.path.isdir(dirpath) and \
+           (now - os.path.getatime(dirpath)) / 3600 >= CONFIG["DeleteTaskAfter"]:
+            log.write("Deleting old task %s\n" % filename)
+            call(["rm", "-rf", dirpath])
+
+    log.close()
diff --git a/src/retrace/retrace.py b/src/retrace/retrace.py
index 33488bba..dfebcb62 100644
--- a/src/retrace/retrace.py
+++ b/src/retrace/retrace.py
@@ -22,13 +22,7 @@ PACKAGE_PARSER = re.compile("^(.+)-([0-9]+(\.[0-9]+)*-[0-9]+)\.([^-]+)$")
 DF_OUTPUT_PARSER = re.compile("^([^ ^\t]*)[ \t]+([0-9]+)[ \t]+([0-9]+)[ \t]+([0-9]+)[ \t]+([0-9]+%)[ \t]+(.*)$")
 DU_OUTPUT_PARSER = re.compile("^([0-9]+)")
 URL_PARSER = re.compile("^/([0-9]+)/?")
-RELEASE_PARSERS = {
-  "fedora": re.compile("^Fedora[^0-9]+([0-9]+)[^\(]\(([^\)]+)\)$"),
-}
-
-GUESS_RELEASE_PARSERS = {
-  "fedora": re.compile("\.fc([0-9]+)"),
-}
+WORKER_RUNNING_PARSER = re.compile("^([0-9]+)[ \t]+[0-9]+[ \t]+([^ ^\t]+)[ \t]+.*abrt-retrace-worker ([0-9]+)$")
 
 HANDLE_ARCHIVE = {
   "application/x-xz-compressed-tar": {
@@ -260,6 +254,81 @@ def get_active_tasks():
 
     return tasks
 
+def run_ps():
+    pipe = Popen(["ps", "-eo", "pid,ppid,etime,cmd"], stdout=PIPE).stdout
+    lines = pipe.readlines()
+    pipe.close()
+
+    return lines
+
+def get_running_tasks(ps_output=None):
+    if not ps_output:
+        ps_output = run_ps()
+
+    result = []
+
+    for line in ps_output:
+        match = WORKER_RUNNING_PARSER.match(line)
+        if match:
+            result.append((int(match.group(1)), int(match.group(3)), match.group(2)))
+
+    return result
+
+def get_process_tree(pid, ps_output):
+    result = [pid]
+
+    parser = re.compile("^([0-9]+)[ \t]+(%d).*$" % pid)
+
+    for line in ps_output:
+        match = parser.match(line)
+        if match:
+            pid = int(match.group(1))
+            result.extend(get_process_tree(pid, ps_output))
+
+    return result
+
+def kill_process_and_childs(process_id, ps_output=None):
+    result = True
+
+    if not ps_output:
+        ps_output = run_ps()
+
+    for pid in get_process_tree(process_id, ps_output):
+        try:
+            os.kill(pid, 9)
+        except OSError, ex:
+            result = False
+
+    return result
+
+def cleanup_task(taskid):
+    null = open("/dev/null", "w")
+
+    savedir = "%s/%d" % (CONFIG["SaveDir"], taskid)
+    if os.path.isfile("%s/mock.cfg" % savedir):
+        call(["mock", "-r", "../../%s/mock" % savedir, "--scrub=all"],
+             stdout=null, stderr=null)
+
+    call(["rm", "-rf", "%s/crash" % savedir, "%s/mock.cfg" % savedir],
+         stdout=null, stderr=null)
+
+    rawlog = "%s/log" % savedir
+    newlog = "%s/retrace_log" % savedir
+    if os.path.isfile(rawlog):
+        try:
+            os.rename(rawlog, newlog)
+        except:
+            pass
+
+    try:
+        log = open(newlog, "a")
+        log.write("Killed by garbage collector\n")
+        log.close()
+    except:
+        pass
+
+    null.close()
+
 def init_crashstats_db():
     try:
         con = sqlite3.connect("%s/%s" % (CONFIG["SaveDir"], CONFIG["DBFile"]))