Adding kprobes test scripts.

12 files changed, 923 insertions, 0 deletions

diff --git a/scripts/kprobes_test/.gitignore b/scripts/kprobes_test/.gitignore
new file mode 100644
index 00000000..2f836aac
--- /dev/null
+++ b/scripts/kprobes_test/.gitignore
@@ -0,0 +1,2 @@
+*~
+*.pyc
diff --git a/scripts/kprobes_test/README b/scripts/kprobes_test/README
new file mode 100644
index 00000000..c2c0fcb9
--- /dev/null
+++ b/scripts/kprobes_test/README
@@ -0,0 +1,57 @@
+Here are the original instructions for 'gen_whitelist', which tested
+systemtap probe safety.  This modified code tests the underlying
+kprobes themselves.
+
+To start a kprobes test run:
+   runtest whitelist.exp
+
+----
+Here is an implementation of generating whitelist for safe-mode probes
+based on the discussion in former thread
+(http://sourceware.org/ml/systemtap/2006-q3/msg00574.html).
+
+Its main idea is:
+ 1) Fetch a group of probe points from probes.pending, probe them and
+    run some workloads(e.g. runltp) parallely meanwhile.
+
+ 2) If the probe test ends without crash, those actually triggered
+    probe points are moved into probes.passed and those untriggered 
+    are into probes.untriggered; 
+    If the probe test crashes the system, it will be resumed
+    automatically after system reboot. Those probe points which have 
+    been triggered are also moved into probes.passed, but those 
+    untriggered ones are moved into probes.failed.
+
+ 3) Repeat the above until probes.pending becomes empty, then:
+    Normally, probes.pending is reinitialized from probes.failed and
+    probes.untriggered, then start the next iteration; 
+    But if max running level (now 3) is reached, or probes.pending,
+    probes.failed and probes.untriggered are all empty, stop the 
+    whole test.
+
+To be able to resume after a crash, this test will append its startup
+codes to the end of /etc/rc.d/rc.local at the beginning and remove 
+them at the end of the test.
+
+The gen_tapset_all.sh is used to generate a "probes.all" file based
+on current tapset of systemtap.
+
+It is suggested to use a script in a remote server to restart the 
+test machine automatically in case it is crashed by the probe test.
+
+Instructions:
+ 0) Please jump to 6) directly if you use all default settings
+ 1) Define your list of probe points in a file named "probes.all";
+    otherwise, it will be automatically extracted from 
+      stap -p2 -e 'probe kernel.function("*"){}
+ 2) Define your own workloads by changing the "benchs" list variable; 
+    otherwise, several LTP testcases will be used
+ 3) Define how may times you want the test to be iterated by changing
+    MAX_RUNNING_LEVEL variable; otherwise, 3 is the default
+ 4) Define the group size for different iteration level by changing 
+    proper_current_size() function; otherwise, they will be set in 
+    a decreasing order based on the length of probes.all
+ 5) Remove /stp_genwhitelist_running if you want to start a test from
+    the beginning after a system crash
+ 6) Start a test:
+      runtest whitelist.exp
diff --git a/scripts/kprobes_test/config_opts.py b/scripts/kprobes_test/config_opts.py
new file mode 100644
index 00000000..6163db50
--- /dev/null
+++ b/scripts/kprobes_test/config_opts.py
@@ -0,0 +1,32 @@
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+import os
+import sys
+
+# Here we set up the default config options.  These can be overridden
+# by the config file.
+config_opts = dict()
+
+# Various file names
+config_opts['probes_result'] = 'probe.out'
+config_opts['probes_all'] = 'probes.all'
+config_opts['probes_pending'] = 'probes.pending'
+config_opts['probes_current'] = 'probes.current'
+config_opts['probes_passed'] = 'probes.passed'
+config_opts['probes_failed'] = 'probes.failed'
+config_opts['probes_untriggered'] = 'probes.untriggered'
+config_opts['probes_unregistered'] = 'probes.unregistered'
+
+# Read in the config file
+print "Reading config file..."
+cfg = os.path.join(os.getcwd(), 'default.cfg')
+if os.path.exists(cfg):
+    execfile(cfg)
+else:
+    print >>sys.stderr, ("Could not find required config file: %s" % cfg)
+    sys.exit(1)
diff --git a/scripts/kprobes_test/default.cfg b/scripts/kprobes_test/default.cfg
new file mode 100644
index 00000000..c1c8acdb
--- /dev/null
+++ b/scripts/kprobes_test/default.cfg
@@ -0,0 +1,16 @@
+# -*-python-*-
+
+# Commands to run to generate a system load
+config_opts['load_cmds'] = [
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'syscalls', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'nfs', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'rpc', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'ipc', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'dio', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'fs', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'mm', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'tcp_cmds', ],
+    # [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'ltp-aio-stress.part1', ],
+    [ '/usr/local/ltp/runltp', '-t', '600s', '-f', 'lvm.part1' ],
+    ]
+
diff --git a/scripts/kprobes_test/gen_code.py b/scripts/kprobes_test/gen_code.py
new file mode 100755
index 00000000..1361f98e
--- /dev/null
+++ b/scripts/kprobes_test/gen_code.py
@@ -0,0 +1,80 @@
+#!/usr/bin/python
+
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+import os
+import sys
+from config_opts import config_opts
+
+def gen_files(dir, subset):
+    f = open('%s/kprobe_defs.h' % dir, 'w')
+
+    # Output the array of kp_data structs
+    print >>f, "static struct kp_data kp_data[] = {"
+    i = 0
+    while i < len(subset):
+        print >>f, ("\t{ .kp={ .symbol_name=\"%s\", .pre_handler=&handler_pre, }, .use_count=ATOMIC_INIT(0) },"
+                    % (subset[i]))
+        i += 1
+    print >>f, "};"
+    print >>f
+    f.close()
+
+    # Generate the Makefile
+    f = open('Makefile', 'w')
+    print >>f, """
+EXTRA_CFLAGS :=
+EXTRA_CFLAGS += -freorder-blocks
+EXTRA_CFLAGS += -Wno-unused -Werror
+obj-m := kprobe_module.o"""
+    f.close()
+
+def run_make_cmd(cmd):
+    # Before running make, fix up the environment a bit.  Clean out a
+    # few variables that /lib/modules/${KVER}/build/Makefile uses.
+    os.unsetenv("ARCH")
+    os.unsetenv("KBUILD_EXTMOD")
+    os.unsetenv("CROSS_COMPILE")
+    os.unsetenv("KBUILD_IMAGE")
+    os.unsetenv("KCONFIG_CONFIG")
+    os.unsetenv("INSTALL_PATH");
+
+    print "Running", cmd
+    return os.system(cmd)
+
+def gen_module():
+    f = open(config_opts['probes_current'])
+    probes = f.readlines()
+    f.close()
+    if len(probes) == 0:
+        print >>sys.stderr, ("Error: no probe points in %s"
+                             % config_opts['probes_current'])
+        return -1
+
+    # Cleanup each probe by stripping whitespace
+    i = 0
+    while i < len(probes):
+        probes[i] = probes[i].rstrip()
+        i += 1
+
+    # Generate necessary files
+    gen_files(os.getcwd(), probes)
+
+    # Try to build the module - add "V=1" at the end for more verbosity
+    os.system('rm -f ./kprobe_module.ko')
+    (sysname, nodename, release, version, machine) = os.uname()
+    cmd = ("make -C \"/lib/modules/%s/build\" M=\"%s\" modules >build.log 2>&1"
+           % (release, os.getcwd()))
+    rc = run_make_cmd(cmd)
+    if os.WEXITSTATUS(rc) != 0:
+        print >>sys.stderr, "Error: Make failed, see build.log for details"
+        return -1
+    return 0
+
+rc = gen_module()
+sys.exit(rc)
diff --git a/scripts/kprobes_test/is_probed.py b/scripts/kprobes_test/is_probed.py
new file mode 100755
index 00000000..192abd88
--- /dev/null
+++ b/scripts/kprobes_test/is_probed.py
@@ -0,0 +1,55 @@
+#!/usr/bin/python
+
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+import re
+import sys
+import os
+from config_opts import config_opts
+
+# Parse the output file, looking for probe points
+pp_re = re.compile(": (-?\d+) (\S+)$")
+f = open(config_opts['probes_result'], 'r')
+pp = dict()
+line = f.readline()
+while line:
+    match = pp_re.search(line)
+    if match:
+        pp[match.group(2)] =  int(match.group(1))
+    line = f.readline()
+f.close()
+
+if len(pp.keys()) == 0:
+    print >>sys.stderr, "No data found?"
+    sys.exit(1)
+
+# Parse the list of probe points.
+f = open(config_opts['probes_current'], 'r')
+passed = open(config_opts['probes_passed'], 'a')
+failed = open(config_opts['probes_failed'], 'a')
+untriggered = open(config_opts['probes_untriggered'], 'a')
+unregistered = open(config_opts['probes_unregistered'], 'a')
+line = f.readline().strip()
+while line:
+    if pp.has_key(line):
+        if pp[line] > 0:
+            passed.write(line + '\n')
+        elif pp[line] == 0:
+            untriggered.write(line + '\n')
+        elif pp[line] == -1:
+            unregistered.write(line + '\n')
+        else:
+            failed.write(line + '\n')
+    line = f.readline().strip()
+f.close()
+passed.close()
+failed.close()
+untriggered.close()
+unregistered.close()
+
+sys.exit(0)
diff --git a/scripts/kprobes_test/kprobe_module.c b/scripts/kprobes_test/kprobe_module.c
new file mode 100644
index 00000000..0ba6c1db
--- /dev/null
+++ b/scripts/kprobes_test/kprobe_module.c
@@ -0,0 +1,65 @@
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/kprobes.h>
+
+struct kp_data {
+	struct kprobe kp;
+	atomic_t use_count;
+};
+
+static int handler_pre(struct kprobe *p, struct pt_regs *regs);
+
+#include "kprobe_defs.h"
+
+
+/* kprobe pre_handler: called just before the probed instruction is executed */
+static int handler_pre(struct kprobe *p, struct pt_regs *regs)
+{
+	struct kp_data *k = container_of(p, struct kp_data, kp);
+	atomic_inc(&k->use_count);
+	return 0;
+}
+
+static int __init kprobe_init(void)
+{
+	int ret;
+	int probes_registered = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(kp_data); i++) {
+		ret = register_kprobe(&kp_data[i].kp);
+		if (ret != 0)
+			atomic_set(&kp_data[i].use_count, -1);
+		else
+			probes_registered++;
+	}
+	if (probes_registered == 0) {
+		for (i = 0; i < ARRAY_SIZE(kp_data); i++) {
+			printk(KERN_INFO "-1 %s\n", kp_data[i].kp.symbol_name);
+		}
+		printk(KERN_INFO "kprobe_module unloaded\n");
+		return ret;
+	}
+	printk(KERN_INFO "Planted kprobes\n");
+	return 0;
+}
+
+static void __exit kprobe_exit(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(kp_data); i++) {
+		if (atomic_read(&kp_data[i].use_count) != -1)
+			unregister_kprobe(&kp_data[i].kp);
+	}
+	printk(KERN_INFO "kprobes unregistered\n");
+	for (i = 0; i < ARRAY_SIZE(kp_data); i++) {
+		printk(KERN_INFO "%d %s\n", atomic_read(&kp_data[i].use_count),
+		       kp_data[i].kp.symbol_name);
+	}
+	printk(KERN_INFO "kprobe_module unloaded\n");
+}
+
+module_init(kprobe_init)
+module_exit(kprobe_exit)
+MODULE_LICENSE("GPL");
diff --git a/scripts/kprobes_test/monitor_system.py b/scripts/kprobes_test/monitor_system.py
new file mode 100755
index 00000000..59d49e7d
--- /dev/null
+++ b/scripts/kprobes_test/monitor_system.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+# This script monitors a remote system that is running the kprobes
+# test.  If several consecutive 'ping's fail, the system is rebooted.
+#
+# This script takes as an argument a config filename, whose contents
+# should look like the following:
+#
+#   config_opts['system_name'] = "SYSTEM_NAME"
+#   config_opts['restart_cmds'] = [
+#       'CMD1',
+#       'CMD2',
+#       ]
+#
+# As an example, here is a config file used when monitoring a kvm
+# instance:
+#
+#   config_opts['system_name'] = "dhcp-148"
+#   config_opts['restart_cmds'] = [
+#       'sudo virsh destroy kvm-rawhide-64-1',
+#       'sudo virsh start kvm-rawhide-64-1',
+#       ]
+
+import sys
+import os
+import time
+
+if len(sys.argv) != 2:
+    print >>sys.stderr, "Usage: %s config_file" % sys.argv[0]
+    sys.exit(1)
+cfg = sys.argv[1]
+
+# Read in the config file
+if not os.path.exists(cfg):
+    print >>sys.stderr, ("Could not find required config file: %s" % cfg)
+    sys.exit(1)
+
+print "Reading config file %s..." % cfg
+config_opts = dict()
+execfile(cfg)
+if not config_opts.has_key('system_name'):
+    print >>sys.stderr, "Missing required config opt 'system_name'"
+    sys.exit(1)
+if not config_opts.has_key('restart_cmds'):
+    print >>sys.stderr, "Missing required config opt 'restart_cmds'"
+    sys.exit(1)
+
+errors = 0
+while 1:
+    rc = os.system("ping -c 1 %s" % config_opts['system_name'])
+    # If ping worked, system is still up and running.  Wait a minute
+    # and try again.
+    if os.WEXITSTATUS(rc) == 0:
+        time.sleep(60)
+        errors = 0
+
+    # If the ping failed, increase the error count.  If we've got 3
+    # consecutive errors, assume the machine has crashed and restart
+    # it.
+    else:
+        errors += 1
+        if errors < 3:
+            time.sleep(30)
+        else:
+            print >>sys.stderr, "Restarting %s..." % config_opts['system_name']
+            # Run each restart command
+
+            for cmd in config_opts['restart_cmds']:
+                print >>sys.stderr, "Running '%s'..." % cmd
+                os.system(cmd)
+            # Sleep for 5 minutes to give the system a chance to boot
+            print >>sys.stderr, "Sleeping for 5 minutes..."
+            time.sleep(5 * 60)
+            errors = 0
+            
diff --git a/scripts/kprobes_test/readelf.py b/scripts/kprobes_test/readelf.py
new file mode 100755
index 00000000..d031625b
--- /dev/null
+++ b/scripts/kprobes_test/readelf.py
@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+import re
+import sys
+import os
+#import pickle
+import subprocess
+from config_opts import config_opts
+
+# Read the output of eu-readelf on vmlinux
+(sysname, nodename, release, version, machine) = os.uname()
+cmd = "eu-readelf --symbols /usr/lib/debug/lib/modules/%s/vmlinux" % release
+print "Running", cmd
+p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+lines = p.stdout.readlines()
+p.wait()
+if p.returncode != 0:
+    print >>sys.stderr, "Error: eu-readelf failed."
+    sys.exit(p.returncode)
+
+# Parse the output
+kprobes_text_start = 0
+kprobes_text_end = 0
+syms = dict()
+func_re = re.compile("^\s*\d+:\s+([0-9a-f]+)\s+\d+\s+FUNC\s+\S+\s+\S+\s+\d+\s+(\S+)$")
+notype_re = re.compile("^\s*\d+:\s+([0-9a-f]+)\s+\d+\s+NOTYPE\s+\S+\s+\S+\s+\d+\s+(\S+)$")
+for line in lines:
+    match = notype_re.match(line)
+    if match:
+        addr = match.group(1)
+        name = match.group(2)
+        if name == "__kprobes_text_start":
+            kprobes_text_start = long(addr, 16)
+        elif name == "__kprobes_text_end":
+            kprobes_text_end = long(addr, 16)
+        continue
+
+    match = func_re.match(line)
+    if match:
+        addr = match.group(1)
+        name = match.group(2)
+        syms[name] = long(addr, 16)
+
+# Now we've parsed everything.  Now we need to go back and remove all
+# symbols between '__kprobes_text_start' and '__kprobes_text_end',
+# since they are already protected from kprobes.  We couldn't do this
+# in the loop above, since we might encounter symbols that need to be
+# removed before we found the start/end of the kprobes section.
+if kprobes_text_start == 0 or kprobes_text_end == 0:
+    print "Error - didn't find kprobes_test_start(%d) or kprobes_text_end(%d)" \
+        % (kprobes_text_start, kprobes_text_end)
+    sys.exit(1)
+
+for name in syms.keys():
+    if syms[name] >= kprobes_text_start and syms[name] < kprobes_text_end:
+        print "deleting", name
+        del syms[name]
+
+## Save data
+#f = open('%s.syms' % (release), 'w')
+#p = pickle.Pickler(f)
+#p.dump(syms)
+#f.close()
+
+# Write the data out in text format
+f = open(config_opts['probes_all'], 'w')
+for name in syms.keys():
+    print >>f, name
+f.close()
diff --git a/scripts/kprobes_test/run_module.py b/scripts/kprobes_test/run_module.py
new file mode 100755
index 00000000..74465d94
--- /dev/null
+++ b/scripts/kprobes_test/run_module.py
@@ -0,0 +1,121 @@
+#!/usr/bin/python
+
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+import sys
+import os
+import os.path
+import time
+import select
+from config_opts import config_opts
+
+def run_module():
+    # Find the current size of /var/log/messages
+    logfile = '/var/log/messages'
+    print >>sys.stderr, "Getting size of %s..." % logfile
+
+    start_pos = os.path.getsize(logfile)
+    l = open(logfile, 'r')
+    l.seek(start_pos)
+
+    # Insert the module
+    print >>sys.stderr, "Inserting module..."
+    rc = os.system("/sbin/insmod %s" % os.path.join(os.getcwd(),
+                                                    'kprobe_module.ko'))
+    if os.WEXITSTATUS(rc) != 0:
+        # This is actually semi-OK, which is why there is no error
+        # message here, only a notice.  This might mean that every
+        # probe tried cannot be registered.
+        print >>sys.stderr, "Notice: insmod failed"
+    else:
+        # Run the load generate commands (if any).  Note we ignore the
+        # return values, since we don't really care if the commands
+        # succeed or not.
+        if config_opts.has_key('load_cmds'):
+            num_waits = 0
+
+            # Redirect the output to 'run_module.log' by modifying
+            # what stdout points to.
+            old_stdout = os.dup(sys.stdout.fileno())
+            fd = os.open('run_module.log', os.O_CREAT | os.O_WRONLY)
+            os.dup2(fd, sys.stdout.fileno())
+
+            # Run the commands.
+            for cmd in config_opts['load_cmds']:
+                pid = os.spawnvp(os.P_NOWAIT, cmd[0], cmd)
+                num_waits += 1
+            while num_waits > 0:
+                (pid, status) = os.waitpid(-1, 0)
+                num_waits -= 1
+
+            # Restore old value of stdout.
+            os.close(fd)
+            os.dup2(old_stdout, sys.stdout.fileno())
+
+        # Remove the module
+        print >>sys.stderr, "Removing module..."
+        rc = os.system("/sbin/rmmod kprobe_module")
+        if os.WEXITSTATUS(rc) != 0:
+            print >>sys.stderr, "Error: rmmod failed"
+            return -1
+
+    # Now we have to wait until everything is flushed to the logfile
+    f = open(config_opts['probes_result'], 'w')
+    while 1:
+        # Find the ending size of /var/log/messages
+        end_pos = os.path.getsize(logfile)
+
+        if end_pos < start_pos:
+            # The log files have been rotated.  Read any leftover data,
+            # then reopen file.
+            data = l.read()
+            if data:
+                f.write(data)
+
+                # See if we can find 'kprobe_module unloaded' in the
+                # data we just read.
+                if data.find('kprobe_module unloaded') != -1:
+                    break
+        
+            l.close()
+            l = open(logfile, 'r')
+            start_pos = 0
+            continue
+
+        # Try to wait until data is available
+        while 1:
+            try:
+                input, output, exc = select.select([l.fileno()], [], [], 60)
+                break
+            except select.error, err:
+                if err[0] != EINTR:
+                    raise
+
+        # Get the new stuff logged to /var/log/messages
+        data = l.read(end_pos - start_pos + 1)
+        if not data:
+            # ignore EOF
+            time.sleep(2)
+            continue
+
+        # Write results data
+        f.write(data)
+
+        # See if we can find 'kprobe_module unloaded' in the data we
+        # just read.
+        if data.find('kprobe_module unloaded') == -1:
+            start_pos = end_pos
+        else:
+            break
+
+    l.close()
+    f.close()
+    return 0
+
+rc = run_module()
+sys.exit(rc)
diff --git a/scripts/kprobes_test/whitelist.exp b/scripts/kprobes_test/whitelist.exp
new file mode 100755
index 00000000..37c1502c
--- /dev/null
+++ b/scripts/kprobes_test/whitelist.exp
@@ -0,0 +1,162 @@
+# Test kernel functions for kprobe safety.
+#
+# Copyright (C) 2006 IBM Corp.
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+# 
+# Based on software written by Gui Jian <guijian@cn.ibm.com>
+#
+
+load_lib "whitelist_lib.exp"
+
+set MAX_RUNNING_LEVEL 3
+
+set GENKPROBES_RUNNING "/genkprobes_running"
+set PROBES_RESULT "probe.out"
+set PROBES_ALL "probes.all"
+set PROBES_PENDING "probes.pending"
+set PROBES_CURRENT "probes.current"
+set PROBES_PASSED "probes.passed"
+set PROBES_FAILED "probes.failed"
+set PROBES_UNTRIGGERED "probes.untriggered"
+set PROBES_UNREGISTERED "probes.unregistered"
+
+proc proper_current_size {level inputfile} {
+     set totalsize [get_linesize $inputfile]
+     switch $level {
+         {1} {set currentsize [expr $totalsize/9]
+	     # Use a max size of 1000
+	     if {$currentsize > 1000} {
+		 set currentsize 1000
+	     }
+	     # Use a min size of 50
+	     if {$currentsize < 50} {
+		 set currentsize 50
+	     }
+	 }
+         {2} {set currentsize [expr $totalsize/49]
+	     # Use a max size of 400
+	     if {$currentsize > 400} {
+		 set currentsize 400
+	     }
+	     # Use a min size of 10
+	     if {$currentsize < 10} {
+		 set currentsize 10
+	     }
+	 }
+         {3} {set currentsize 100}
+         {4} {set currentsize 10}
+         default {
+              puts "Reached unexpected iteration level: $level"
+              set currentsize $totalsize
+         }
+     }
+     if {$currentsize <= 0} {
+         set currentsize 5
+     }
+     return $currentsize
+}
+
+#
+# Main routine of the whole test
+#
+if {[get_running_level] == 0} {
+    # Not started yet, start the whole test from the scratch
+    # Append the startup code to /etc/init.d/rc.local if not yet
+    register_service
+    # Check whether probes.all is empty or not given
+    if {[get_linesize $PROBES_ALL] == 0} {
+        init_probes_all
+    }
+    # Set current_running_level as 1 to indicate a new test started
+    init_running_level
+    # Initialize intermediate files based on probe.all
+    exec rm -f $PROBES_PENDING $PROBES_CURRENT
+    exec rm -f $PROBES_PASSED $PROBES_UNTRIGGERED $PROBES_FAILED $PROBES_UNREGISTERED
+    file copy $PROBES_ALL $PROBES_PENDING
+    exec touch $PROBES_PASSED $PROBES_UNTRIGGERED $PROBES_FAILED $PROBES_UNREGISTERED
+    sync_discs
+    puts "Start a fresh stp_genwhitelist test."
+} else {
+    # Maybe started already, so do some cleaning if necessary
+    garbage_collect $ALREADY_CRASHED
+    puts "Recovered from last maybe crashed probe test."
+}
+
+set current_size_const [proper_current_size [get_running_level] $PROBES_ALL]
+puts "current_size_const is initialized as $current_size_const"
+
+while {1} {
+    puts "Current size of probes.pending is [get_linesize $PROBES_PENDING]"
+    if {[get_linesize $PROBES_PENDING] == 0} {
+        # Check whether we need the next iteration or not
+        global MAX_RUNNING_LEVEL
+        # incr running_level for the start of a new iteration
+	set old_running_level [get_running_level]
+        incr_running_level
+        puts "Running level increased to [get_running_level]"
+        if {[get_running_level] > $MAX_RUNNING_LEVEL} {
+            puts "Reached max iteration limit: [get_running_level]"
+            break
+        } else {
+	    exec cp $PROBES_PASSED $PROBES_PASSED-pass$old_running_level
+	    exec cp $PROBES_FAILED $PROBES_FAILED-pass$old_running_level
+	    exec cp $PROBES_UNTRIGGERED $PROBES_UNTRIGGERED-pass$old_running_level
+
+            puts "Current running level is [get_running_level]"
+            exec rm -f $PROBES_PENDING
+            if {[get_linesize $PROBES_FAILED] > 0} {
+                # Append probes.failed to probes.pending
+                exec cat $PROBES_FAILED >> $PROBES_PENDING
+                file delete $PROBES_FAILED
+                exec touch $PROBES_FAILED
+                puts "Append $PROBES_FAILED to $PROBES_PENDING, now size=[get_linesize $PROBES_PENDING]"
+            } 
+            if {[get_linesize $PROBES_UNTRIGGERED] > 0} {
+                # Append probes.untriggered to probes.pending
+                exec cat $PROBES_UNTRIGGERED >> $PROBES_PENDING
+                file delete $PROBES_UNTRIGGERED
+                exec touch $PROBES_UNTRIGGERED
+                puts "Append $PROBES_UNTRIGGERED to $PROBES_PENDING, now size=[get_linesize $PROBES_PENDING]"
+            } 
+            if {[get_linesize $PROBES_PENDING] == 0} {
+                # No more pending probe points
+                puts "No more iterations needed. Stopped."
+                break
+            }
+            # set new value of current_size_const for new iteration level
+            set current_size_const [proper_current_size [get_running_level] $PROBES_ALL]
+            puts "current_size_const is set as $current_size_const now"
+            continue
+        }
+    } 
+    # Now, non-empty probes.pending should be ready
+    # Generate probes.current
+    exec rm -f $PROBES_CURRENT
+    exec head -n $current_size_const $PROBES_PENDING > $PROBES_CURRENT
+    exec tail -n+[expr $current_size_const+1] $PROBES_PENDING > /tmp/whitelist_tmpfile
+    exec mv /tmp/whitelist_tmpfile $PROBES_PENDING
+    sync_discs
+
+    set d [exec date]
+    puts "Ready to do current probe test at $d"
+    # Do actual probe test
+    do_current_test
+    set d [exec date]
+    puts "Completed one probe test successfully at $d"
+
+    # No crash fortunately, so do some cleaning to prepare for next test
+    garbage_collect $NO_CRASH
+}
+
+# Congratulations for arriving here
+# Remove all temporary files and unregister myself
+puts "Now removing all temporary files, unregistering the service and exit."
+exec rm -f $PROBES_PENDING $PROBES_CURRENT $GENKPROBES_RUNNING 
+exec rm -f $PROBES_RESULT /tmp/whitelist_tmpfile
+unregister_service
+exit 0
diff --git a/scripts/kprobes_test/whitelist_lib.exp b/scripts/kprobes_test/whitelist_lib.exp
new file mode 100755
index 00000000..122c320b
--- /dev/null
+++ b/scripts/kprobes_test/whitelist_lib.exp
@@ -0,0 +1,175 @@
+# Defines the supporting procedures used by whitelist.exp
+#
+# Copyright (C) 2006 IBM Corp.
+# Copyright (C) 2008 Red Hat Inc.
+# 
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+# 
+# Based on software written by Gui Jian <guijian@cn.ibm.com>
+#
+set NO_CRASH 0
+set ALREADY_CRASHED 1
+
+set startup_line_in_RCLOCAL "cd $env(PWD) && runtest whitelist.exp&"
+set RCLOCAL "/etc/rc.d/rc.local"
+
+set GROUP_NUM 1
+
+proc do_current_test {} {
+    global GROUP_NUM
+    set testname "L[get_running_level]_G$GROUP_NUM"
+    set GROUP_NUM [expr $GROUP_NUM + 1]
+
+    # Generate the C code
+    if {[catch {exec ./gen_code.py} results]} {
+	puts results
+	if {[lindex $::errorCode 0] eq "CHILDSTATUS"} {
+	    set status [lindex $::errorCode 2]
+	} else {
+	    # Some kind of unexpected failure
+	    set status -1
+	}
+	
+	if {$status != 0} {
+	    return;
+	}
+    }
+    wait
+    set module_path "[pwd]/kprobe_module.ko"
+    whitelist_run $testname $module_path
+}
+
+proc get_linesize {filename} {
+    if [file readable $filename] {
+      scan [exec wc -l $filename] "%d" lines
+      if {[info exists lines] && $lines > 0} {
+          return $lines
+      }
+    }
+    return 0
+}
+
+proc get_running_level {} {
+    global GENKPROBES_RUNNING
+    if [file readable $GENKPROBES_RUNNING] {
+        scan [exec cat $GENKPROBES_RUNNING] "%d" current_running_level
+        if {[info exists current_running_level] && $current_running_level > 0} {
+            return $current_running_level
+        }
+    }
+    return 0
+}
+
+proc sync_discs {} {
+    exec sync
+    exec sync
+    exec sync
+    exec sleep 5
+    return
+}
+
+# Notice throughout garbage_collect we make sure the discs are synced
+# before we return.  Otherwise, if the changes don't get flushed to
+# disk and the system crashes, we'll end up in an infinite loop of
+# crashes.
+proc garbage_collect {{already_crashed $NO_CRASH}} {
+    global PROBES_RESULT
+    global PROBES_CURRENT
+    global PROBES_PASSED
+    global PROBES_FAILED
+    global PROBES_UNTRIGGERED
+
+    if {[get_linesize $PROBES_CURRENT] == 0} {
+	puts "Empty $PROBES_CURRENT, returning"
+        exec rm -f $PROBES_CURRENT $PROBES_RESULT
+	sync_discs
+        return
+    }
+    if {[get_linesize $PROBES_RESULT] == 0} {
+	puts "Empty $PROBES_RESULT"
+        if {$already_crashed} {
+            exec cat $PROBES_CURRENT >> $PROBES_FAILED
+        } else {
+            exec cat $PROBES_CURRENT >> $PROBES_UNTRIGGERED
+        }
+        exec rm -f $PROBES_CURRENT $PROBES_RESULT
+	sync_discs
+        return
+    }
+    
+    # both probes.current and probe.out are non-empty
+    if [catch {open $PROBES_CURRENT r} Infile] {
+        puts "Failed to open $PROBES_CURRENT"
+        exec rm -f $PROBES_CURRENT $PROBES_RESULT
+	sync_discs
+        return
+    }
+
+    puts "Running ./is_probed.py"
+    exec ./is_probed.py
+
+    catch {close $Infile}
+    exec rm -f $PROBES_CURRENT $PROBES_RESULT
+    sync_discs
+    return
+}
+
+proc incr_running_level {} {
+    global GENKPROBES_RUNNING
+    set newlevel [expr [get_running_level]+1]
+    if { $newlevel > 0 } {
+        exec echo $newlevel > $GENKPROBES_RUNNING
+    } else {
+        exec echo 0 > $GENKPROBES_RUNNING
+    }
+    return
+}
+
+proc init_probes_all {} {
+    exec ./readelf.py
+}
+
+proc init_running_level {} {
+    global GENKPROBES_RUNNING
+    exec echo 1 > $GENKPROBES_RUNNING
+}
+
+proc register_service {} {
+    global startup_line_in_RCLOCAL
+    global RCLOCAL
+    exec sed -i -n -e "/runtest whitelist.exp/!p" $RCLOCAL
+    exec echo $startup_line_in_RCLOCAL >> $RCLOCAL
+}
+
+proc unregister_service {} {
+    global RCLOCAL
+    exec sed -i -n -e "/runtest whitelist.exp/!p" $RCLOCAL
+}
+
+proc whitelist_run { TEST_NAME MODULE } {
+    if {[info procs installtest_p] != "" && ![installtest_p]} { 
+        untested $TEST_NAME; 
+        return;
+    }
+
+    set status 0
+    if {[catch {exec ./run_module.py} results]} {
+	puts $results
+	if {[lindex $::errorCode 0] eq "CHILDSTATUS"} {
+	    set status [lindex $::errorCode 2]
+	} else {
+	    # Some kind of unexpected failure
+	    set status -1
+	}
+    }
+    if {$status != 0} {
+	fail $TEST_NAME;
+	return;
+    }
+    pass "$TEST_NAME end successfully";
+
+    wait
+}