tests/eventscripts: Unit tests for $CTDB_NFS_DUMP_STUCK_THREADS

Includes minor test infrastructure updates.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit cd4358b01c6c3d413b431f5760029d2b163b9c03)

5 files changed, 63 insertions, 1 deletions

diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.113.sh b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh
new file mode 100755
index 0000000000..caa49892a0
--- /dev/null
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "knfsd down, 6 iterations, dump 5 threads, none hung"
+
+# knfsd fails and attempts to restart it fail.
+setup_nfs
+rpc_services_down "nfs"
+
+# Additionally, any hung threads should have stack traces dumped.
+CTDB_NFS_DUMP_STUCK_THREADS=5
+FAKE_NFSD_THREAD_PIDS=""
+
+iterate_test 6 'ok_null' \
+    2 'rpc_set_service_failure_response "nfsd"' \
+    4 'rpc_set_service_failure_response "nfsd"' \
+    6 'rpc_set_service_failure_response "nfsd"'
diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.114.sh b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh
new file mode 100755
index 0000000000..8279395cde
--- /dev/null
+++ b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "knfsd down, 6 iterations, dump 5 threads, 3 hung"
+
+# knfsd fails and attempts to restart it fail.
+setup_nfs
+rpc_services_down "nfs"
+
+# Additionally, any hung threads should have stack traces dumped.
+CTDB_NFS_DUMP_STUCK_THREADS=5
+FAKE_NFSD_THREAD_PIDS="1001 1002 1003"
+
+iterate_test 6 'ok_null' \
+    2 'rpc_set_service_failure_response "nfsd"' \
+    4 'rpc_set_service_failure_response "nfsd"' \
+    6 'rpc_set_service_failure_response "nfsd"'
diff --git a/ctdb/tests/eventscripts/etc-ctdb/rc.local b/ctdb/tests/eventscripts/etc-ctdb/rc.local
index bc98f72c2b..9cd4d55136 100755
--- a/ctdb/tests/eventscripts/etc-ctdb/rc.local
+++ b/ctdb/tests/eventscripts/etc-ctdb/rc.local
@@ -36,6 +36,9 @@ get_proc ()
 	fs/nfsd/threads)
 	    echo "$FAKE_NFSD_THREAD_PIDS" | wc -w
 	    ;;
+	*/stack)
+	    echo "[<ffffffff87654321>] fake_stack_trace_for_pid_${1}+0x0/0xff"
+	    ;;
 	*)
 	    echo "get_proc: \"$1\" not implemented"
 	    exit 1
diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index cb08b30dce..6e2f15ce2e 100644
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -556,6 +556,7 @@ setup_nfs ()
     export CTDB_NFS_SKIP_SHARE_CHECK="no"
 
     export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS
+    export CTDB_NFS_DUMP_STUCK_THREADS
 
     # Reset the failcounts for nfs services.
     eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*'
@@ -675,7 +676,19 @@ program $_pn version $_ver is not available"
 			case "${_progname}${_action#restart}" in
 			    nfsd)
 				_t="\
-Trying to restart NFS service
+Trying to restart NFS service"
+
+				if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then
+				    for _pid in $FAKE_NFSD_THREAD_PIDS ; do
+					_t="\
+$_t
+Stack trace for stuck nfsd thread [${_pid}]:
+[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff"
+				    done
+				fi
+
+				_t="\
+${_t}
 Starting nfslock: OK
 Starting nfs: OK"
 				;;
diff --git a/ctdb/tests/eventscripts/stubs/pidof b/ctdb/tests/eventscripts/stubs/pidof
new file mode 100755
index 0000000000..b6ad6d83a8
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/pidof
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+case "$1" in
+    nfsd)
+	echo "$FAKE_NFSD_THREAD_PIDS"
+	;;
+    *)
+	echo "pidof: \"$1\" not implemented"
+	exit 1
+esac