diff options
| author | Martin Schwenke <martin@meltin.net> | 2014-11-14 13:59:16 +1100 |
|---|---|---|
| committer | Amitay Isaacs <amitay@samba.org> | 2014-11-18 04:17:10 +0100 |
| commit | 968401ccdc217d0addb6235739b84dbb9d23e651 (patch) | |
| tree | 55f6fde1e0bc421504715eaa354c56f0ab936463 | |
| parent | 1f49e1ab5b317812c0ad482404fb224368726846 (diff) | |
| download | samba-968401ccdc217d0addb6235739b84dbb9d23e651.tar.gz samba-968401ccdc217d0addb6235739b84dbb9d23e651.tar.xz samba-968401ccdc217d0addb6235739b84dbb9d23e651.zip | |
ctdb-scripts: Dump stack traces for hung mountd, rquotad, statd processes
Add a corresponding new unit test for statd.
Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
| -rwxr-xr-x | ctdb/config/functions | 3 | ||||
| -rwxr-xr-x | ctdb/tests/eventscripts/60.nfs.monitor.143.sh | 15 | ||||
| -rw-r--r-- | ctdb/tests/eventscripts/scripts/local.sh | 10 | ||||
| -rwxr-xr-x | ctdb/tests/eventscripts/stubs/pidof | 3 |
4 files changed, 30 insertions, 1 deletions
diff --git a/ctdb/config/functions b/ctdb/config/functions index 377bc4ddf2..49d09f8540 100755 --- a/ctdb/config/functions +++ b/ctdb/config/functions @@ -402,11 +402,13 @@ _nfs_restart_rpc_service () mountd) echo "Trying to restart $_prog_name [${_p}]" killall -q -9 "$_p" + nfs_dump_some_threads "$_p" $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT ;; rquotad) echo "Trying to restart $_prog_name [${_p}]" killall -q -9 "$_p" + nfs_dump_some_threads "$_p" $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT ;; lockd) @@ -416,6 +418,7 @@ _nfs_restart_rpc_service () statd) echo "Trying to restart $_prog_name [${_p}]" killall -q -9 "$_p" + nfs_dump_some_threads "$_p" $_maybe_background $_p \ ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \ ${STATD_PORT:+-p} $STATD_PORT \ diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.143.sh b/ctdb/tests/eventscripts/60.nfs.monitor.143.sh new file mode 100755 index 0000000000..732de9c394 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.143.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "statd down, 2 iterations, stuck process" + +# statd fails and the first attempt to restart it succeeds. + +setup_nfs +rpc_services_down "status" +CTDB_NFS_DUMP_STUCK_THREADS=2 +FAKE_RPC_THREAD_PIDS="1001" + +iterate_test 2 'ok_null' \ + 2 'rpc_set_service_failure_response "statd"' diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index 2c5738eefa..cea6c4daf8 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -713,7 +713,7 @@ setup_nfs () export CTDB_NFS_SKIP_SHARE_CHECK="no" export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS - export CTDB_NFS_DUMP_STUCK_THREADS + export CTDB_NFS_DUMP_STUCK_THREADS FAKE_RPC_THREAD_PIDS # Reset the failcounts for nfs services. eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' @@ -878,6 +878,14 @@ ${_bg}Starting nfslock: OK" ;; *) _t="Trying to restart $_progname [${_p}]" + if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then + for _pid in $FAKE_RPC_THREAD_PIDS ; do + _t="\ +$_t +Stack trace for ${_p}[${_pid}]: +[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff" + done + fi esac _out="${_out}${_out:+${_nl}}${_t}" ;; diff --git a/ctdb/tests/eventscripts/stubs/pidof b/ctdb/tests/eventscripts/stubs/pidof index b6ad6d83a8..b84aef0acd 100755 --- a/ctdb/tests/eventscripts/stubs/pidof +++ b/ctdb/tests/eventscripts/stubs/pidof @@ -4,6 +4,9 @@ case "$1" in nfsd) echo "$FAKE_NFSD_THREAD_PIDS" ;; + rpc.statd|rpc.rquotad|rpc.mountd) + echo "$FAKE_RPC_THREAD_PIDS" + ;; *) echo "pidof: \"$1\" not implemented" exit 1 |