Merge remote branch 'martins/tests'

(This used to be ctdb commit 71080adfe0828e15bf2f4b0bee7d52e06fff3449)

24 files changed, 586 insertions, 50 deletions

diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index ab7abb9fac..9003b396c2 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -1,5 +1,8 @@
 # Hey Emacs, this is a -*- shell-script -*- !!!  :-)
 
+# Print a message and exit.
+die () { echo "$@" >&2 ; exit 1 ; }
+
 # Augment PATH with relevant stubs/ directories.  We do this by actually
 # setting PATH, and also by setting $EVENTSCRIPTS_PATH and then
 # prepending that to $PATH in rc.local to avoid the PATH reset in
@@ -30,8 +33,7 @@ if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc" ] ; then
 elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc" ] ; then
     CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_DIR}/etc"
 else
-    echo "Unable to set \$CTDB_ETCDIR" >&2
-    exit 1
+    die "Unable to set \$CTDB_ETCDIR"
 fi
 export CTDB_ETCDIR
 
@@ -42,8 +44,7 @@ elif [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" ] ; then
 elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" ] ; then
     CTDB_BASE="${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb"
 else
-    echo "Unable to set \$CTDB_BASE" >&2
-    exit 1
+    die "Unable to set \$CTDB_BASE"
 fi
 export CTDB_BASE
 
@@ -109,7 +110,7 @@ tcp_port_down ()
 {
     for _i ; do
 	debug "Marking TCP port \"${_i}\" as not listening"
-	FAKE_NETSTAT_TCP_LISTEN=$(echo "$FAKE_NETSTAT_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
+	FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g")
     done
 }
 
@@ -177,6 +178,11 @@ ethtool_interfaces_up ()
     done
 }
 
+setup_nmap_output_filter ()
+{
+    OUT_FILTER="-e 's@^(DEBUG: # Nmap 5.21 scan initiated) .+ (as:)@\1 DATE \2@' -e 's@^(DEBUG: # Nmap done at) .+ (--)@\1 DATE \2@'"
+}
+
 ######################################################################
 
 # CTDB fakery
@@ -232,6 +238,10 @@ setup_ctdb ()
     export FAKE_CTDB_NODES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/nodes-down"
     mkdir -p "$FAKE_CTDB_NODES_DOWN"
     rm -f "$FAKE_CTDB_NODES_DOWN"/*
+
+    export FAKE_CTDB_SCRIPTSTATUS="$EVENTSCRIPTS_TESTS_VAR_DIR/scriptstatus"
+    mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
+    rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
 }
 
 
@@ -280,6 +290,25 @@ ctdb_get_1_public_address ()
     echo "${_addrs%% *}" | sed -r -e 's#(.*)/(.*)@(.*)#\3 \1 \2#g'
 }
 
+ctdb_not_implemented ()
+{
+    export CTDB_NOT_IMPLEMENTED="$1"
+    ctdb_not_implemented="\
+DEBUG: ctdb: command \"$1\" not implemented in stub"
+}
+
+ctdb_fake_scriptstatus ()
+{
+    _code="$1"
+    _status="$2"
+    _err_out="$3"
+
+    _d1=$(date '+%s.%N')
+    _d2=$(date '+%s.%N')
+
+    echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
+}
+
 ######################################################################
 
 # Samba fakery
@@ -303,8 +332,13 @@ setup_samba ()
 	export CTDB_SAMBA_SKIP_SHARE_CHECK="no"
 	export CTDB_MANAGED_SERVICES="foo samba winbind bar"
 
-	export FAKE_NETSTAT_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
+	export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139"
 	export FAKE_WBINFO_FAIL="no"
+
+	# Some things in 50.samba are backgrounded and waited for.  If
+	# we don't sleep at all then timeouts can happen.  This avoids
+	# that...  :-)
+	export FAKE_SLEEP_FORCE=0.1
     else
 	debug "Marking Samba services as down, not listening and not managed by CTDB"
         # Get into known state.
@@ -321,7 +355,7 @@ setup_samba ()
 	unset CTDB_MANAGES_SAMBA
 	unset CTDB_MANAGES_WINBIND
 
-	export FAKE_NETSTAT_TCP_LISTEN=""
+	export FAKE_TCP_LISTEN=""
 	export FAKE_WBINFO_FAIL="yes"
     fi
 
@@ -403,9 +437,7 @@ rpc_services_up ()
 	    rquotad)  _t="1:2" ;;
 	    nlockmgr) _t="3:4" ;;
 	    status)   _t="1:1" ;;
-	    *)
-		echo "Internal error - unsupported RPC service \"${_i}\""
-		exit 1
+	    *) die "Internal error - unsupported RPC service \"${_i}\"" ;;
 	esac
 
 	FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}"
@@ -533,8 +565,7 @@ Starting nfslock: OK"
 setup_vsftpd ()
 {
     if [ "$1" != "down" ] ; then
-	echo "setup_vsftpd up not implemented!!!"
-	exit 1
+	die "setup_vsftpd up not implemented!!!"
     else
 	debug "Setting up VSFTPD environment: service down, not managed by CTDB"
 
@@ -553,8 +584,7 @@ setup_vsftpd ()
 setup_httpd ()
 {
     if [ "$1" != "down" ] ; then
-	echo "setup_httpd up not implemented!!!"
-	exit 1
+	die "setup_httpd up not implemented!!!"
     else
 	debug "Setting up HTTPD environment: service down, not managed by CTDB"
 
@@ -578,16 +608,27 @@ define_test ()
     desc="$1"
 
     _f="$0"
-    _f="${_f#./}"      # strip leading ./
-    _f="${_f#simple/}" # strip leading simple/
-    _f="${_f%%/*}"     # if subdir, strip off file
-    _f="${_f%.sh}"     # strip off .sh suffix if any
+    _f="${_f#./}"          # strip leading ./
+    _f="${_f#simple/}"     # strip leading simple/
+    _f="${_f#multievent/}" # strip leading multievent/
+    _f="${_f%%/*}"         # if subdir, strip off file
+    _f="${_f%.sh}"         # strip off .sh suffix if any
 
-    # Remaining format should be NN.service.event.NNN:
+    # Remaining format should be NN.service.event.NNN or NN.service.NNN:
     _num="${_f##*.}"
     _f="${_f%.*}"
-    event="${_f##*.}"
-    script="${_f%.*}"
+    case "$_f" in
+	*.*.*)
+	    script="${_f%.*}"
+	    event="${_f##*.}"
+	    ;;
+	*.*)
+	    script="$_f"
+	    unset event
+	    ;;
+	*)
+	    die "Internal error - unknown testcase filename format"
+    esac
 
     printf "%-14s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
 }
@@ -637,25 +678,24 @@ result_print ()
 	if [ -n "$_iteration" ] ; then
 	    cat <<EOF
 
-##################################################
-##################################################
+==================================================
 Iteration $_iteration
 EOF
 	fi
 
 cat <<EOF
-##################################################
+--------------------------------------------------
 Output (Exit status: ${_rc}):
-##################################################
+--------------------------------------------------
 $_out
 EOF
     fi
 
     if ! $_passed ; then
 	cat <<EOF
-##################################################
+--------------------------------------------------
 Required output (Exit status: ${required_rc}):
-##################################################
+--------------------------------------------------
 $required_output
 EOF
     fi
@@ -668,20 +708,19 @@ result_footer ()
     if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then
 
 	cat <<EOF
-##################################################
+--------------------------------------------------
 CTDB_BASE="$CTDB_BASE"
 CTDB_ETCDIR="$CTDB_ETCDIR"
 ctdb client is "$(which ctdb)"
-##################################################
+--------------------------------------------------
 EOF
     fi
 
-    echo
-
     if $_passed ; then
 	echo "PASSED"
 	return 0
     else
+	echo
 	echo "FAILED"
 	return 1
     fi
@@ -699,11 +738,19 @@ EOF
 # useful for debugging.
 simple_test ()
 {
-    echo "Running \"${CTDB_BASE}/events.d/$script $event\""
+    [ -n "$event" ] || die 'simple_test: $event not set'
+
+    echo "Running \"$script $event${1:+ }$*\""
     _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
     _rc=$?
 
-    if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+    if [ -n "$OUT_FILTER" ] ; then
+	_fout=$(echo "$_out" | eval sed -r $OUT_FILTER)
+    else
+	_fout="$_out"
+    fi
+
+    if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
 	_passed=true
     else
 	_passed=false
@@ -713,6 +760,17 @@ simple_test ()
     result_footer "$_passed"
 }
 
+simple_test_event ()
+{
+    # If something has previously failed then don't continue.
+    : ${_passed:=true}
+    $_passed || return 1
+
+    event="$1" ; shift
+    echo "##################################################"
+    simple_test "$@"
+}
+
 # Run an eventscript iteratively.
 # - 1st argument is the number of iterations.
 # - 2nd argument is something to eval to do setup for every iteration.
@@ -733,6 +791,8 @@ simple_test ()
 # iteration.
 iterate_test ()
 {
+    [ -n "$event" ] || die 'simple_test: $event not set'
+
     args=""
     if [ "$1" = "--" ] ; then
 	shift
@@ -747,7 +807,7 @@ iterate_test ()
     _setup_default="$2"
     shift 2
 
-    echo "Running $_repeats iterations of \"${CTDB_BASE}/events.d/$script $event\" $args"
+    echo "Running $_repeats iterations of \"$script $event\" $args"
 
     _result=true
 
@@ -765,7 +825,13 @@ iterate_test ()
 	_out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1)
 	_rc=$?
 
-	if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+    if [ -n "$OUT_FILTER" ] ; then
+	_fout=$(echo "$_out" | eval sed -r $OUT_FILTER)
+    else
+	_fout="$_out"
+    fi
+
+	if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then
 	    _passed=true
 	else
 	    _passed=false
diff --git a/ctdb/tests/eventscripts/etc/sysconfig/ctdb b/ctdb/tests/eventscripts/etc/sysconfig/ctdb
new file mode 120000
index 0000000000..1c6e2b9f05
--- /dev/null
+++ b/ctdb/tests/eventscripts/etc/sysconfig/ctdb
@@ -0,0 +1 @@
+../../../../config/ctdb.sysconfig
\ No newline at end of file
diff --git a/ctdb/tests/eventscripts/multievent/10.interface.001.sh b/ctdb/tests/eventscripts/multievent/10.interface.001.sh
new file mode 100755
index 0000000000..5ef254c536
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/10.interface.001.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, removeip"
+
+setup_ctdb
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+simple_test_event "releaseip" $public_address
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.001.sh b/ctdb/tests/eventscripts/multievent/60.nfs.001.sh
new file mode 100755
index 0000000000..a6608db83e
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.001.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, ipreallocated -> reconfigure"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+EOF
+
+simple_test_event "ipreallocated"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.002.sh b/ctdb/tests/eventscripts/multievent/60.nfs.002.sh
new file mode 100755
index 0000000000..93f645a618
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.002.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+# This currently assumes that ctdb scriptstatus will always return a
+# good status (when replaying).  That should change and we will need
+# to split this into 2 tests.
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.003.sh b/ctdb/tests/eventscripts/multievent/60.nfs.003.sh
new file mode 100755
index 0000000000..929eaf2ae5
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.003.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure, replay error"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err="foo: bar error occurred"
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ctdb_fake_scriptstatus 1 "ERROR" "$err"
+
+required_result 1 <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+$err
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.004.sh b/ctdb/tests/eventscripts/multievent/60.nfs.004.sh
new file mode 100755
index 0000000000..addbf44c9c
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.004.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure, replay timedout"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err="waiting, waiting..."
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ctdb_fake_scriptstatus -62 "TIMEDOUT" "$err"
+
+required_result 1 <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+[Replay of TIMEDOUT scriptstatus - note incorrect return code.] $err
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.005.sh b/ctdb/tests/eventscripts/multievent/60.nfs.005.sh
new file mode 100755
index 0000000000..9e652013ca
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.005.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "takeip, monitor -> reconfigure, replay disabled"
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err=""
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ctdb_fake_scriptstatus -8 "DISABLED" "$err"
+
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+Replaying previous status for this script due to reconfigure...
+[Replay of DISABLED scriptstatus - note incorrect return code.] $err
+EOF
+
+simple_test_event "monitor"
diff --git a/ctdb/tests/eventscripts/multievent/60.nfs.006.sh b/ctdb/tests/eventscripts/multievent/60.nfs.006.sh
new file mode 100755
index 0000000000..35f8b836e1
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/60.nfs.006.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "reconfigure (synthetic), twice"
+# This checks that the lock is released...
+
+setup_nfs
+
+public_address=$(ctdb_get_1_public_address)
+
+err=""
+
+ok <<EOF
+Reconfiguring service "nfs"...
+Starting nfslock: OK
+Starting nfs: OK
+EOF
+
+simple_test_event "reconfigure"
+simple_test_event "reconfigure"
diff --git a/ctdb/tests/eventscripts/run_tests.sh b/ctdb/tests/eventscripts/run_tests.sh
index cb2c8f2950..ec17b5f7a7 100755
--- a/ctdb/tests/eventscripts/run_tests.sh
+++ b/ctdb/tests/eventscripts/run_tests.sh
@@ -15,6 +15,12 @@ for i ; do
 	    export EVENTSCRIPT_TESTS_VERBOSE="yes"
 	    shift
 	    ;;
+	-T)
+	    # This will cause tests to fail but is good for debugging
+	    # individual tests when they fail.
+	    export EVENTSCRIPTS_TESTS_TRACE="sh -x"
+	    shift
+	    ;;
 	-*)
 	    opts="$opts $i"
 	    shift
diff --git a/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh b/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
index fe2afd1d9f..0f8b1a53c6 100755
--- a/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
+++ b/ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh
@@ -8,4 +8,4 @@ setup_vsftpd "down"
 
 ok_null
 
-simple_test $cmd
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh b/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh
index c46a45c0c7..0a9ceac428 100755
--- a/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh
+++ b/ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh
@@ -8,4 +8,4 @@ setup_httpd "down"
 
 ok_null
 
-simple_test $cmd
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh
new file mode 100755
index 0000000000..e4151f3743
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, default tcp checker, debug"
+
+# This has to go before the setup, otherwise it will write a dud file.
+export CTDB_DEBUGLEVEL=4
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 <<EOF
+ERROR: samba tcp port 139 is not responding
+DEBUG: "ctdb checktcpport 139" was able to bind to port
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh
new file mode 100755
index 0000000000..d06a16f90a
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport not implemented"
+
+# TODO: create nmap stub
+export CTDB_NOT_IMPLEMENTED="checktcpport"
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 "ERROR: samba tcp port 139 is not responding"
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
new file mode 100755
index 0000000000..4bc854b623
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport not implemented, debug"
+
+ctdb_not_implemented "checktcpport"
+
+# This has to go before the setup, otherwise it will write a dud file.
+export CTDB_DEBUGLEVEL=4
+
+setup_nmap_output_filter
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 <<EOF
+DEBUG: ctdb_check_ports - checker ctdb not implemented
+DEBUG: output from checker was:
+DEBUG: ctdb checktcpport 445 (exited with 1) with output:
+$ctdb_not_implemented
+ERROR: samba tcp port 139 is not responding
+DEBUG: nmap -n -oG - -PS 127.0.0.1 -p 445,139 shows this output:
+DEBUG: # Nmap 5.21 scan initiated DATE as: nmap -n -oG - -PS 127.0.0.1 -p 445,139
+DEBUG: Host: 127.0.0.1 ()	Status: Up
+DEBUG: Host: 127.0.0.1 ()	Ports: 445/open/tcp//microsoft-ds///, 139/closed/tcp//netbios-ssn///
+DEBUG: # Nmap done at DATE -- 1 IP address (1 host up) scanned in 0.04 seconds
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
new file mode 100755
index 0000000000..80264cc24e
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport/nmap not implemented, debug"
+
+ctdb_not_implemented "checktcpport"
+export FAKE_NMAP_NOT_FOUND="yes"
+
+# This has to go before the setup, otherwise it will write a dud file.
+export CTDB_DEBUGLEVEL=4
+
+setup_nmap_output_filter
+
+setup_samba
+tcp_port_down 139
+
+required_result 1 <<EOF
+DEBUG: ctdb_check_ports - checker ctdb not implemented
+DEBUG: output from checker was:
+DEBUG: ctdb checktcpport 445 (exited with 1) with output:
+$ctdb_not_implemented
+DEBUG: ctdb_check_ports - checker nmap not implemented
+DEBUG: output from checker was:
+DEBUG: sh: nmap: command not found
+ERROR: samba tcp port 139 is not responding
+DEBUG: netstat -l -t -n shows this output:
+DEBUG: Active Internet connections (servers only)
+DEBUG: Proto Recv-Q Send-Q Local Address           Foreign Address         State
+DEBUG: tcp        0      0 0.0.0.0:445             0.0.0.0:*               LISTEN
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh b/ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh
new file mode 100755
index 0000000000..4651290663
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "port 139 down, ctdb checktcpport/nmap/netstat not implemented"
+
+ctdb_not_implemented "checktcpport"
+export FAKE_NMAP_NOT_FOUND="yes"
+export FAKE_NETSTAT_NOT_FOUND="yes"
+
+setup_nmap_output_filter
+
+setup_samba
+tcp_port_down 139
+
+required_result 127 <<EOF
+INTERNAL ERROR: ctdb_check_ports - no working checkers in CTDB_TCP_PORT_CHECKERS="ctdb nmap netstat"
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh
new file mode 100755
index 0000000000..4abe68eaba
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "2nd share missing"
+
+setup_nfs
+
+shares_missing "ERROR: nfs directory \"%s\" not available" 2
+
+required_result 1 "$MISSING_SHARES_TEXT"
+
+simple_test
diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh
new file mode 100755
index 0000000000..865b1c635a
--- /dev/null
+++ b/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "2nd share missing, skipping share checks"
+
+setup_nfs
+export CTDB_NFS_SKIP_SHARE_CHECK="yes"
+
+shares_missing "ERROR: nfs directory \"%s\" not available" 2
+
+ok_null
+
+simple_test
diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index ce06a35cbd..155f518ddc 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -2,6 +2,8 @@
 
 prog="ctdb"
 
+not_implemented_exit_code=1
+
 usage ()
 {
     cat >&2 <<EOF
@@ -9,26 +11,28 @@ Usage: $prog [-Y] cmd
 
 A fake CTDB stub that prints items depending on the variables
 FAKE_CTDB_PNN (default 0) depending on command-line options.
-
-Note that -Y is ignored.
-
 EOF
     exit 1
 }
 
-# $POSIXLY_CORRECT means that the command passed to onnode can take
-# options and getopt won't reorder things to make them options to this
-# script.
+not_implemented ()
+{
+    echo "${prog}: command \"$1\" not implemented in stub" >&2
+    exit $not_implemented_exit_code
+}
+
+# Don't set $POSIXLY_CORRECT here.
 _temp=$(getopt -n "$prog" -o "Yvh" -l help -- "$@") || \
     usage
 
 eval set -- "$_temp"
 
 verbose=false
+machine_readable=false
 
 while true ; do
     case "$1" in
-	-Y) shift ;;
+	-Y) machine_readable=true ; shift ;;
 	-v) verbose=true ; shift ;;
 	--) shift ; break ;;
 	-h|--help|*) usage ;; # * shouldn't happen, so this is reasonable.
@@ -51,6 +55,13 @@ setup_pstore ()
     mkdir -p "$pstore_dir"
 }
 
+# For testing backward compatibility...
+for i in $CTDB_NOT_IMPLEMENTED ; do
+    if [ "$i" = "$1" ] ; then
+	not_implemented "$i"
+    fi
+done
+
 case "$1" in
     ip)
 	# NOTE: all nodes share the same public addresses file.
@@ -159,8 +170,58 @@ case "$1" in
 		exit 1
 	esac
 	;;
+    getdebug)
+	case "${CTDB_DEBUGLEVEL:-0}" in
+	    -3) _t="EMERG"  ;;
+	    -2) _t="ALERT"  ;;
+	    -1) _t="CRIT"   ;;
+	    0) _t="ERR"     ;;
+	    1) _t="WARNING" ;;
+	    2) _t="NOTICE"  ;;
+	    3) _t="INFO"    ;;
+	    4) _t="DEBUG"   ;;
+	    *) _t="ERR"     ;;
+	esac
+
+	cat<<EOF
+:Name:Level:
+:${_t}:${CTDB_DEBUGLEVEL}:
+EOF
+	;;
+    checktcpport)
+	for _i in $FAKE_TCP_LISTEN ; do
+	    if [ "$2" = "${_i##*:}" ] ; then
+		exit 98
+	    fi
+	done
+
+	exit 0
+	;;
+    scriptstatus)
+	$machine_readable || not_implemented "$1, without -Y"
+	[ "$2" != "all" ] || not_implemented "scriptstatus all"
+	# For now just assume everything is good.
+	echo ":Type:Name:Code:Status:Start:End:Error Output...:"
+	for _i in "$CTDB_BASE/events.d/"*.* ; do
+	    _d1=$(date '+%s.%N')
+	    _b="${_i##*/}" # basename
+
+	    _f="$FAKE_CTDB_SCRIPTSTATUS/$_b"
+	    if [ -r "$_f" ] ; then
+		read _code _status _err_out <"$_f"
+	    else
+		_code="0"
+		_status="OK"
+		if [ ! -x "$_i" ] ; then
+		    _status="DISABLED"
+		    _code="-8"
+		fi
+		_err_out=""
+	    fi
+	    _d2=$(date '+%s.%N')
+	    echo ":${2:-monitor}:${_b}:${_code}:${_status}:${_d1}:${_d2}:${_err_out}:"
+	done
+	;;
     *)
-	echo "${prog}: command \"$1\" not implemented in stub"
-	exit 1
+	not_implemented "$1"
 esac
-
diff --git a/ctdb/tests/eventscripts/stubs/exportfs b/ctdb/tests/eventscripts/stubs/exportfs
index 72a11d5741..46c65225be 100755
--- a/ctdb/tests/eventscripts/stubs/exportfs
+++ b/ctdb/tests/eventscripts/stubs/exportfs
@@ -3,5 +3,11 @@
 opts="10.0.0.0/16(rw,async,insecure,no_root_squash,no_subtree_check)"
 
 for i in $FAKE_SHARES ; do
-    echo "${i} ${opts}"
+    # Directories longer than 15 characters are printed on their own
+    # line.
+    if [ ${#i} -ge 15 ] ; then
+	printf '%s\n\t\t%s\n' "$i" "$opts"
+    else
+	printf '%s\t%s\n' "$i" "$opts"
+    fi
 done
diff --git a/ctdb/tests/eventscripts/stubs/netstat b/ctdb/tests/eventscripts/stubs/netstat
index cf0656651a..f46c936519 100755
--- a/ctdb/tests/eventscripts/stubs/netstat
+++ b/ctdb/tests/eventscripts/stubs/netstat
@@ -2,13 +2,20 @@
 
 prog="netstat"
 
+# Pretty that we're the shell and that this command could not be
+# found.
+if [ "$FAKE_NETSTAT_NOT_FOUND" = "yes" ] ; then
+    echo "sh: ${prog}: command not found" >&2
+    exit 127
+fi
+
 usage ()
 {
     cat >&2 <<EOF
 Usage: $prog [ -t | --unix ] [ -n ] [ -a ] [ -l ]
 
 A fake netstat stub that prints items depending on the variables
-FAKE_NETSTAT_TCP_ESTABLISHED, FAKE_NETSTAT_TCP_LISTEN,
+FAKE_NETSTAT_TCP_ESTABLISHED, FAKE_TCP_LISTEN,
 FAKE_NETSTAT_UNIX_LISTEN, depending on command-line options.
 
 Note that -n is ignored.
@@ -73,7 +80,7 @@ if $tcp ; then
     done
 
     if $all || $listen ; then
-	for i in $FAKE_NETSTAT_TCP_LISTEN ; do
+	for i in $FAKE_TCP_LISTEN ; do
 	    printf "$tcp_fmt" $i "0.0.0.0:*" "LISTEN"
 	done
     fi
diff --git a/ctdb/tests/eventscripts/stubs/nmap b/ctdb/tests/eventscripts/stubs/nmap
new file mode 100755
index 0000000000..f01fe32d9c
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/nmap
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+prog="nmap"
+
+# Pretty that we're the shell and that this command could not be
+# found.
+if [ "$FAKE_NMAP_NOT_FOUND" = "yes" ] ; then
+    echo "sh: ${prog}: command not found" >&2
+    exit 127
+fi
+
+usage ()
+{
+    cat >&2 <<EOF
+Usage: $prog -n -oG - -PS 127.0.0.1 -p <port>[,<port> ...]
+
+A fake nmap stub that prints items depending on the variable
+FAKE_TCP_LISTEN and the ports specified.
+
+Note that all options apart from -p are ignored.
+
+EOF
+    exit 1
+}
+
+ports=""
+
+parse_options ()
+{
+    _temp=$(getopt -n "$prog" -a -o "np:" -l help -l PS: -l oG: -- "$@")
+
+    [ $? != 0 ] && usage
+
+    eval set -- "$_temp"
+
+    while true ; do
+	case "$1" in
+	    -n) shift ;;
+	    --oG|--PS) shift 2 ;;
+	    -p) ports="${ports}${ports:+ }${2//,/ }" ; shift 2 ;;
+	    --) shift ; break ;;
+	    -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable.
+	esac
+    done
+
+    [ $# -gt 0 ] && usage
+
+    [ -n "$ports" ] || usage
+}
+
+# For printing out...
+args="$*"
+
+parse_options "$@"
+
+port_states=""
+
+for p in $ports ; do
+    pn=$(getent services "$p" | sed -e 's@[[:space:]].*@@')
+    for i in $FAKE_TCP_LISTEN ; do
+	lp="${i##*:}"
+	if [ "$p" = "$lp" ] ; then
+	    port_states="${port_states}${port_states:+, }${p}/open/tcp//${pn}///"
+	    continue 2
+	fi
+    done
+    port_states="${port_states}${port_states:+, }${p}/closed/tcp//${pn}///"
+done
+
+cat <<EOF
+# Nmap 5.21 scan initiated $(date) as: nmap $args
+Host: 127.0.0.1 ()	Status: Up
+Host: 127.0.0.1 ()	Ports: $port_states
+# Nmap done at $(date) -- 1 IP address (1 host up) scanned in 0.04 seconds
+EOF
diff --git a/ctdb/tests/eventscripts/stubs/sleep b/ctdb/tests/eventscripts/stubs/sleep
new file mode 100755
index 0000000000..e4542444de
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/sleep
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if [ "$FAKE_SLEEP_REALLY" = "yes" ] ; then
+    /bin/sleep "$@"
+elif [ -n "$FAKE_SLEEP_FORCE" ] ; then
+    /bin/sleep "$FAKE_SLEEP_FORCE"
+else
+    :
+fi