New test 09_ctdb_ping.sh. Add documentation and command-line

processing to all tests.  New script ctdb_test_env sets up environment
for tests, is now sourced by run_tests, and can also take a test on
the command-line, complete with options.  Various cleanups and
improvements.  Document tests that have been properly implemented in
ctdbd.sh.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 826e85fe5291067b8d0b9c22918d63024aa6141c)

20 files changed, 560 insertions, 95 deletions

diff --git a/ctdb/tests/ctdbd.sh b/ctdb/tests/ctdbd.sh
index d2b7405594..ba43abe48c 100755
--- a/ctdb/tests/ctdbd.sh
+++ b/ctdb/tests/ctdbd.sh
@@ -5,10 +5,10 @@ killall -q ctdbd
 tests/start_daemons.sh 2 || exit 1
 
 echo "Testing ping"
-$VALGRIND bin/ctdb ping || exit 1
+$VALGRIND bin/ctdb ping || exit 1  # Done
 
 echo "Testing status"
-$VALGRIND bin/ctdb status || exit 1
+$VALGRIND bin/ctdb status || exit 1 # Done: implied
 
 echo "Testing statistics"
 $VALGRIND bin/ctdb -n all statistics || exit 1
@@ -30,18 +30,18 @@ echo "Testing getdbmap"
 $VALGRIND bin/ctdb getdbmap || exit 1
 
 echo "Testing status"
-$VALGRIND bin/ctdb status || exit 1
+$VALGRIND bin/ctdb status || exit 1 # Done: implied
 
 echo "Testing variables"
 $VALGRIND bin/ctdb listvars || exit 1
-$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
-$VALGRIND bin/ctdb setvar TraverseTimeout 10 || exit 1
-$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
+$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1  # Done
+$VALGRIND bin/ctdb setvar TraverseTimeout 10 || exit 1 # Done
+$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1 # Done
 
 sleep 1
 
 echo "Testing shutdown"
-$VALGRIND bin/ctdb shutdown -n all || exit 1
+$VALGRIND bin/ctdb shutdown -n all || exit 1 # Done: implied by 09_ctdb_ping.sh
 
 sleep 1
 
diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env
new file mode 100755
index 0000000000..4fa3e30f79
--- /dev/null
+++ b/ctdb/tests/scripts/ctdb_test_env
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd)
+var_dir=$CTDB_DIR/tests/var
+
+export CTDB_NUM_NODES=3
+
+######################################################################
+
+if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+    export CTDB_NODES_SOCKETS=""
+    for i in $(seq 1 $CTDB_NUM_NODES) ; do
+	CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}"
+    done
+fi
+
+######################################################################
+
+ctdb_bin_dir=$CTDB_DIR/bin
+ctdb_tools_dir=$CTDB_DIR/tools
+ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd)
+
+PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}"
+
+export TEST_WRAP="${ctdb_test_scripts_dir}/test_wrap"
+
+"$@"
diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash
index 3713c98e49..bc660dbc23 100644
--- a/ctdb/tests/scripts/ctdb_test_functions.bash
+++ b/ctdb/tests/scripts/ctdb_test_functions.bash
@@ -10,7 +10,7 @@ fail ()
 
 #. /root/SOFS/autosofs/scripts/tester.bash
 
-test_begin ()
+ctdb_test_begin ()
 {
     local name="$1"
 
@@ -22,7 +22,7 @@ test_begin ()
     echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--"
 }
 
-test_end ()
+ctdb_test_end ()
 {
     local name="$1" ; shift
     local status="$1" ; shift
@@ -50,33 +50,102 @@ test_end ()
 
 }
 
-
-test_exit() {
+test_exit ()
+{
     exit $(($testfailures+0))
 }
 
-test_run ()
+ctdb_test_exit ()
+{
+    if ! onnode 0 $TEST_WRAP cluster_is_healthy ; then
+	echo "Restarting ctdb on all nodes to get back into known state..."
+	restart_ctdb
+    fi
+
+    test_exit
+}
+
+ctdb_test_run ()
 {
     local name="$1" ; shift
     
     [ -n "$1" ] || set -- "$name"
 
-    test_begin "$name"
+    ctdb_test_begin "$name"
 
     local status=0
     "$@" || status=$?
 
-    test_end "$name" "$status" "$*"
+    ctdb_test_end "$name" "$status" "$*"
     
     return $status
 }
 
+ctdb_test_usage()
+{
+    local status=${1:-2}
+    
+    cat <<EOF
+Usage: $0 [option]
+
+Options:	
+    -h, --help          show this screen.
+    -v, --version       show test case version.
+    --category          show the test category (ACL, CTDB, Samba ...).
+    -d, --description   show test case description.
+    --summary           show short test case summary.
+EOF
+
+    exit $status
+}
+
+ctdb_test_version ()
+{
+    [ -n "$CTDB_DIR" ] || fail "Can not determine version."
+
+    (cd "$CTDB_DIR" && git describe)
+}
+
+ctdb_test_cmd_options()
+{
+    [ -n "$1" ] || return 0
+
+    case "$1" in
+        -h|--help)        ctdb_test_usage 0   ;;
+        -v|--version)     ctdb_test_version   ;;
+        --category)       echo "CTDB"         ;; 
+        -d|--description) test_info           ;;
+	--summary)        test_info | head -1 ;;
+	*)
+	    echo "Error: Unknown parameter = $1"
+	    echo
+	    ctdb_test_usage 2
+	    ;;
+    esac
+
+    exit 0
+}
+
+ctdb_test_init () 
+{
+    scriptname=$(basename "$0")
+    testfailures=0
+
+    ctdb_test_cmd_options $@
+}
+
 ########################################
 
 # Sets: $out
 try_command_on_node ()
 {
     local nodespec="$1" ; shift
+
+    local verbose=false
+    if [ "$nodespec" = "-v" ] ; then
+	verbose=true
+	nodespec="$1" ; shift
+    fi
     local cmd="$*"
 
     out=$(onnode -q "$nodespec" "$cmd" 2>&1) || {
@@ -85,12 +154,17 @@ try_command_on_node ()
 	echo "$out"
 	exit 1
     }
+
+    if $verbose ; then
+	echo "Output of \"$cmd\":"
+	echo "$out"
+    fi
 }
 
 sanity_check_output ()
 {
     local min_lines="$1"
-    local regexp="$2" # Should be anchored to match whole lines.
+    local regexp="$2" # Should be anchored as necessary.
     local output="$3"
 
     local ret=0
@@ -109,7 +183,7 @@ sanity_check_output ()
     # Note that this is reversed.
     if [ $status -eq 0 ] ; then
 	echo "BAD: unexpected lines in output:"
-	echo "$unexpected"
+	echo "$unexpected" | cat -A
 	ret=1
     else
 	echo "Output lines look OK"
@@ -126,7 +200,7 @@ wait_until ()
 {
     local timeout="$1" ; shift # "$@" is the command...
 
-    echo -n "|${timeout}|"
+    echo -n "<${timeout}|"
     while [ $timeout -gt 0 ] ; do
 	if "$@" ; then
 	    echo '|'
@@ -145,7 +219,7 @@ wait_until ()
 
 sleep_for ()
 {
-    echo -n "|${1}|"
+    echo -n "=${1}|"
     for i in $(seq 1 $1) ; do
 	echo -n '.'
 	sleep 1
@@ -198,18 +272,12 @@ node_has_status ()
 
     local bits
     case "$status" in
-	banned)
-	    bits="?:1:?:?"
-	    ;;
-	unbanned)
-	    bits="?:0:?:?"
-	    ;;
-	disabled)
-	    bits="?:?:1:?"
-	    ;;
-	enabled)
-	    bits="?:?:0:?"
-	    ;;
+	(disconnected) bits="1:?:?:?" ;;
+	(connected)    bits="0:?:?:?" ;;
+	(banned)       bits="?:1:?:?" ;;
+	(unbanned)     bits="?:0:?:?" ;;
+	(disabled)     bits="?:?:1:?" ;;
+	(enabled)      bits="?:?:0:?" ;;
 	*)
 	    echo "node_has_status: unknown status \"$status\""
 	    return 1
@@ -366,12 +434,3 @@ restart_ctdb ()
     echo "ctdb is ready"
 }
 
-ctdb_test_exit ()
-{
-    if ! onnode 0 $TEST_WRAP cluster_is_healthy ; then
-	echo "Restarting ctdb on all nodes to get back into known state..."
-	restart_ctdb
-    fi
-
-    test_exit
-}
diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests
index 9827ebed63..9779bf336d 100755
--- a/ctdb/tests/scripts/run_tests
+++ b/ctdb/tests/scripts/run_tests
@@ -1,28 +1,9 @@
 #!/bin/bash
 
-export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd)
-var_dir=$CTDB_DIR/tests/var
-
-export CTDB_NUM_NODES=3
-
-######################################################################
-
-if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
-    export CTDB_NODES_SOCKETS=""
-    for i in $(seq 1 $CTDB_NUM_NODES) ; do
-	CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}"
-    done
-fi
-
-######################################################################
-
-ctdb_bin_dir=$CTDB_DIR/bin
-ctdb_tools_dir=$CTDB_DIR/tools
-ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd)
-
-PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}"
-
-export TEST_WRAP="${ctdb_test_scripts_dir}/test_wrap"
+# The ability of ctdb_test_env to take tests on the command-line is
+# nice, but here we need to hack around it with that colon to reset
+# the arguments that it sees.
+. $(dirname $0)/ctdb_test_env :
 
 . ctdb_test_functions.bash
 
@@ -53,8 +34,8 @@ done
 ######################################################################
 
 for f; do
-    [ -x $f ] || fail "test $f is not executable"
-    test_run "$f"
+    [ -x $f ] || fail "test \"$f\" is not executable"
+    ctdb_test_run "$f"
 done
 
 test_exit
diff --git a/ctdb/tests/simple/00_ctdb_init.sh b/ctdb/tests/simple/00_ctdb_init.sh
index 13b4303fd2..d3b592d10f 100755
--- a/ctdb/tests/simple/00_ctdb_init.sh
+++ b/ctdb/tests/simple/00_ctdb_init.sh
@@ -1,8 +1,32 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Restart the ctdbd daemons of a CTDB cluster.
+
+No error if ctdbd is not already running on the cluster.
+
+Prerequisites:
+
+* Nodes must be accessible via 'onnode'.
+
+Steps:
+
+1. Restart the ctdb daemons on all nodes using a method according to
+   the test environment and platform.
+
+Expected results:
+
+* The cluster is healthy within a reasonable timeframe.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
-echo "Restartng ctdb on all nodes..."
+echo "Restarting ctdb on all nodes..."
 restart_ctdb
diff --git a/ctdb/tests/simple/00_ctdb_onnode.sh b/ctdb/tests/simple/00_ctdb_onnode.sh
index bd269d73ce..c5736df911 100755
--- a/ctdb/tests/simple/00_ctdb_onnode.sh
+++ b/ctdb/tests/simple/00_ctdb_onnode.sh
@@ -1,11 +1,29 @@
 #!/bin/bash
 
-# Do a recursive "onnode all" to make sure all the nodes can connect
-# to each other.  On a cluster this ensures that SSH keys are known
-# between all hosts, which will stop output being corrupted with
-# messages about nodes being added to the list of known hosts.
+test_info()
+{
+    cat <<EOF
+Use 'onnode' to confirm connectivity between all cluster nodes.
+
+Steps:
+
+1. Do a recursive "onnode all" to make sure all the nodes can connect
+   to each other.  On a cluster this ensures that SSH keys are known
+   between all hosts, which will stop output being corrupted with
+   messages about nodes being added to the list of known hosts.
+
+Expected results:
+
+* 'onnode' works between all nodes.
+EOF
+}
 
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
+
+# 
+
 echo "Checking connectivity between nodes..."
 onnode all onnode all true
diff --git a/ctdb/tests/simple/01_ctdb_version.sh.disabled b/ctdb/tests/simple/01_ctdb_version.sh.disabled
index 36e0368ff6..6d9c179ddd 100755
--- a/ctdb/tests/simple/01_ctdb_version.sh.disabled
+++ b/ctdb/tests/simple/01_ctdb_version.sh.disabled
@@ -1,15 +1,42 @@
 #!/bin/bash
 
-. /root/SOFS/autosofs/scripts/functions
+test_info()
+{
+    cat <<EOF
+Verify the output of the 'ctdb version' command.
+
+This test assumes an RPM-based installation and needs to be skipped on
+non-RPM systems.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run the 'ctdb version' command on one of the cluster nodes.
+3. Compare the version displayed with that listed by the rpm command
+   for the ctdb package.
+
+Expected results:
+
+* The 'ctdb version' command displays the ctdb version number.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
 
 set -e
 
-onnode 0 /root/SOFS/autosofs/scripts/cluster_is_healthy.sh
+onnode 0 $TEST_WRAP cluster_is_healthy
 
 rpm_ver_cmd="rpm -q ctdb"
-ctdb_ver_cmd="ctdb version"
+ctdb_ver_cmd="ctdb version" ||true
 
-rpm_ver=$($rpm_ver_cmd)
+rpm_ver=$($rpm_ver_cmd) || true
 echo "$rpm_ver_cmd"
 echo "  $rpm_ver"
 
@@ -17,8 +44,8 @@ ctdb_ver=$(onnode 0 $ctdb_ver_cmd)
 echo "$ctdb_ver_cmd"
 echo "  $ctdb_ver"
 
-set -x  
+set -x
 
 [ "${ctdb_ver#CTDB version: }" = "${rpm_ver#ctdb-}" ]
 
-test_exit
+ctdb_test_exit
diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh
index fd0af39c9c..b5d4984a0f 100755
--- a/ctdb/tests/simple/02_ctdb_listvars.sh
+++ b/ctdb/tests/simple/02_ctdb_listvars.sh
@@ -1,7 +1,34 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb listvars' shows a list of all tunable variables.
+
+This test simply checks that at least 5 sane looking lines are
+printed.  It does not check that the list is complete or that the
+values are sane.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb listvars' and verify that it shows a list of tunable
+   variables and their current values.
+
+Expected results:
+
+* 'ctdb listvars' works as expected.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh
index f21b29b5b0..12bb6b9156 100755
--- a/ctdb/tests/simple/03_ctdb_getvar.sh
+++ b/ctdb/tests/simple/03_ctdb_getvar.sh
@@ -1,7 +1,35 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb getvar' works correctly.
+
+Expands on the steps below as it actually checks the values of all
+variables listed by 'ctdb listvars'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb getvars <varname>' with a valid variable name (possibly
+   obtained via 'ctdb listvars'.
+3. Verify that the command displays the correct value of the variable
+   (corroborate with the value shown by 'ctdb listvars'.
+
+Expected results:
+
+* 'ctdb getvar' shows the correct value of the variable.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh
index d3b543d379..d2cbc95395 100755
--- a/ctdb/tests/simple/04_ctdb_setvar.sh
+++ b/ctdb/tests/simple/04_ctdb_setvar.sh
@@ -1,10 +1,39 @@
 #!/bin/bash
 
-# Doesn't strictly follow the procedure, since it doesn't pick a
-# variable from the output of "ctdb listvars".
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb setvar' works correctly.
+
+Doesn't strictly follow the procedure outlines below, since it doesn't
+pick a variable from the output of 'ctdb listvars'.  However, it
+verifies the value with 'ctdb getvar' in addition to 'ctdb listvars'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get a list of all the ctdb tunable variables, using the 'ctdb
+   listvars' command.
+3. Set the value of one of the variables using the 'setvar' control on
+   one of the nodes.  E.g. 'ctdb setvar DeterministicIPs 0'.
+4. Verify that the 'listvars' control now shows the new value for the
+   variable.
+
+Expected results:
+
+* After setting a value using 'ctdb setvar', 'ctdb listvars' shows the
+  modified value of the variable.
+EOF
+}
 
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 9337a40fc8..57d74e8147 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -1,7 +1,31 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb listnodes' shows the list of nodes in a ctdb cluster.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb listnodes' on all the nodes of the cluster.
+3. Verify that one all the nodes the command displays a list of
+   current cluster nodes.
+
+Expected results:
+
+* 'ctdb listnodes' displays the correct information.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh
index 4a3290c16b..ed2fe88bb0 100755
--- a/ctdb/tests/simple/06_ctdb_getpid.sh
+++ b/ctdb/tests/simple/06_ctdb_getpid.sh
@@ -1,7 +1,33 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb getpid' works as expected.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb getpid -n <number>' on the nodes to check the PID of the
+   ctdbd process.
+3. Verify that the output is valid.
+4. Verify that with the '-n all' option the command shows the PIDs on
+   all the nodes
+
+Expected results:
+
+* 'ctdb getpid' shows valid output.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/07_ctdb_process_exists.sh b/ctdb/tests/simple/07_ctdb_process_exists.sh
index 17f12314e2..e6d0da0d5b 100755
--- a/ctdb/tests/simple/07_ctdb_process_exists.sh
+++ b/ctdb/tests/simple/07_ctdb_process_exists.sh
@@ -1,12 +1,43 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb process-exists' shows correct information.
+
+The implementation is creative about how it gets PIDs for existing and
+non-existing processes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. On one of the cluster nodes, get the PID of an existing process
+   (using ps wax).
+3. Run 'ctdb process-exists <pid>' on the node and verify that the
+   correct output is shown.
+4. Run 'ctdb process-exists <pid>' with a pid of a non-existent
+   process and verify that the correct output is shown.
+
+Expected results:
+
+* 'ctdb process-exists' shows the correct output.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
 
 # Create a background process on node 2 that will last for 60 seconds.
+# It should still be there when we check.
 try_command_on_node 2 'sleep 60 >/dev/null 2>&1 & echo $!'
 pid="$out"
 
@@ -25,7 +56,7 @@ fi
 
 # Now just echo the PID of the shell from the onnode process on node
 # 2.  This PID will disappear and PIDs shouldn't roll around fast
-# enough to trick the test...  but there is a chance that will happen.
+# enough to trick the test...  but there is a chance that will happen!
 try_command_on_node 2 'echo $$'
 pid="$out"
 
diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
index b0aac1ccae..1adaa822f1 100755
--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
@@ -1,7 +1,33 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify the operation of 'ctdb isnotrecmaster'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb isnotrecmaster' on each node.
+
+3. Verify that only 1 node shows the output 'This node is the
+   recmaster' and all the other nodes show the output 'This node is
+   not the recmaster'.
+
+Expected results:
+
+* 'ctdb isnotrecmaster' shows the correct output.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh
new file mode 100755
index 0000000000..4e02b6a120
--- /dev/null
+++ b/ctdb/tests/simple/09_ctdb_ping.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify the operation of the 'ctdb ping' command.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run the 'ctdb ping' command on one of the nodes and verify that it
+   shows valid and expected output. 
+3. Shutdown one of the cluster nodes, using the 'ctdb shutdown'
+   command. 
+4. Run the 'ctdb ping -n <node>' command from another node to this
+   node. 
+5. Verify that the command is not successful since th ctdb daemon is
+   not running on the node.
+
+Expected results:
+
+* The 'ctdb ping' command shows valid and expected output.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $TEST_WRAP cluster_is_healthy
+
+try_command_on_node -v 1 'ctdb ping -n 2'
+
+sanity_check_output \
+    1 \
+    '^response from 2 time=[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
+    "$out"
+
+try_command_on_node -v 0 'ctdb shutdown -n 2'
+
+onnode 0 $TEST_WRAP wait_until_node_has_status 2 disconnected
+
+try_command_on_node 1 'ctdb ping -n 2'
+
+sanity_check_output \
+    2 \
+    "(: ctdb_control error: 'ctdb_control to disconnected node'|Unable to get ping response from node 2)" \
+    "$out"
+
+echo "Expect a restart..."
+
+ctdb_test_exit
diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh
index de1b4a8629..75edccd155 100755
--- a/ctdb/tests/simple/11_ctdb_ip.sh
+++ b/ctdb/tests/simple/11_ctdb_ip.sh
@@ -1,7 +1,32 @@
 #!/bin/bash
 
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb ip' shows the correct output.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run 'ctdb ip' on one of the nodes and verify the list of IP
+   addresses displayed (cross check the result with the output of
+   'ip addr show' on the node).
+3. Verify that colon-separated output is generated with the -Y option.
+
+Expected results:
+
+* 'ctdb ip' shows the list of public IPs being served by a node.
+EOF
+}
+
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/31_ctdb_disable_simple.sh b/ctdb/tests/simple/31_ctdb_disable_simple.sh
index f49af88396..f2839d585d 100755
--- a/ctdb/tests/simple/31_ctdb_disable_simple.sh
+++ b/ctdb/tests/simple/31_ctdb_disable_simple.sh
@@ -1,12 +1,38 @@
 #!/bin/bash
 
-# From node 1, disable node 2.  Make sure that according to "ctdb ip"
-# the public addresses are taken over and according to "ctdb status"
-# the node appears to be disabled.  Don't actually check if the
-# address has been correctly taken over.
+test_info()
+{
+    cat <<EOF
+Verify the operation of 'ctdb disable'.
+
+This is a superficial test of the 'ctdb disable' command.  It trusts
+information from CTDB that indicates that the IP failover has happened
+correctly.  Another test should check that the failover has actually
+happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable one of the nodes using 'ctdb disable -n <node>'.
+3. Verify that the status of the node changes to 'disabled'.
+4. Verify that the IP addreses served by the disabled node are failed
+   over to other nodes.
+
+Expected results:
+
+* The status of the disabled node changes as expected and IP addresses
+  failover as expected.
+EOF
+}
 
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
@@ -36,6 +62,6 @@ else
     testfailures=1
 fi
 
-echo "Expect a restart here..."
+echo "Expect a restart..."
 
 ctdb_test_exit
diff --git a/ctdb/tests/simple/32_ctdb_enable_simple.sh b/ctdb/tests/simple/32_ctdb_enable_simple.sh
index 94b5beebb0..a9975eb576 100755
--- a/ctdb/tests/simple/32_ctdb_enable_simple.sh
+++ b/ctdb/tests/simple/32_ctdb_enable_simple.sh
@@ -1,12 +1,43 @@
 #!/bin/bash
 
-# From node 1, disable node 2.  Make sure that according to "ctdb ip"
-# the public addresses are taken over and according to "ctdb status"
-# the node appears to be disabled.  Don't actually check if the
-# address has been correctly taken over.
+test_info()
+{
+    cat <<EOF
+Verify the operation of 'ctdb enable'.
+
+This is a superficial test of the 'ctdb enable' command.  It trusts
+information from CTDB that indicates that the IP failover has happened
+correctly.  Another test should check that the failover has actually
+happened at the networking level.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable one of the nodes using 'ctdb disable -n <node>'.
+3. Verify that the status of the node changes to 'disabled'.
+4. Verify that the public IP addreses served by the disabled node are
+   failed over to other nodes.
+5. Enable the disabled node using 'ctdb enable -n '<node>'.
+6. Verify that the status changes back to 'OK'.
+7. Verify that the public IP addreses served by the disabled node are
+   failed back to the node.
+
+
+Expected results:
+
+* The status of a re-enabled node changes as expected and IP addresses
+  fail back as expected.
+EOF
+}
 
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 # Note that this doesn't work reliably over NFS!
 ctdb_trigger_recovered_file="/tmp/ctdb-trigger-recovered"
 
diff --git a/ctdb/tests/simple/41_ctdb_ban_simple.sh b/ctdb/tests/simple/41_ctdb_ban_simple.sh
index e5d4bdbcee..5b5ca57810 100755
--- a/ctdb/tests/simple/41_ctdb_ban_simple.sh
+++ b/ctdb/tests/simple/41_ctdb_ban_simple.sh
@@ -1,10 +1,8 @@
 #!/bin/bash
 
-CATEGORY="CTDB"
-
 test_info()
 {
-cat <<EOF
+    cat <<EOF
 Verify the operation of the 'ctdb ban' command.
 
 This is a superficial test of the 'ctdb ban' command.  It trusts
@@ -32,12 +30,13 @@ Expected results:
 
 * The status of the banned nodes changes as expected and IP addresses
   failover as expected.
-
 EOF
 }
 
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy
diff --git a/ctdb/tests/simple/42_ctdb_unban_simple.sh b/ctdb/tests/simple/42_ctdb_unban_simple.sh
index 05b363a916..96499efeb6 100755
--- a/ctdb/tests/simple/42_ctdb_unban_simple.sh
+++ b/ctdb/tests/simple/42_ctdb_unban_simple.sh
@@ -1,10 +1,8 @@
 #!/bin/bash
 
-CATEGORY="CTDB"
-
 test_info()
 {
-cat <<EOF
+    cat <<EOF
 Verify the operation of the 'ctdb unban' command.
 
 This is a superficial test of the 'ctdb uban' command.  It trusts
@@ -32,12 +30,13 @@ Steps:
 Expected results:
 
 * The 'ctdb unban' command successfully unbans a banned node.
-
 EOF
 }
 
 . ctdb_test_functions.bash
 
+ctdb_test_init "$@"
+
 set -e
 
 onnode 0 $TEST_WRAP cluster_is_healthy