Delete some unstructured tests ({fetch,peristent,transaction}.sh) and

replace them with new simple tests (52_ctdb_fetch.sh,
53_ctdb_transaction.sh, 61_ctdb_persistent_safe.sh,
62_ctdb_persistent_unsafe.sh).  Remove "_simple" from some test
filenames in the simple subdirectory - that's redundant.  Always run
ctdb as $CTDB to allow $VALGRIND magic to be used.  Use pgrep/pkill to
detect/kill local daemons so those running under valgrind can be found
too - to support this, always run local daemons with the full path to
the executable.  run_tests now supports -s option to print sumamry
when done - with more and more tests, it is getting hard to follow
progress.  Sort the output of commands in 06_ctdb_getpid.sh to make
sure they compare nicely and also allow the processes' executables to
be called "memcheck" to catch those running under valgrind.  Remove
redundant calls to onnode in commands run from calls
try_command_on_node in some tests.  41_ctdb_ban.sh and
42_ctdb_unban.sh avoid banning the recmaster, since this causes the
recmaster to be reassigned and all nodes to be unbanned.  Minor
cleanups.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 33cdf3e4bcfadf8e20822ca352babf7acca16821)

34 files changed, 319 insertions, 93 deletions

diff --git a/ctdb/tests/simple/01_ctdb_version.sh b/ctdb/tests/simple/01_ctdb_version.sh
index 2e4ef187f34..1cc506fbc00 100755
--- a/ctdb/tests/simple/01_ctdb_version.sh
+++ b/ctdb/tests/simple/01_ctdb_version.sh
@@ -39,7 +39,7 @@ if ! try_command_on_node -v 0 "rpm -q ctdb" ; then
 fi
 rpm_ver="${out#ctdb-}"
 
-try_command_on_node -v 0 "ctdb version"
+try_command_on_node -v 0 "$CTDB version"
 ctdb_ver="${out#CTDB version: }"
 
 if [ "$ctdb_ver" = "$rpm_ver" ] ; then
diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh
index 1c0ee3dfe5f..ddd77f71f85 100755
--- a/ctdb/tests/simple/02_ctdb_listvars.sh
+++ b/ctdb/tests/simple/02_ctdb_listvars.sh
@@ -33,7 +33,7 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node -v 0 "ctdb listvars"
+try_command_on_node -v 0 "$CTDB listvars"
 
 sanity_check_output \
     5 \
diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh
index 0e527aa319f..cae0f6984bc 100755
--- a/ctdb/tests/simple/03_ctdb_getvar.sh
+++ b/ctdb/tests/simple/03_ctdb_getvar.sh
@@ -34,13 +34,13 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node -v 0 "ctdb listvars"
+try_command_on_node -v 0 "$CTDB listvars"
 
 echo "Veryifying all variable values using \"ctdb getvar\"..."
 
 echo "$out" |
 while read var x val ; do
-    try_command_on_node 0 "ctdb getvar $var"
+    try_command_on_node 0 "$CTDB getvar $var"
 
     val2=$(echo $out | sed -e 's@.*[[:space:]]@@')
 
diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh
index 19655f70060..9348308f2d4 100755
--- a/ctdb/tests/simple/04_ctdb_setvar.sh
+++ b/ctdb/tests/simple/04_ctdb_setvar.sh
@@ -40,7 +40,7 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 var="RecoverTimeout"
 
-try_command_on_node -v 0 ctdb getvar $var
+try_command_on_node -v 0 $CTDB getvar $var
 
 val="${out#*= }"
 
@@ -48,11 +48,11 @@ echo "Going to try incrementing it..."
 
 incr=$(($val + 1))
 
-try_command_on_node 0 ctdb setvar $var $incr
+try_command_on_node 0 $CTDB setvar $var $incr
 
 echo "That seemed to work, let's check the value..."
 
-try_command_on_node -v 0 ctdb getvar $var
+try_command_on_node -v 0 $CTDB getvar $var
 
 newval="${out#*= }"
 
@@ -62,7 +62,7 @@ if [ "$incr" != "$newval" ] ; then
 fi
 
 echo "Look's good!  Now verifying with \"ctdb listvars\""
-try_command_on_node -v 0 "ctdb listvars | grep '^$var'"
+try_command_on_node -v 0 "$CTDB listvars | grep '^$var'"
 
 check="${out#*= }"
 
@@ -73,7 +73,7 @@ if [ "$incr" != "$check" ] ; then
 fi
 
 echo "Look's good!  Putting the old value back..."
-cmd="ctdb setvar $var $val"
+cmd="$CTDB setvar $var $val"
 try_command_on_node 0 $cmd
 
 echo "All done..."
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 1db3a0b580c..6e6af1f8b15 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -30,7 +30,7 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node -v 0 "ctdb listnodes"
+try_command_on_node -v 0 "$CTDB listnodes"
 
 num_nodes=$(echo "$out" | wc -l)
 
@@ -47,7 +47,7 @@ echo "Checking other nodes..."
 n=1
 while [ $n -lt $num_nodes ] ; do
     echo -n "Node ${n}: "
-    try_command_on_node $n "ctdb listnodes"
+    try_command_on_node $n "$CTDB listnodes"
     if [ "$out_0" = "$out" ] ; then
 	echo "OK"
     else
diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh
index b12a296490a..b068d77f567 100755
--- a/ctdb/tests/simple/06_ctdb_getpid.sh
+++ b/ctdb/tests/simple/06_ctdb_getpid.sh
@@ -34,25 +34,25 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 # This is an attempt at being independent of the number of nodes
 # reported by "ctdb getpid -n all".
-try_command_on_node 0 "ctdb listnodes | wc -l"
+try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
 echo "There are $num_nodes nodes..."
 
 # Call getpid a few different ways and make sure the answer is always the same.
 
-try_command_on_node -v 0 "onnode -q all ctdb getpid"
+try_command_on_node -v 0 "onnode -q all $CTDB getpid | sort"
 pids_onnode="$out"
 
-try_command_on_node -v 0 "ctdb getpid -n all"
+try_command_on_node -v 0 "$CTDB getpid -n all | sort"
 pids_getpid_all="$out"
 
 cmd=""
 n=0
 while [ $n -lt $num_nodes ] ; do
-    cmd="${cmd}${cmd:+; }ctdb getpid -n $n"
+    cmd="${cmd}${cmd:+; }$CTDB getpid -n $n"
     n=$(($n + 1))
 done
-try_command_on_node -v 0 "$cmd"
+try_command_on_node -v 0 "( $cmd ) | sort"
 pids_getpid_n="$out"
 
 if [ "$pids_onnode" = "$pids_getpid_all" -a \
@@ -73,6 +73,9 @@ while [ $n -lt $num_nodes ] ; do
     echo -n "Node ${n}, PID ${pid} looks to be running \"$out\" - "
     if [ "$out" = "ctdbd" ] ; then
 	echo "GOOD!"
+    elif [ -n "$VALGRIND" -a "$out" = "memcheck" ] ; then
+	# We could check cmdline too if this isn't good enough.
+	echo "GOOD enough!"
     else
 	echo "BAD!"
 	testfailures=1
diff --git a/ctdb/tests/simple/07_ctdb_process_exists.sh b/ctdb/tests/simple/07_ctdb_process_exists.sh
index 573202003fd..01c186c87ba 100755
--- a/ctdb/tests/simple/07_ctdb_process_exists.sh
+++ b/ctdb/tests/simple/07_ctdb_process_exists.sh
@@ -46,7 +46,7 @@ pid="$out"
 echo "Checking for PID $pid on node $test_node"
 # set -e is good, but avoid it here
 status=0
-onnode 0 "ctdb process-exists ${test_node}:${pid}" || status=$?
+onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
 echo "$out"
 
 if [ $status -eq 0 ] ; then
@@ -65,7 +65,7 @@ pid="$out"
 echo "Checking for PID $pid on node $test_node"
 # set -e is good, but avoid it here
 status=0
-onnode 0 "ctdb process-exists ${test_node}:${pid}" || status=$?
+onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$?
 echo "$out"
 
 if [ $status -ne 0 ] ; then
diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
index c4a4b65c27a..f5f9c6c077f 100755
--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
@@ -32,7 +32,7 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-cmd='ctdb isnotrecmaster || true'
+cmd="$CTDB isnotrecmaster || true"
 try_command_on_node all "$cmd"
 echo "Output of \"$cmd\":"
 echo "$out"
diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh
index 7e8df020672..bd23f77b59c 100755
--- a/ctdb/tests/simple/09_ctdb_ping.sh
+++ b/ctdb/tests/simple/09_ctdb_ping.sh
@@ -35,18 +35,18 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node -v 0 'ctdb ping -n 1'
+try_command_on_node -v 0 "$CTDB ping -n 1"
 
 sanity_check_output \
     1 \
     '^response from 1 time=[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
     "$out"
 
-try_command_on_node 0 'ctdb shutdown -n 1'
+try_command_on_node 0 "$CTDB shutdown -n 1"
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status 1 disconnected
 
-try_command_on_node -v 0 '! ctdb ping -n 1'
+try_command_on_node -v 0 "! $CTDB ping -n 1"
 
 sanity_check_output \
     1 \
diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh
index 941a8ef9e06..cad89866b17 100755
--- a/ctdb/tests/simple/11_ctdb_ip.sh
+++ b/ctdb/tests/simple/11_ctdb_ip.sh
@@ -32,7 +32,7 @@ set -e
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 echo "Getting list of public IPs..."
-try_command_on_node -v 1 ctdb ip -n all
+try_command_on_node -v 1 $CTDB ip -n all
 ips=$(echo "$out" | sed -e '1d')
 colons=$(echo "$ips" | sed -e 's@^@:@' -e 's@$@:@' -e 's@ @:@')
 
@@ -48,7 +48,7 @@ done <<<"$ips" # bashism to avoid problem setting variable in pipeline.
 
 [ "$testfailures" != 1 ] && echo "Looks good!"
 
-cmd="ctdb -Y ip -n all | sed -e '1d'"
+cmd="$CTDB -Y ip -n all | sed -e '1d'"
 echo "Checking that \"$cmd\" produces expected output..."
 
 try_command_on_node 1 "$cmd"
diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh
index 4e22a33fc09..2f4eefdff67 100755
--- a/ctdb/tests/simple/12_ctdb_getdebug.sh
+++ b/ctdb/tests/simple/12_ctdb_getdebug.sh
@@ -30,10 +30,10 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node 0 "ctdb listnodes | wc -l"
+try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
 
-try_command_on_node -v 1 "onnode -q all ctdb getdebug"
+try_command_on_node -v 1 "onnode -q all $CTDB getdebug"
 getdebug_onnode="$out"
 
 sanity_check_output \
@@ -41,13 +41,13 @@ sanity_check_output \
     '^Node [[:digit:]] is at debug level [[:alpha:]]+ \([[:digit:]]\)$' \
     "$out"
 
-try_command_on_node -v 1 "onnode -q 1 ctdb getdebug -n all"
+try_command_on_node -v 1 "$CTDB getdebug -n all"
 getdebug_all="$out"
 
 cmd=""
 n=0
 while [ $n -lt $num_nodes ] ; do
-    cmd="${cmd}${cmd:+; }ctdb getdebug -n $n"
+    cmd="${cmd}${cmd:+; }$CTDB getdebug -n $n"
     n=$(($n + 1))
 done
 try_command_on_node -v 1 "$cmd"
@@ -69,7 +69,7 @@ while read line ; do
     colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}"
 done <<<"$getdebug_onnode"
 
-cmd="ctdb -Y getdebug -n all"
+cmd="$CTDB -Y getdebug -n all"
 echo "Checking that \"$cmd\" produces expected output..."
 
 try_command_on_node 1 "$cmd"
diff --git a/ctdb/tests/simple/13_ctdb_setdebug.sh b/ctdb/tests/simple/13_ctdb_setdebug.sh
index 2260c92559a..dec19e17ab5 100755
--- a/ctdb/tests/simple/13_ctdb_setdebug.sh
+++ b/ctdb/tests/simple/13_ctdb_setdebug.sh
@@ -45,7 +45,7 @@ get_debug ()
 
     local out
     
-    try_command_on_node -v 0 "onnode -q ${node} ctdb getdebug"
+    try_command_on_node -v $node "$CTDB getdebug"
     check_debug=$(echo "$out" |
 	sed -r -e 's@Node [[:digit:]] is at debug level ([[:alpha:]]+) \(-?[[:digit:]]\)$@\1@')
 }
@@ -56,7 +56,7 @@ set_and_check_debug ()
     local level="$2"
 
     echo "Setting debug level on node ${node} to ${level}."
-    try_command_on_node 0 "onnode -q ${node} ctdb setdebug ${level}"
+    try_command_on_node $node "$CTDB setdebug ${level}"
 
     local check_debug
     get_debug $node
diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh
index 74046cb7e53..202e0a0ffba 100755
--- a/ctdb/tests/simple/14_ctdb_statistics.sh
+++ b/ctdb/tests/simple/14_ctdb_statistics.sh
@@ -35,11 +35,11 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 pattern='^(CTDB version 1|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts)|[[:space:]]+[[:alpha:]_]+_latency[[:space:]]+[[:digit:]]+\.[[:digit:]]+[[:space:]]sec)$'
 
-try_command_on_node -v 0 "onnode -q 1 ctdb statistics"
+try_command_on_node -v 1 "$CTDB statistics"
 
 sanity_check_output 38 "$pattern" "$out"
 
-try_command_on_node -v 0 "onnode -q 1 ctdb statistics -n all"
+try_command_on_node -v 1 "$CTDB statistics -n all"
 
 sanity_check_output 38 "$pattern" "$out"
 
diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
index 5a8d3e08609..9de83c011f5 100755
--- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh
+++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
@@ -32,7 +32,7 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node 0 "ctdb listnodes | wc -l"
+try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
 
 get_stat ()
@@ -61,15 +61,15 @@ n=0
 while [ $n -lt $num_nodes ] ; do
     echo "Getting initial statistics for node ${n}..."
     
-    try_command_on_node -v $n ctdb statistics
+    try_command_on_node -v $n $CTDB statistics
 
     before_req_control=$(get_stat "req_control" "$out")
     before_reply_control=$(get_stat "reply_control" "$out")
     before_node_packets_recv=$(get_stat "node_packets_recv" "$out")
 
-    try_command_on_node $n ctdb statisticsreset
+    try_command_on_node $n $CTDB statisticsreset
 
-    try_command_on_node -v $n ctdb statistics
+    try_command_on_node -v $n $CTDB statistics
 
     after_req_control=$(get_stat "req_control" "$out")
     after_reply_control=$(get_stat "reply_control" "$out")
diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
index 1ce46ca440a..50fc9c5cb94 100755
--- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh
+++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
@@ -43,7 +43,7 @@ set -e
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 echo "Getting list of public IPs..."
-try_command_on_node 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
 # When selecting test_node we just want a node that has public IPs.
 # This will work and is economically semi-randomly.  :-)
@@ -101,7 +101,7 @@ done
 
 if [ -n "$add_ip" ] ; then
     echo "Adding IP: ${add_ip/:/ on interface }"
-    try_command_on_node $test_node ctdb addip ${add_ip/:/ }
+    try_command_on_node $test_node $CTDB addip ${add_ip/:/ }
 
     echo "Waiting for IP to be added..."
     wait_until 60 ips_are_on_nodeglob $test_node $test_node_ips ${add_ip%/*}
diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
index 56586492ac7..822c7023c26 100755
--- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
+++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
@@ -39,7 +39,7 @@ set -e
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 echo "Getting list of public IPs..."
-try_command_on_node -v 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
 
 # Select an IP/node to remove.
 num_ips=$(echo "$out" | wc -l)
@@ -53,7 +53,7 @@ while [ $i -le $num_to_remove ] ; do
 done <<<"$out"
 
 echo "Attempting to remove ${ip_to_remove} from node ${test_node}."
-try_command_on_node $test_node ctdb delip $ip_to_remove
+try_command_on_node $test_node $CTDB delip $ip_to_remove
 
 echo "Sleeping..."
 sleep_for 1
diff --git a/ctdb/tests/simple/18_ctdb_freeze.sh b/ctdb/tests/simple/18_ctdb_freeze.sh
index c91373f84bd..5f2ccb0a137 100755
--- a/ctdb/tests/simple/18_ctdb_freeze.sh
+++ b/ctdb/tests/simple/18_ctdb_freeze.sh
@@ -40,7 +40,7 @@ test_node=1
 
 echo "Freezing node $test_node"
 
-try_command_on_node 0 ctdb freeze -n $test_node
+try_command_on_node 0 $CTDB freeze -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
 
diff --git a/ctdb/tests/simple/19_ctdb_thaw.sh b/ctdb/tests/simple/19_ctdb_thaw.sh
index ff9c6305ab5..c764205911e 100755
--- a/ctdb/tests/simple/19_ctdb_thaw.sh
+++ b/ctdb/tests/simple/19_ctdb_thaw.sh
@@ -44,13 +44,13 @@ test_node=1
 
 echo "Freezing node $test_node"
 
-try_command_on_node 0 ctdb freeze -n $test_node
+try_command_on_node 0 $CTDB freeze -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
 
 echo "That worked!  Now thawing node $test_node"
 
-try_command_on_node 0 ctdb thaw -n $test_node
+try_command_on_node 0 $CTDB thaw -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unfrozen
 
diff --git a/ctdb/tests/simple/20_ctdb_getmonmode.sh b/ctdb/tests/simple/20_ctdb_getmonmode.sh
index 215f0cfb54a..802f302945e 100755
--- a/ctdb/tests/simple/20_ctdb_getmonmode.sh
+++ b/ctdb/tests/simple/20_ctdb_getmonmode.sh
@@ -40,7 +40,7 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 test_node=1
 
-try_command_on_node -v 0 ctdb getmonmode -n $test_node
+try_command_on_node -v 0 $CTDB getmonmode -n $test_node
 
 sanity_check_output \
     1 \
@@ -49,7 +49,7 @@ sanity_check_output \
 
 colons=$(printf ':mode:\n:0:')
 
-try_command_on_node -v 0 ctdb -Y getmonmode -n $test_node
+try_command_on_node -v 0 $CTDB -Y getmonmode -n $test_node
 
 if [ "$out" = "$colons" ] ; then
     echo "Looks OK"
@@ -58,7 +58,7 @@ else
     testfailures=1
 fi
 
-try_command_on_node -v 0 ctdb disablemonitor -n $test_node
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
 
diff --git a/ctdb/tests/simple/21_ctdb_disablemonitor.sh b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
index 333e4fc908d..85855e4ca4d 100755
--- a/ctdb/tests/simple/21_ctdb_disablemonitor.sh
+++ b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
@@ -47,7 +47,7 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 test_node=1
 
 # We need this for later, so we know how long to sleep.
-try_command_on_node -v 0 ctdb getvar MonitorInterval -n $test_node
+try_command_on_node -v 0 $CTDB getvar MonitorInterval -n $test_node
 monitor_interval="${out#*= }"
 echo "Monitor interval on node $test_node is $monitor_interval seconds."
 
@@ -71,7 +71,7 @@ try_command_on_node -v $test_node ls -l "$detected"
 echo "Waiting until recovery is complete..."
 wait_until 30 onnode $test_node ! test -e "$recovered_flag"
 
-try_command_on_node -v 0 ctdb disablemonitor -n $test_node
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
 
 sanity_check_output \
     1 \
diff --git a/ctdb/tests/simple/22_ctdb_enablemonitor.sh b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
index 995a8ae2dc3..f7d68091c85 100755
--- a/ctdb/tests/simple/22_ctdb_enablemonitor.sh
+++ b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
@@ -45,11 +45,11 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 test_node=1
 
 # We need this for later, so we know how long to sleep.
-try_command_on_node -v 0 ctdb getvar MonitorInterval -n $test_node
+try_command_on_node -v 0 $CTDB getvar MonitorInterval -n $test_node
 monitor_interval="${out#*= }"
 echo "Monitor interval on node $test_node is $monitor_interval seconds."
 
-try_command_on_node -v 0 ctdb disablemonitor -n $test_node
+try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
 
 sanity_check_output \
     1 \
@@ -77,7 +77,7 @@ try_command_on_node $test_node test ! -e "$detected"
 
 echo "OK: flag file was not created so monitoring must be disabled."
 
-try_command_on_node -v 0 ctdb enablemonitor -n $test_node
+try_command_on_node -v 0 $CTDB enablemonitor -n $test_node
 
 sanity_check_output \
     1 \
diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
index cb775be3645..a0fec6ba80c 100755
--- a/ctdb/tests/simple/23_ctdb_moveip.sh
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@@ -40,7 +40,7 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 # Restart when done since things are likely to be broken.
 ctdb_test_exit_hook="restart_ctdb"
 
-try_command_on_node 0 "ctdb listnodes | wc -l"
+try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
 echo "There are $num_nodes nodes..."
 
@@ -50,7 +50,7 @@ if [ $num_nodes -lt 2 ] ; then
 fi
 
 echo "Getting list of public IPs..."
-try_command_on_node -v 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
 
 # Select an IP/node to move.
 num_ips=$(echo "$out" | wc -l)
@@ -76,13 +76,13 @@ while [ $test_node -eq $to_node ] ; do
 done
 
 echo "Turning off DeterministicIPs..."
-try_command_on_node 0 ctdb setvar DeterministicIPs 0 -n all
+try_command_on_node 0 $CTDB setvar DeterministicIPs 0 -n all
 
 echo "Turning on NoIPFailback..."
-try_command_on_node 0 ctdb setvar NoIPFailback 1 -n all
+try_command_on_node 0 $CTDB setvar NoIPFailback 1 -n all
 
 echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}."
-try_command_on_node $test_node ctdb moveip $ip_to_move $to_node
+try_command_on_node $test_node $CTDB moveip $ip_to_move $to_node
 
 if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ip_to_move ; then
     echo "IP moved from ${test_node}."
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index 8c419210610..49ebda07429 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -41,7 +41,7 @@ make_temp_db_filename ()
     awk '{printf "%s.tdb\n", $1}'
 }
 
-try_command_on_node -v 0 "ctdb getdbmap"
+try_command_on_node -v 0 "$CTDB getdbmap"
 
 db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$'
 
@@ -52,8 +52,8 @@ num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
 for i in $(seq 1 5) ; do
     f=$(make_temp_db_filename)
     echo "Creating test database: $f"
-    try_command_on_node 0 ctdb attach "$f"
-    try_command_on_node 0 ctdb getdbmap
+    try_command_on_node 0 $CTDB attach "$f"
+    try_command_on_node 0 $CTDB getdbmap
     sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out"
     num=$(echo "$out" | sed -n -e '1s/^.*://p')
     if [ $num = $(($num_db_init + $i)) ] ; then
diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh
index babaafbb160..2b719657ccf 100755
--- a/ctdb/tests/simple/25_dumpmemory.sh
+++ b/ctdb/tests/simple/25_dumpmemory.sh
@@ -30,7 +30,7 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node -v 0 "ctdb dumpmemory"
+try_command_on_node -v 0 "$CTDB dumpmemory"
 
 pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks \(ref [[:digit:]]+\)[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ \(total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks\))$'
 
@@ -38,10 +38,10 @@ sanity_check_output 10 "$pat" "$out"
 
 echo "Checking output using '-n all'..."
 
-try_command_on_node 0 "ctdb listnodes"
+try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
-try_command_on_node 0 "ctdb dumpmemory" -n all
+try_command_on_node 0 "$CTDB dumpmemory" -n all
 sanity_check_output 10 "$pat" "$out"
 
 if [ $(fgrep -c 'full talloc report on' <<<"$out") -eq  $num_nodes ] ; then
diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index 91f04ef4634..72e0acd7a65 100755
--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -42,21 +42,19 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 test_node=1
 
-try_command_on_node 0 "ctdb listnodes"
+try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 echo "There are $num_nodes nodes."
 
 echo "Shutting down node ${test_node}..."
-try_command_on_node $test_node ctdb shutdown
+try_command_on_node $test_node $CTDB shutdown
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disconnected
 
-msg="ctdb_control error: 'ctdb_control to disconnected node'"
+pat="ctdb_control error: 'ctdb_control to disconnected node'|Node $test_node is DISCONNECTED"
 
 for i in ip disable enable "ban 0" unban listvars ; do
-    try_command_on_node -v 0 ! ctdb $i -n $test_node
-
-    pat="ctdb_control error: 'ctdb_control to disconnected node'|Node $test_node is DISCONNECTED"
+    try_command_on_node -v 0 ! $CTDB $i -n $test_node
 
     if egrep -q "$pat" <<<"$out" ; then
 	echo "OK: \"ctdb ${i}\" fails with \"disconnected node\""
diff --git a/ctdb/tests/simple/31_ctdb_disable_simple.sh b/ctdb/tests/simple/31_ctdb_disable.sh
index 00c21687d55..bba03b14cef 100755
--- a/ctdb/tests/simple/31_ctdb_disable_simple.sh
+++ b/ctdb/tests/simple/31_ctdb_disable.sh
@@ -38,7 +38,7 @@ set -e
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 echo "Getting list of public IPs..."
-try_command_on_node 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
 # When selecting test_node we just want a node that has public IPs.
 # This will work and is economically semi-randomly.  :-)
@@ -55,7 +55,7 @@ echo "Selected node ${test_node} with IPs: $ips"
 
 echo "Disabling node $test_node"
 
-try_command_on_node 1 ctdb disable -n $test_node
+try_command_on_node 1 $CTDB disable -n $test_node
 
 # Avoid a potential race condition...
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled
diff --git a/ctdb/tests/simple/32_ctdb_enable_simple.sh b/ctdb/tests/simple/32_ctdb_enable.sh
index 64a49ec7d09..7f3da4cd000 100755
--- a/ctdb/tests/simple/32_ctdb_enable_simple.sh
+++ b/ctdb/tests/simple/32_ctdb_enable.sh
@@ -45,7 +45,7 @@ set -e
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
 echo "Getting list of public IPs..."
-try_command_on_node 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
 # When selecting test_node we just want a node that has public IPs.
 # This will work and is economically semi-randomly.  :-)
@@ -61,7 +61,7 @@ done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 echo "Selected node ${test_node} with IPs: $ips"
 
 echo "Disabling node $test_node"
-try_command_on_node 1 ctdb disable -n $test_node
+try_command_on_node 1 $CTDB disable -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node disabled
 
@@ -73,7 +73,7 @@ else
 fi
 
 echo "Reenabling node $test_node"
-try_command_on_node 1 ctdb enable -n $test_node
+try_command_on_node 1 $CTDB enable -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node enabled
 
diff --git a/ctdb/tests/simple/41_ctdb_ban_simple.sh b/ctdb/tests/simple/41_ctdb_ban.sh
index 44ce9f50a7d..3567562e130 100755
--- a/ctdb/tests/simple/41_ctdb_ban_simple.sh
+++ b/ctdb/tests/simple/41_ctdb_ban.sh
@@ -41,18 +41,24 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
 echo "Getting list of public IPs..."
-try_command_on_node 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
-# When selecting test_node we just want a node that has public IPs.
-# This will work and is economically semi-randomly.  :-)
-read x test_node <<<"$out"
+# When selecting test_node we want a node that has public IPs and that
+# is not the recmaster.  We pick the first one that satisfies both
+# conditions.  We avoid the recmaster because banning the recmaster
+# (obviously) causes the recmaster to change... and changing the
+# recmaster causes all nodes to become unbanned!
+test_node=""
 
 ips=""
 while read ip pnn ; do
-    if [ "$pnn" = "$test_node" ] ; then
-	ips="${ips}${ips:+ }${ip}"
-    fi
+    [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
+    [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 
 echo "Selected node ${test_node} with IPs: $ips"
@@ -60,7 +66,7 @@ echo "Selected node ${test_node} with IPs: $ips"
 ban_time=15
 
 echo "Banning node $test_node for $ban_time seconds"
-try_command_on_node 1 ctdb ban $ban_time -n $test_node
+try_command_on_node 1 $CTDB ban $ban_time -n $test_node
 
 # Avoid a potential race condition...
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
diff --git a/ctdb/tests/simple/42_ctdb_unban_simple.sh b/ctdb/tests/simple/42_ctdb_unban.sh
index f09925ec333..1256ddd4042 100755
--- a/ctdb/tests/simple/42_ctdb_unban_simple.sh
+++ b/ctdb/tests/simple/42_ctdb_unban.sh
@@ -41,18 +41,21 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+echo "Finding out which node is the recovery master..."
+try_command_on_node -v 0 "$CTDB recmaster"
+recmaster=$out
+
 echo "Getting list of public IPs..."
-try_command_on_node 0 'ctdb ip -n all | sed -e "1d"'
+try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
-# When selecting test_node we just want a node that has public IPs.
-# This will work and is economically semi-randomly.  :-)
-read x test_node <<<"$out"
+# See 41_ctdb_ban.sh for an explanation of why test_node is chosen
+# like this.
+test_node=""
 
 ips=""
 while read ip pnn ; do
-    if [ "$pnn" = "$test_node" ] ; then
-	ips="${ips}${ips:+ }${ip}"
-    fi
+    [ -z "$test_node" -a $recmaster -ne $pnn ] && test_node=$pnn
+    [ "$pnn" = "$test_node" ] && ips="${ips}${ips:+ }${ip}"
 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 
 echo "Selected node ${test_node} with IPs: $ips"
@@ -60,7 +63,7 @@ echo "Selected node ${test_node} with IPs: $ips"
 ban_time=60
 
 echo "Banning node $test_node for $ban_time seconds"
-try_command_on_node 1 ctdb ban $ban_time -n $test_node
+try_command_on_node 1 $CTDB ban $ban_time -n $test_node
 
 # Avoid a potential race condition...
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node banned
@@ -73,7 +76,7 @@ else
 fi
 
 echo "Unbanning node $test_node"
-try_command_on_node 1 ctdb unban -n $test_node
+try_command_on_node 1 $CTDB unban -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unbanned
 
diff --git a/ctdb/tests/simple/51_ctdb_bench.sh b/ctdb/tests/simple/51_ctdb_bench.sh
index 6996ca678cb..7a1abc83f0f 100755
--- a/ctdb/tests/simple/51_ctdb_bench.sh
+++ b/ctdb/tests/simple/51_ctdb_bench.sh
@@ -34,11 +34,11 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-try_command_on_node 0 "ctdb listnodes"
+try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
 
 echo "Running ctdb_bench on all $num_nodes nodes."
-try_command_on_node -v -pq all ctdb_bench -n $num_nodes
+try_command_on_node -v -pq all $VALGRIND ctdb_bench -n $num_nodes
 
 # Get the last line of output.
 while read line ; do
diff --git a/ctdb/tests/simple/52_ctdb_fetch.sh b/ctdb/tests/simple/52_ctdb_fetch.sh
new file mode 100755
index 00000000000..6d4b66a9d21
--- /dev/null
+++ b/ctdb/tests/simple/52_ctdb_fetch.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Run the ctdb_fetch test and sanity check the output.
+
+This doesn't test for performance regressions or similarly anything
+useful.  Only vague sanity checking of results is done.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run ctdb_fetch on all nodes with default options.
+3. Ensure that the number of +ve and -ive messages are within 1% of
+   each other.
+4. Ensure that the number of messages per second is greater than 10.
+
+Expected results:
+
+* ctdb_fetch runs without error and prints reasonable results.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+echo "Running ctdb_fetch on all $num_nodes nodes."
+try_command_on_node -v -pq all $VALGRIND ctdb_fetch -n $num_nodes
+
+# Get the last line of output.
+while read line ; do
+    prev=$line
+done <<<"$out"
+
+pat='^(Ring: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\)[[:space:]]?|Waiting for cluster[[:space:]]?)+$'
+sanity_check_output 1 "$pat" "$out"
+
+# $prev should look like this:
+#    Ring: 10670.93 msgs/sec (+ve=53391 -ve=53373)
+stuff="${prev##*Ring: }"
+mps="${stuff% msgs/sec*}"
+
+if [ ${mps%.*} -ge 10 ] ; then
+    echo "OK: $mps msgs/sec >= 10 msgs/sec"
+else
+    echo "BAD: $mps msgs/sec < 10 msgs/sec"
+    exit 1
+fi
+
+stuff="${stuff#*msgs/sec (+ve=}"
+positive="${stuff%% *}"
+
+if [ $positive -gt 0 ] ; then
+    echo "OK: +ive ($positive) > 0"
+else
+    echo "BAD: +ive ($positive) = 0"
+    exit 1
+fi
+
+stuff="${stuff#*-ve=}"
+negative="${stuff%)}"
+
+if [ $negative -gt 0 ] ; then
+    echo "OK: -ive ($negative) > 0"
+else
+    echo "BAD: -ive ($negative) = 0"
+    exit 1
+fi
+
+perc_diff=$(( ($positive - $negative) * 100 / $positive ))
+perc_diff=${perc_diff#-}
+
+if [ $perc_diff -le 1 ] ; then
+    echo "OK: percentage difference between +ive and -ive ($perc_diff%) <= 1%"
+else
+    echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > 1%"
+    exit 1
+fi
+
+ctdb_test_exit
diff --git a/ctdb/tests/simple/53_ctdb_transaction.sh b/ctdb/tests/simple/53_ctdb_transaction.sh
new file mode 100755
index 00000000000..383db9ec785
--- /dev/null
+++ b/ctdb/tests/simple/53_ctdb_transaction.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_transaction test succeeds.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_transaction on each node with a 30 second
+   timeout.
+3. Ensure that all ctdb_transaction processes complete successfully.
+
+Expected results:
+
+* ctdb_transaction runs without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$VALGRIND ctdb_transaction --timelimit=30"
+
+echo "Running ctdb_transaction on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
diff --git a/ctdb/tests/simple/61_ctdb_persistent_safe.sh b/ctdb/tests/simple/61_ctdb_persistent_safe.sh
new file mode 100755
index 00000000000..6064e6fa3b6
--- /dev/null
+++ b/ctdb/tests/simple/61_ctdb_persistent_safe.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_persistent test succeeds for safe persistent writes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_persistent on each node with a 30 second
+   timeout.
+3. Ensure that all ctdb_persistent processes complete successfully.
+
+Expected results:
+
+* ctdb_persistent tests safe persistent writes without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$VALGRIND ctdb_persistent --timelimit=30"
+
+echo "Running ctdb_persistent on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit
diff --git a/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
new file mode 100755
index 00000000000..e0810212baa
--- /dev/null
+++ b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_persistent test succeeds for unsafe persistent writes.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_persistent on each node with a 30 second
+   timeout and with the --unsafe-writes option.
+3. Ensure that all ctdb_persistent processes complete successfully.
+
+Expected results:
+
+* ctdb_persistent tests unsafe persistent writes without error.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+t="$VALGRIND ctdb_persistent --unsafe-writes --timelimit=30"
+
+echo "Running ctdb_persistent on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+
+ctdb_test_exit