Test suite: better debug info when the cluster is unexpectedly unhealthy.

cluster_is_healthy() is now run locally in tests and internally causes
_cluster_is_healthy() to be run on node 0.  When it detects that the
cluster is unhealthy and $ctdb_test_restart_scheduled is not true,
debug information is printed.  This replaces the previous use of
$CTDB_TEST_CLEANING_UP.

To avoid spurious debug on expected restarts, added scheduled
restarts to several tests.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit ee7caae3a55a64fb50cd28fa2fd4663c5dd83b4f)

34 files changed, 44 insertions, 35 deletions

diff --git a/ctdb/tests/simple/01_ctdb_version.sh b/ctdb/tests/simple/01_ctdb_version.sh
index 69a861b721..02cdf6ae70 100755
--- a/ctdb/tests/simple/01_ctdb_version.sh
+++ b/ctdb/tests/simple/01_ctdb_version.sh
@@ -31,7 +31,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 if ! try_command_on_node -v 0 "rpm -q ctdb" ; then
     echo "No useful output from rpm, SKIPPING rest of test".
diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh
index 935e676b92..093c1578f4 100755
--- a/ctdb/tests/simple/02_ctdb_listvars.sh
+++ b/ctdb/tests/simple/02_ctdb_listvars.sh
@@ -31,7 +31,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB listvars"
 
diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh
index 0ff0eef419..197df99b0f 100755
--- a/ctdb/tests/simple/03_ctdb_getvar.sh
+++ b/ctdb/tests/simple/03_ctdb_getvar.sh
@@ -32,7 +32,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB listvars"
 
diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh
index 336eb39287..e8eda3abc4 100755
--- a/ctdb/tests/simple/04_ctdb_setvar.sh
+++ b/ctdb/tests/simple/04_ctdb_setvar.sh
@@ -36,7 +36,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh
index 4e4df5d0ff..7bcb414241 100755
--- a/ctdb/tests/simple/05_ctdb_listnodes.sh
+++ b/ctdb/tests/simple/05_ctdb_listnodes.sh
@@ -28,7 +28,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB listnodes"
 
diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh
index 3989d00c79..5df8e84593 100755
--- a/ctdb/tests/simple/06_ctdb_getpid.sh
+++ b/ctdb/tests/simple/06_ctdb_getpid.sh
@@ -30,7 +30,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # This is an attempt at being independent of the number of nodes
 # reported by "ctdb getpid -n all".
diff --git a/ctdb/tests/simple/07_ctdb_process_exists.sh b/ctdb/tests/simple/07_ctdb_process_exists.sh
index da896714ce..6222ee6d24 100755
--- a/ctdb/tests/simple/07_ctdb_process_exists.sh
+++ b/ctdb/tests/simple/07_ctdb_process_exists.sh
@@ -34,7 +34,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 test_node=1
 
diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
index 1704032fc4..4f1e09689d 100755
--- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
+++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh
@@ -30,7 +30,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 cmd="$CTDB isnotrecmaster || true"
 try_command_on_node all "$cmd"
diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh
index bc315dfee0..e283683c91 100755
--- a/ctdb/tests/simple/09_ctdb_ping.sh
+++ b/ctdb/tests/simple/09_ctdb_ping.sh
@@ -33,7 +33,10 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
 
 try_command_on_node -v 0 "$CTDB ping -n 1"
 
@@ -42,7 +45,7 @@ sanity_check_output \
     '^response from 1 time=[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \
     "$out"
 
-try_command_on_node 0 "$CTDB shutdown -n 1"
+try_command_on_node -v 0 "$CTDB shutdown -n 1"
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status 1 disconnected
 
diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh
index c05526cd21..40cd795628 100755
--- a/ctdb/tests/simple/11_ctdb_ip.sh
+++ b/ctdb/tests/simple/11_ctdb_ip.sh
@@ -29,7 +29,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 echo "Getting list of public IPs..."
 try_command_on_node -v 1 $CTDB ip -n all
diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh
index 05f49caad8..76c3f1b875 100755
--- a/ctdb/tests/simple/12_ctdb_getdebug.sh
+++ b/ctdb/tests/simple/12_ctdb_getdebug.sh
@@ -28,7 +28,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
diff --git a/ctdb/tests/simple/13_ctdb_setdebug.sh b/ctdb/tests/simple/13_ctdb_setdebug.sh
index a0e0485778..b3a202d8bb 100755
--- a/ctdb/tests/simple/13_ctdb_setdebug.sh
+++ b/ctdb/tests/simple/13_ctdb_setdebug.sh
@@ -36,7 +36,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 get_debug ()
 {
diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh
index 257ca4a3ca..58c76be33a 100755
--- a/ctdb/tests/simple/14_ctdb_statistics.sh
+++ b/ctdb/tests/simple/14_ctdb_statistics.sh
@@ -31,7 +31,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 pattern='^(CTDB version 1|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts)|[[:space:]]+([[:alpha:]_]+_latency|max_reclock_[[:alpha:]]+)[[:space:]]+[[:digit:]-]+\.[[:digit:]]+[[:space:]]sec)$'
 
diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
index 5df5c07930..146807f294 100755
--- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh
+++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh
@@ -30,7 +30,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
index 7eacd12187..aaee472873 100755
--- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh
+++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
@@ -40,7 +40,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
index 6cda8c17f2..4ea34bd270 100755
--- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
+++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
@@ -36,7 +36,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/18_ctdb_freeze.sh b/ctdb/tests/simple/18_ctdb_freeze.sh
index 6e13139f53..7c558ad0cb 100755
--- a/ctdb/tests/simple/18_ctdb_freeze.sh
+++ b/ctdb/tests/simple/18_ctdb_freeze.sh
@@ -34,7 +34,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/19_ctdb_thaw.sh b/ctdb/tests/simple/19_ctdb_thaw.sh
index 86b1393755..a9b19659b6 100755
--- a/ctdb/tests/simple/19_ctdb_thaw.sh
+++ b/ctdb/tests/simple/19_ctdb_thaw.sh
@@ -38,7 +38,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 test_node=1
 
diff --git a/ctdb/tests/simple/20_ctdb_getmonmode.sh b/ctdb/tests/simple/20_ctdb_getmonmode.sh
index 76c5391f90..afbf24d5ef 100755
--- a/ctdb/tests/simple/20_ctdb_getmonmode.sh
+++ b/ctdb/tests/simple/20_ctdb_getmonmode.sh
@@ -36,7 +36,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/21_ctdb_disablemonitor.sh b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
index f9b31a3254..5bf7e83650 100755
--- a/ctdb/tests/simple/21_ctdb_disablemonitor.sh
+++ b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
@@ -42,7 +42,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 test_node=1
 
diff --git a/ctdb/tests/simple/22_ctdb_enablemonitor.sh b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
index 3f0b082d5b..6b40427640 100755
--- a/ctdb/tests/simple/22_ctdb_enablemonitor.sh
+++ b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
@@ -40,7 +40,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
index a78ab5e182..68a7f70f91 100755
--- a/ctdb/tests/simple/23_ctdb_moveip.sh
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@@ -35,7 +35,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index 224cb2a2b2..f8377caac2 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -29,7 +29,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 # Reset configuration
 ctdb_restart_when_done
diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh
index d499599fdf..4a6892cb5d 100755
--- a/ctdb/tests/simple/25_dumpmemory.sh
+++ b/ctdb/tests/simple/25_dumpmemory.sh
@@ -28,7 +28,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node -v 0 "$CTDB dumpmemory"
 
diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index fd1e1e6802..fae0c316d2 100755
--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -38,7 +38,10 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
 
 test_node=1
 
diff --git a/ctdb/tests/simple/31_ctdb_disable.sh b/ctdb/tests/simple/31_ctdb_disable.sh
index 98e52f59dd..c513c51257 100755
--- a/ctdb/tests/simple/31_ctdb_disable.sh
+++ b/ctdb/tests/simple/31_ctdb_disable.sh
@@ -35,7 +35,10 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
+
+# Reset configuration
+ctdb_restart_when_done
 
 echo "Getting list of public IPs..."
 try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
diff --git a/ctdb/tests/simple/32_ctdb_enable.sh b/ctdb/tests/simple/32_ctdb_enable.sh
index 5e189de37d..a6e60d8d89 100755
--- a/ctdb/tests/simple/32_ctdb_enable.sh
+++ b/ctdb/tests/simple/32_ctdb_enable.sh
@@ -42,7 +42,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 echo "Getting list of public IPs..."
 try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
diff --git a/ctdb/tests/simple/41_ctdb_ban.sh b/ctdb/tests/simple/41_ctdb_ban.sh
index 8299588ed8..40cd2c1196 100755
--- a/ctdb/tests/simple/41_ctdb_ban.sh
+++ b/ctdb/tests/simple/41_ctdb_ban.sh
@@ -39,7 +39,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 echo "Finding out which node is the recovery master..."
 try_command_on_node -v 0 "$CTDB recmaster"
diff --git a/ctdb/tests/simple/42_ctdb_unban.sh b/ctdb/tests/simple/42_ctdb_unban.sh
index dc3464d194..bd97ec8ff1 100755
--- a/ctdb/tests/simple/42_ctdb_unban.sh
+++ b/ctdb/tests/simple/42_ctdb_unban.sh
@@ -39,7 +39,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 echo "Finding out which node is the recovery master..."
 try_command_on_node -v 0 "$CTDB recmaster"
diff --git a/ctdb/tests/simple/51_ctdb_bench.sh b/ctdb/tests/simple/51_ctdb_bench.sh
index 75c63a772a..e629c01e93 100755
--- a/ctdb/tests/simple/51_ctdb_bench.sh
+++ b/ctdb/tests/simple/51_ctdb_bench.sh
@@ -32,7 +32,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
diff --git a/ctdb/tests/simple/52_ctdb_fetch.sh b/ctdb/tests/simple/52_ctdb_fetch.sh
index fbc49cda68..3f0ecffb5c 100755
--- a/ctdb/tests/simple/52_ctdb_fetch.sh
+++ b/ctdb/tests/simple/52_ctdb_fetch.sh
@@ -32,7 +32,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
diff --git a/ctdb/tests/simple/53_ctdb_transaction.sh b/ctdb/tests/simple/53_ctdb_transaction.sh
index 9189414c40..a7dac50971 100755
--- a/ctdb/tests/simple/53_ctdb_transaction.sh
+++ b/ctdb/tests/simple/53_ctdb_transaction.sh
@@ -28,7 +28,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
diff --git a/ctdb/tests/simple/61_ctdb_persistent_safe.sh b/ctdb/tests/simple/61_ctdb_persistent_safe.sh
index 845d180d80..4faf0a96d7 100755
--- a/ctdb/tests/simple/61_ctdb_persistent_safe.sh
+++ b/ctdb/tests/simple/61_ctdb_persistent_safe.sh
@@ -28,7 +28,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)
diff --git a/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
index 2a7e3abcd1..eb5706ed1e 100755
--- a/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
+++ b/ctdb/tests/simple/62_ctdb_persistent_unsafe.sh
@@ -28,7 +28,7 @@ ctdb_test_init "$@"
 
 set -e
 
-onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+cluster_is_healthy
 
 try_command_on_node 0 "$CTDB listnodes"
 num_nodes=$(echo "$out" | wc -l)