Clean up handling the of CTDB restarts in testcases.

Glitches during restarts of the CTDB cluster have been causing some
tests to fail.  This is because restarts are initiated in the body of
many tests.  This adds a simple function ctdb_restart_when_done, which
schedules a restart using an existing hook in the test exit code.
This function is now used in tests that need to restart CTDB.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit d440e83bb4f0c19c085915d0f0e87cc0dabbc569)

11 files changed, 43 insertions, 25 deletions

diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash
index 1930ae163a..210e6c4853 100644
--- a/ctdb/tests/scripts/ctdb_test_functions.bash
+++ b/ctdb/tests/scripts/ctdb_test_functions.bash
@@ -591,6 +591,10 @@ setup_ctdb ()
 
 restart_ctdb ()
 {
+    if [ "$1" = "-v" ] ; then
+	echo "Restarting CTDB (scheduled)..."
+    fi
+    
     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
 	daemons_stop
 	daemons_start $CTDB_TEST_NUM_DAEMONS
@@ -615,6 +619,13 @@ restart_ctdb ()
     echo "ctdb is ready"
 }
 
+ctdb_restart_when_done ()
+{
+    ctdb_test_exit_hook_add restart_ctdb -v
+}
+
+
+
 #######################################
 
 install_eventscript ()
diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh
index 9348308f2d..ceb6c2361d 100755
--- a/ctdb/tests/simple/04_ctdb_setvar.sh
+++ b/ctdb/tests/simple/04_ctdb_setvar.sh
@@ -38,6 +38,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 var="RecoverTimeout"
 
 try_command_on_node -v 0 $CTDB getvar $var
@@ -67,8 +70,7 @@ try_command_on_node -v 0 "$CTDB listvars | grep '^$var'"
 check="${out#*= }"
 
 if [ "$incr" != "$check" ] ; then
-    echo "Nope, that didn't work.  Restarting ctdb to get back into known state..."
-    restart_ctdb
+    echo "Nope, that didn't work..."
     exit 1
 fi
 
diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
index 50fc9c5cb9..634a39aec5 100755
--- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh
+++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh
@@ -42,6 +42,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 echo "Getting list of public IPs..."
 try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'"
 
@@ -112,7 +115,4 @@ else
     testfailures=1
 fi
 
-echo "Restarting cluster to restore configuration..."
-restart_ctdb
-
 ctdb_test_exit
diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
index 822c7023c2..bab45ddd50 100755
--- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
+++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
@@ -38,6 +38,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 echo "Getting list of public IPs..."
 try_command_on_node -v 0 "$CTDB ip -n all | sed -e '1d'"
 
@@ -65,8 +68,7 @@ while read ip pnn ; do
 done <<<"$out" # bashism to avoid problem setting variable in pipeline.
 
 if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then
-    echo "That worked!  Restarting cluster to restore configuration..."
-    restart_ctdb
+    echo "GOOD: That worked!"
 else
     echo "BAD: The remove IP address is still there!"
     testfailures=1
diff --git a/ctdb/tests/simple/18_ctdb_freeze.sh b/ctdb/tests/simple/18_ctdb_freeze.sh
index 5f2ccb0a13..009e2a745c 100755
--- a/ctdb/tests/simple/18_ctdb_freeze.sh
+++ b/ctdb/tests/simple/18_ctdb_freeze.sh
@@ -36,6 +36,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 echo "Freezing node $test_node"
@@ -44,8 +47,6 @@ try_command_on_node 0 $CTDB freeze -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node frozen
 
-echo "That worked!  Restarting cluster to restore configuration..."
-
-restart_ctdb
+echo "GOOD: That worked!"
 
 ctdb_test_exit
diff --git a/ctdb/tests/simple/20_ctdb_getmonmode.sh b/ctdb/tests/simple/20_ctdb_getmonmode.sh
index 802f302945..6a55341297 100755
--- a/ctdb/tests/simple/20_ctdb_getmonmode.sh
+++ b/ctdb/tests/simple/20_ctdb_getmonmode.sh
@@ -38,6 +38,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 try_command_on_node -v 0 $CTDB getmonmode -n $test_node
@@ -62,8 +65,6 @@ try_command_on_node -v 0 $CTDB disablemonitor -n $test_node
 
 onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
 
-echo "That worked!  Restarting cluster to restore configuration..."
-
-restart_ctdb
+echo "GOOD: That worked!"
 
 ctdb_test_exit
diff --git a/ctdb/tests/simple/21_ctdb_disablemonitor.sh b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
index 85855e4ca4..23ae052f5c 100755
--- a/ctdb/tests/simple/21_ctdb_disablemonitor.sh
+++ b/ctdb/tests/simple/21_ctdb_disablemonitor.sh
@@ -57,7 +57,7 @@ detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
 recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
 try_command_on_node $test_node touch "$recovered_flag"
 
-ctdb_test_exit_hook="onnode $test_node rm -vf $trigger"
+ctdb_test_exit_hook_add "onnode $test_node rm -vf $trigger"
 
 echo "Creating trigger file on node $test_node to make it unhealthy..."
 try_command_on_node $test_node touch "$trigger"
diff --git a/ctdb/tests/simple/22_ctdb_enablemonitor.sh b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
index f7d68091c8..98fc7e2700 100755
--- a/ctdb/tests/simple/22_ctdb_enablemonitor.sh
+++ b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
@@ -42,6 +42,9 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
+# Reset configuration
+ctdb_restart_when_done
+
 test_node=1
 
 # We need this for later, so we know how long to sleep.
@@ -64,7 +67,7 @@ detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
 recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
 try_command_on_node $test_node touch "$recovered_flag"
 
-ctdb_test_exit_hook="onnode $test_node rm -vf $trigger; restart_ctdb"
+ctdb_test_exit_hook_add "onnode $test_node rm -vf $trigger"
 
 echo "Creating trigger file on node $test_node to see if it goes unhealthy..."
 try_command_on_node $test_node touch "$trigger"
@@ -96,6 +99,6 @@ onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $mon
 
 try_command_on_node -v $test_node ls -l "$detected"
 
-echo "OK, that all worked.  Expect a restart..."
+echo "GOOD: That all worked..."
 
 ctdb_test_exit
diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
index c91bbbee9e..ca7fe11fb4 100755
--- a/ctdb/tests/simple/23_ctdb_moveip.sh
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@@ -37,8 +37,8 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-# Restart when done since things are likely to be broken.
-ctdb_test_exit_hook="restart_ctdb"
+# Reset configuration
+ctdb_restart_when_done
 
 try_command_on_node 0 "$CTDB listnodes | wc -l"
 num_nodes="$out"
@@ -100,6 +100,6 @@ else
     exit 1
 fi
 
-echo "OK, that worked... expect a restart..."
+echo "GOOD: That worked..."
 
 ctdb_test_exit
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
index 49ebda0742..c621a304ad 100755
--- a/ctdb/tests/simple/24_ctdb_getdbmap.sh
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -31,8 +31,8 @@ set -e
 
 onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 
-# Restart when done since things are likely to be broken.
-ctdb_test_exit_hook="restart_ctdb"
+# Reset configuration
+ctdb_restart_when_done
 
 make_temp_db_filename ()
 {
@@ -70,6 +70,6 @@ for i in $(seq 1 5) ; do
     fi
 done
 
-echo "OK, that worked... expect a restart..."
+echo "GOOD: That worked..."
 
 ctdb_test_exit
diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
index 72e0acd7a6..4287cba1fb 100755
--- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
+++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh
@@ -64,8 +64,6 @@ for i in ip disable enable "ban 0" unban listvars ; do
     fi
 done
 
-echo "That all looks OK.  Restarting cluster..."
-
-restart_ctdb
+echo "OK, that all worked.  Expect a restart..."
 
 ctdb_test_exit