test: add test 54_ctdb_transaction_recovery.sh

This is like the 53_ctdb_transaction test, but it additionally
runs a loop with recoveries while the transactions are running.

When called like this, the transaction loops run for 10 minutes:

CTDB_TEST_TIMELIMIT=600 tests/scripts/run_tests tests/simple/54_ctdb_transaction_recovery.sh

The default timelimit is 30 seconds.

Michael

(This used to be ctdb commit 2ff2679e8f3d50ebf735f2c420898a84268bdc95)

1 files changed, 66 insertions, 0 deletions

diff --git a/ctdb/tests/simple/54_ctdb_transaction_recovery.sh b/ctdb/tests/simple/54_ctdb_transaction_recovery.sh
new file mode 100755
index 0000000000..13a92cccf4
--- /dev/null
+++ b/ctdb/tests/simple/54_ctdb_transaction_recovery.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that the ctdb_transaction test succeeds.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Run two copies of ctdb_transaction on each node with a 30 second
+   timeout.
+3. Ensure that all ctdb_transaction processes complete successfully.
+
+Expected results:
+
+* ctdb_transaction runs without error.
+EOF
+}
+
+recovery_loop()
+{
+	local COUNT=1
+
+	while true ; do
+		echo Recovery $COUNT
+		ctdb recover
+		sleep 2
+		COUNT=$((COUNT + 1))
+	done
+}
+
+recovery_loop_start()
+{
+	recovery_loop > /tmp/recloop.out &
+	RECLOOP_PID=$!
+	ctdb_test_exit_hook_add "kill $RECLOOP_PID >/dev/null 2>&1"
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+cluster_is_healthy
+
+try_command_on_node 0 "$CTDB listnodes"
+num_nodes=$(echo "$out" | wc -l)
+
+if test "x${CTDB_TEST_TIMELIMIT}" == "x" ; then
+	CTDB_TEST_TIMELIMIT=30
+fi
+
+t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}"
+
+echo "Starting recovery loop"
+recovery_loop_start
+
+echo "Running ctdb_transaction on all $num_nodes nodes."
+try_command_on_node -v -pq all "$t & $t"
+