From d741559fa6ba8d2077ecaa7053c1eab7d96aeb31 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 20 Nov 2008 20:40:01 +1100 Subject: Add some simple tests that can be run from within the tree. Signed-off-by: Martin Schwenke (This used to be ctdb commit eacb2ef82ea4809d874158756db973dd1e3fc8fc) --- ctdb/tests/scripts/ctdb_test_functions.bash | 333 ++++++++++++++++++++++++++++ ctdb/tests/scripts/run_tests | 46 ++++ ctdb/tests/scripts/test_wrap | 10 + 3 files changed, 389 insertions(+) create mode 100644 ctdb/tests/scripts/ctdb_test_functions.bash create mode 100755 ctdb/tests/scripts/run_tests create mode 100755 ctdb/tests/scripts/test_wrap (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash new file mode 100644 index 0000000000..d1886b8ebd --- /dev/null +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -0,0 +1,333 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +numnodes=3 + +export CTDB_NODES_SOCKETS="" +for i in $(seq 1 $numnodes) ; do + CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${PWD}/sock.${i}" +done + + +###################################################################### + +fail () +{ + echo "$*" + exit 1 +} + +###################################################################### + +#. /root/SOFS/autosofs/scripts/tester.bash + +test_begin () +{ + local name="$1" + + teststarttime=$(date '+%s') + testduration=0 + + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" + echo "Running test $name ($(date '+%T'))" + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" +} + +test_end () +{ + local name="$1" ; shift + local status="$1" ; shift + # "$@" is command-line + + local interp="SKIPPED" + local statstr=" (reason $*)" + if [ -n "$status" ] ; then + if [ $status -eq 0 ] ; then + interp="PASSED" + statstr="" + echo "ALL OK: $*" + else + interp="FAILED" + statstr=" (status $status)" + testfailures=$(($testfailures+1)) + fi + fi + + testduration=$(($(date +%s)-$teststarttime)) + + echo "==========================================================================" + echo "TEST ${interp}: ${name}${statstr}, duration: $testduration sec." + echo "==========================================================================" + +} + + +test_exit() { + exit $(($testfailures+0)) +} + +test_run () +{ + local name="$1" ; shift + + [ -n "$1" ] || set -- "$name" + + test_begin "$name" + + local status=0 + "$@" || status=$? + + test_end "$name" "$status" "$*" + + return $status +} + +######################################## + +# Sets: $out +try_command_on_node () +{ + local nodespec="$1" ; shift + local cmd="$*" + + out=$(onnode -q "$nodespec" "$cmd" 2>&1) || { + + echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" + echo "$out" + exit 1 + } +} + +sanity_check_output () +{ + local min_lines="$1" + local regexp="$2" # Should be anchored to match whole lines. + local output="$3" + + local ret=0 + + local num_lines=$(echo "$output" | wc -l) + echo "There are $num_lines lines of output" + if [ $num_lines -lt $min_lines ] ; then + echo "BAD: that's less than the required number (${min_lines})" + ret=1 + fi + + local status=0 + local unexpected # local doesn't pass through status of command on RHS. + unexpected=$(echo "$output" | egrep -v "$regexp") || status=$? + + # Note that this is reversed. + if [ $status -eq 0 ] ; then + echo "BAD: unexpected lines in output:" + echo "$unexpected" + ret=1 + else + echo "Output lines look OK" + fi + + return $ret +} + +####################################### + +# Wait until either timeout expires or command succeeds. The command +# will be tried once per second. +wait_until () +{ + local timeout="$1" ; shift # "$@" is the command... + + echo -n "|${timeout}|" + while [ $timeout -gt 0 ] ; do + if "$@" ; then + echo '|' + echo "OK" + return 0 + fi + echo -n . + timeout=$(($timeout - 1)) + sleep 1 + done + + echo "*TIMEOUT*" + + return 1 +} + +sleep_for () +{ + echo -n "|${1}|" + for i in $(seq 1 $1) ; do + echo -n '.' + sleep 1 + done + echo '|' +} + +_cluster_is_healthy () +{ + local out x count line + + out=$(ctdb -Y status 2>&1) || return 1 + + { + read x + count=0 + while read line ; do + count=$(($count + 1)) + [ "${line#:*:*:}" != "0:0:0:0:" ] && return 1 + done + [ $count -gt 0 ] && return $? + } <<<"$out" # Yay bash! +} + +cluster_is_healthy () +{ + if _cluster_is_healthy ; then + echo "Cluster is HEALTHY" + exit 0 + else + echo "Cluster is UNHEALTHY" + exit 1 + fi +} + +wait_until_healthy () +{ + local timeout="${1:-120}" + + echo "Waiting for cluster to become healthy..." + + wait_until 120 _cluster_is_healthy +} + +# Incomplete! Do not use! +node_has_status () +{ + local pnn="$1" + local status="$2" + + local bits + case "$status" in + banned) + bits="?:1:?:?" + ;; + unbanned) + bits="?:0:?:?" + ;; + disabled) + bits="?:?:1:?" + ;; + enabled) + bits="?:?:0:?" + ;; + *) + echo "node_has_status: unknown status \"$status\"" + return 1 + esac + + local out x line + + out=$(ctdb -Y status 2>&1) || return 1 + + { + read x + while read line ; do + [ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0 + done + return 1 + } <<<"$out" # Yay bash! +} + +wait_until_node_has_status () +{ + local pnn="$1" + local status="$2" + local timeout="${3:-30}" + + echo "Waiting until node $pnn has status \"$status\"..." + + wait_until $timeout node_has_status "$pnn" "$status" +} + +# Useful for superficially testing IP failover. +# IPs must be on nodes matching nodeglob. +ips_are_on_nodeglob () +{ + local nodeglob="$1" ; shift + local ips="$*" + + local out + + try_command_on_node 1 ctdb ip + + while read ip pnn ; do + for check in $ips ; do + if [ "$check" = "$ip" ] ; then + case "$pnn" in + ($nodeglob) : ;; + (*) return 1 ;; + esac + ips="${ips/${ip}}" # Remove from list + fi + done + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + ips="${ips// }" # Remove any spaces. + [ -z "$ips" ] +} + +wait_until_ips_are_on_nodeglob () +{ + echo "Waiting for IPs to fail over..." + + wait_until 60 ips_are_on_nodeglob "$@" +} + + +start_daemons () +{ + $CTDB_DIR/tests/start_daemons.sh $numnodes >$CTDB_DIR/var/daemons.log +} + +_restart_ctdb () +{ + if [ -e /etc/redhat-release ] ; then + service ctdb restart + else + /etc/init.d/ctdb restart + fi +} + +restart_ctdb () +{ + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + onnode all ctdb shutdown + start_daemons + else + onnode -pq all $TEST_WRAP _restart_ctdb + fi || return 1 + + onnode -q 1 $TEST_WRAP wait_until_healthy || return 1 + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "ctdb setvar RerecoveryTimeout 1" + + #echo "Sleeping to allow ctdb to settle..." + #sleep_for 10 + + echo "ctdb is ready" +} + +ctdb_test_exit () +{ + if ! onnode 0 $TEST_WRAP cluster_is_healthy ; then + echo "Restarting ctdb on all nodes to get back into known state..." + restart_ctdb + fi + + test_exit +} + +######################################## + +export PATH=/usr/local/autocluster:$PATH diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests new file mode 100755 index 0000000000..eab28f06ea --- /dev/null +++ b/ctdb/tests/scripts/run_tests @@ -0,0 +1,46 @@ +#!/bin/bash + +export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd) + +ctdb_bin_dir="${CTDB_DIR}/bin" +ctdb_tools_dir="${CTDB_DIR}/tools" +ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) + +PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}" + +export TEST_WRAP="${ctdb_test_scripts_dir}/test_wrap" + +. ctdb_test_functions.bash + +usage() { + cat < Date: Fri, 21 Nov 2008 19:01:48 +1100 Subject: Move tests/*.c to tests/src/*.c and adjust Makefile.in accordingly. Move setting of $CTDB_NODES_SOCKETS to tests/scripts/run_tests and make it only happen if $CTDB_TEST_REAL_CLUSTER is not set. Bugfix in function ips_are_on_nodeglob. New/proper implementations of functions stop_daemons and start_daemons, now called by function restart_ctdb. In start_daemons.sh, add public addresses file generation/usage, use new option --nopublicipcheck to ctdbd to avoid crazy behaviour and kill ctdbd more carefully to avoid killing real daemons on a real cluster - this should be able to coexist on a node of a real cluster. start_daemons.sh is temporarily incompatible with start_daemons function, but expecting to replace that script with function calls very soon anyway... Signed-off-by: Martin Schwenke (This used to be ctdb commit 4c54772c5c2fa7d2a25963379b5b96caf0c2521c) --- ctdb/tests/scripts/ctdb_test_functions.bash | 82 ++++++++++++++++++++++------- ctdb/tests/scripts/run_tests | 18 ++++++- 2 files changed, 79 insertions(+), 21 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index d1886b8ebd..3713c98e49 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -1,15 +1,5 @@ # Hey Emacs, this is a -*- shell-script -*- !!! :-) -numnodes=3 - -export CTDB_NODES_SOCKETS="" -for i in $(seq 1 $numnodes) ; do - CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${PWD}/sock.${i}" -done - - -###################################################################### - fail () { echo "$*" @@ -55,7 +45,7 @@ test_end () testduration=$(($(date +%s)-$teststarttime)) echo "==========================================================================" - echo "TEST ${interp}: ${name}${statstr}, duration: $testduration sec." + echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)" echo "==========================================================================" } @@ -258,7 +248,7 @@ ips_are_on_nodeglob () local out - try_command_on_node 1 ctdb ip + try_command_on_node 1 ctdb ip -n all while read ip pnn ; do for check in $ips ; do @@ -283,10 +273,68 @@ wait_until_ips_are_on_nodeglob () wait_until 60 ips_are_on_nodeglob "$@" } +stop_daemons () +{ + echo "Attempting to politely shutdown daemons..." + onnode 1 ctdb shutdown -n all || true + + echo "Sleeping for a while..." + sleep_for 1 + + if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then + echo "Killing remaining daemons..." + killall $CTDB_DIR/bin/ctdbd + + if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then + echo "Once more with feeling.." + killall -9 $CTDB_DIR/bin/ctdbd + fi + fi + + var_dir=$CTDB_DIR/tests/var + rm -rf $var_dir/test.db +} start_daemons () { - $CTDB_DIR/tests/start_daemons.sh $numnodes >$CTDB_DIR/var/daemons.log + local num_nodes="${1:-2}" # default is 2 nodes + shift # "$@" gets passed to ctdbd + + local var_dir=$CTDB_DIR/tests/var + + mkdir -p $var_dir/test.db/persistent + + local nodes=$var_dir/nodes.txt + local public_addresses=$var_dir/public_addresses.txt + rm -f $nodes $public_addresses + + local i + for i in $(seq 1 $num_nodes) ; do + if [ "${CTDB_USE_IPV6}x" != "x" ]; then + echo ::$i >> $nodes + ip addr add ::$i/128 dev lo + else + echo 127.0.0.$i >> $nodes + # 2 public addresses per node, just to make things interesting. + echo "192.0.2.$i/24 lo" >> $public_addresses + echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses + fi + done + + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --public-addresses $public_addresses --nopublicipcheck --event-script-dir=tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" + + echo "Starting $num_nodes ctdb daemons..." + for i in $(seq 1 $num_nodes) ; do + if [ $(id -u) -eq 0 ]; then + ctdb_options="$ctdb_options --public-interface=lo" + fi + + $VALGRIND bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" || return 1 + done + + if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then + ln -sf $var_dir/sock.1 /tmp/ctdb.socket || return 1 + fi } _restart_ctdb () @@ -301,8 +349,8 @@ _restart_ctdb () restart_ctdb () { if [ -n "$CTDB_NODES_SOCKETS" ] ; then - onnode all ctdb shutdown - start_daemons + stop_daemons + start_daemons $CTDB_NUM_NODES else onnode -pq all $TEST_WRAP _restart_ctdb fi || return 1 @@ -327,7 +375,3 @@ ctdb_test_exit () test_exit } - -######################################## - -export PATH=/usr/local/autocluster:$PATH diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index eab28f06ea..9827ebed63 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -1,9 +1,23 @@ #!/bin/bash export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd) +var_dir=$CTDB_DIR/tests/var -ctdb_bin_dir="${CTDB_DIR}/bin" -ctdb_tools_dir="${CTDB_DIR}/tools" +export CTDB_NUM_NODES=3 + +###################################################################### + +if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + export CTDB_NODES_SOCKETS="" + for i in $(seq 1 $CTDB_NUM_NODES) ; do + CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" + done +fi + +###################################################################### + +ctdb_bin_dir=$CTDB_DIR/bin +ctdb_tools_dir=$CTDB_DIR/tools ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}" -- cgit From 5d50f5a91c4ec5bd5dcfee9bb48e2587df077949 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 24 Nov 2008 17:47:09 +1100 Subject: New test 09_ctdb_ping.sh. Add documentation and command-line processing to all tests. New script ctdb_test_env sets up environment for tests, is now sourced by run_tests, and can also take a test on the command-line, complete with options. Various cleanups and improvements. Document tests that have been properly implemented in ctdbd.sh. Signed-off-by: Martin Schwenke (This used to be ctdb commit 826e85fe5291067b8d0b9c22918d63024aa6141c) --- ctdb/tests/scripts/ctdb_test_env | 27 ++++++ ctdb/tests/scripts/ctdb_test_functions.bash | 123 ++++++++++++++++++++-------- ctdb/tests/scripts/run_tests | 31 ++----- 3 files changed, 124 insertions(+), 57 deletions(-) create mode 100755 ctdb/tests/scripts/ctdb_test_env (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env new file mode 100755 index 0000000000..4fa3e30f79 --- /dev/null +++ b/ctdb/tests/scripts/ctdb_test_env @@ -0,0 +1,27 @@ +#!/bin/bash + +export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd) +var_dir=$CTDB_DIR/tests/var + +export CTDB_NUM_NODES=3 + +###################################################################### + +if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + export CTDB_NODES_SOCKETS="" + for i in $(seq 1 $CTDB_NUM_NODES) ; do + CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" + done +fi + +###################################################################### + +ctdb_bin_dir=$CTDB_DIR/bin +ctdb_tools_dir=$CTDB_DIR/tools +ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) + +PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}" + +export TEST_WRAP="${ctdb_test_scripts_dir}/test_wrap" + +"$@" diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 3713c98e49..bc660dbc23 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -10,7 +10,7 @@ fail () #. /root/SOFS/autosofs/scripts/tester.bash -test_begin () +ctdb_test_begin () { local name="$1" @@ -22,7 +22,7 @@ test_begin () echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" } -test_end () +ctdb_test_end () { local name="$1" ; shift local status="$1" ; shift @@ -50,33 +50,102 @@ test_end () } - -test_exit() { +test_exit () +{ exit $(($testfailures+0)) } -test_run () +ctdb_test_exit () +{ + if ! onnode 0 $TEST_WRAP cluster_is_healthy ; then + echo "Restarting ctdb on all nodes to get back into known state..." + restart_ctdb + fi + + test_exit +} + +ctdb_test_run () { local name="$1" ; shift [ -n "$1" ] || set -- "$name" - test_begin "$name" + ctdb_test_begin "$name" local status=0 "$@" || status=$? - test_end "$name" "$status" "$*" + ctdb_test_end "$name" "$status" "$*" return $status } +ctdb_test_usage() +{ + local status=${1:-2} + + cat <&1) || { @@ -85,12 +154,17 @@ try_command_on_node () echo "$out" exit 1 } + + if $verbose ; then + echo "Output of \"$cmd\":" + echo "$out" + fi } sanity_check_output () { local min_lines="$1" - local regexp="$2" # Should be anchored to match whole lines. + local regexp="$2" # Should be anchored as necessary. local output="$3" local ret=0 @@ -109,7 +183,7 @@ sanity_check_output () # Note that this is reversed. if [ $status -eq 0 ] ; then echo "BAD: unexpected lines in output:" - echo "$unexpected" + echo "$unexpected" | cat -A ret=1 else echo "Output lines look OK" @@ -126,7 +200,7 @@ wait_until () { local timeout="$1" ; shift # "$@" is the command... - echo -n "|${timeout}|" + echo -n "<${timeout}|" while [ $timeout -gt 0 ] ; do if "$@" ; then echo '|' @@ -145,7 +219,7 @@ wait_until () sleep_for () { - echo -n "|${1}|" + echo -n "=${1}|" for i in $(seq 1 $1) ; do echo -n '.' sleep 1 @@ -198,18 +272,12 @@ node_has_status () local bits case "$status" in - banned) - bits="?:1:?:?" - ;; - unbanned) - bits="?:0:?:?" - ;; - disabled) - bits="?:?:1:?" - ;; - enabled) - bits="?:?:0:?" - ;; + (disconnected) bits="1:?:?:?" ;; + (connected) bits="0:?:?:?" ;; + (banned) bits="?:1:?:?" ;; + (unbanned) bits="?:0:?:?" ;; + (disabled) bits="?:?:1:?" ;; + (enabled) bits="?:?:0:?" ;; *) echo "node_has_status: unknown status \"$status\"" return 1 @@ -366,12 +434,3 @@ restart_ctdb () echo "ctdb is ready" } -ctdb_test_exit () -{ - if ! onnode 0 $TEST_WRAP cluster_is_healthy ; then - echo "Restarting ctdb on all nodes to get back into known state..." - restart_ctdb - fi - - test_exit -} diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 9827ebed63..9779bf336d 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -1,28 +1,9 @@ #!/bin/bash -export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd) -var_dir=$CTDB_DIR/tests/var - -export CTDB_NUM_NODES=3 - -###################################################################### - -if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then - export CTDB_NODES_SOCKETS="" - for i in $(seq 1 $CTDB_NUM_NODES) ; do - CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" - done -fi - -###################################################################### - -ctdb_bin_dir=$CTDB_DIR/bin -ctdb_tools_dir=$CTDB_DIR/tools -ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) - -PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}" - -export TEST_WRAP="${ctdb_test_scripts_dir}/test_wrap" +# The ability of ctdb_test_env to take tests on the command-line is +# nice, but here we need to hack around it with that colon to reset +# the arguments that it sees. +. $(dirname $0)/ctdb_test_env : . ctdb_test_functions.bash @@ -53,8 +34,8 @@ done ###################################################################### for f; do - [ -x $f ] || fail "test $f is not executable" - test_run "$f" + [ -x $f ] || fail "test \"$f\" is not executable" + ctdb_test_run "$f" done test_exit -- cgit From a04094659cc85718129468a382dd6c1d50798b51 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 27 Nov 2008 18:11:22 +1100 Subject: 4 new tests. Hacked function node_has_status to support frozen/unfrozen via ctdb statistics command. Signed-off-by: Martin Schwenke (This used to be ctdb commit e040a989096cf7d5c0cdece1713ff903cb7568f8) --- ctdb/tests/scripts/ctdb_test_functions.bash | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index bc660dbc23..4df3729f42 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -270,7 +270,7 @@ node_has_status () local pnn="$1" local status="$2" - local bits + local bits fpat case "$status" in (disconnected) bits="1:?:?:?" ;; (connected) bits="0:?:?:?" ;; @@ -278,22 +278,31 @@ node_has_status () (unbanned) bits="?:0:?:?" ;; (disabled) bits="?:?:1:?" ;; (enabled) bits="?:?:0:?" ;; + (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; + (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; *) echo "node_has_status: unknown status \"$status\"" return 1 esac - local out x line + if [ -n "$bits" ] ; then + local out x line - out=$(ctdb -Y status 2>&1) || return 1 + out=$(ctdb -Y status 2>&1) || return 1 - { - read x - while read line ; do - [ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0 - done + { + read x + while read line ; do + [ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0 + done + return 1 + } <<<"$out" # Yay bash! + elif [ -n "$fpat" ] ; then + ctdb statistics -n "$pnn" | egrep -q "$fpat" + else + echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' return 1 - } <<<"$out" # Yay bash! + fi } wait_until_node_has_status () -- cgit From 3cdc0cb7083474b414c4dd41ee39c8c1864cce9b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 3 Dec 2008 15:48:24 +1100 Subject: $PATH only inludes $CTDB_DIR/bin if we're using local sockets. Rename $TEST_WRAP to $CTDB_TEST_WRAPPER - value now set using $CTDB_TEST_REMOTE_SCRIPTS_DIR if that is set. Signed-off-by: Martin Schwenke (This used to be ctdb commit a69545d7dec78eefb85a1598e5db4667cc210bf9) --- ctdb/tests/scripts/ctdb_test_env | 25 +++++++++++++++++-------- ctdb/tests/scripts/ctdb_test_functions.bash | 6 +++--- 2 files changed, 20 insertions(+), 11 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index 4fa3e30f79..9c35fedd0f 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -7,21 +7,30 @@ export CTDB_NUM_NODES=3 ###################################################################### +ctdb_bin_dir=$CTDB_DIR/bin +ctdb_tools_dir=$CTDB_DIR/tools +ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) + +PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}" + +###################################################################### + +if [ -n "$CTDB_TEST_REMOTE_SCRIPTS_DIR" ] ; then + CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_SCRIPTS_DIR}/test_wrap" +else + CTDB_TEST_WRAPPER="${ctdb_test_scripts_dir}/test_wrap" +fi +export CTDB_TEST_WRAPPER + if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then export CTDB_NODES_SOCKETS="" for i in $(seq 1 $CTDB_NUM_NODES) ; do CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" done -fi - -###################################################################### -ctdb_bin_dir=$CTDB_DIR/bin -ctdb_tools_dir=$CTDB_DIR/tools -ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) -PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}" + PATH="${ctdb_bin_dir}:${PATH}" +fi -export TEST_WRAP="${ctdb_test_scripts_dir}/test_wrap" "$@" diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 4df3729f42..9fb5aacf8c 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -57,7 +57,7 @@ test_exit () ctdb_test_exit () { - if ! onnode 0 $TEST_WRAP cluster_is_healthy ; then + if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then echo "Restarting ctdb on all nodes to get back into known state..." restart_ctdb fi @@ -429,10 +429,10 @@ restart_ctdb () stop_daemons start_daemons $CTDB_NUM_NODES else - onnode -pq all $TEST_WRAP _restart_ctdb + onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb fi || return 1 - onnode -q 1 $TEST_WRAP wait_until_healthy || return 1 + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 echo "Setting RerecoveryTimeout to 1" onnode -pq all "ctdb setvar RerecoveryTimeout 1" -- cgit From 9a4f7e4f4cbee9401cba9296c30a816cbd19cd65 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 3 Dec 2008 18:08:21 +1100 Subject: ctdb_test_init now contains a trap to force ctdb_test_exit to be run if the shell exits and ctdb_test_exit cancels this trap. This means that a testcase executing under set -e will call ctdb_test_exit on failure, allowing the cluster to be restarted if necessary so that following tests can complete successfully. ctdb_test_exit now respects $?, so a test will fail if the last thing executed before ctdb_test_exit failed - this probably means the above trap was triggered. Signed-off-by: Martin Schwenke (This used to be ctdb commit 805a426aaee5ecfc5bd1c097069fe58f8241dfe2) --- ctdb/tests/scripts/ctdb_test_functions.bash | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 9fb5aacf8c..bcf401c1de 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -57,6 +57,12 @@ test_exit () ctdb_test_exit () { + local status=$? + + trap - 0 + + [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then echo "Restarting ctdb on all nodes to get back into known state..." restart_ctdb @@ -132,6 +138,8 @@ ctdb_test_init () testfailures=0 ctdb_test_cmd_options $@ + + trap "ctdb_test_exit" 0 } ######################################## -- cgit From 805c5bf1f33d250fa1f9f6fac0e46ff9358c8dee Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 4 Dec 2008 17:19:51 +1100 Subject: New test for getmonmode. Overload node_has_status some more to support checking the monitoring mode. Signed-off-by: Martin Schwenke (This used to be ctdb commit 4e1c079deb0aafb99d4114bb6504ff5ba1cbdeb4) --- ctdb/tests/scripts/ctdb_test_functions.bash | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index bcf401c1de..96a0f9f56e 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -272,13 +272,13 @@ wait_until_healthy () wait_until 120 _cluster_is_healthy } -# Incomplete! Do not use! +# This function is becoming nicely overloaded. Soon it will collapse! :-) node_has_status () { local pnn="$1" local status="$2" - local bits fpat + local bits fpat mpat case "$status" in (disconnected) bits="1:?:?:?" ;; (connected) bits="0:?:?:?" ;; @@ -288,6 +288,8 @@ node_has_status () (enabled) bits="?:?:0:?" ;; (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; + (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; + (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; *) echo "node_has_status: unknown status \"$status\"" return 1 @@ -307,6 +309,8 @@ node_has_status () } <<<"$out" # Yay bash! elif [ -n "$fpat" ] ; then ctdb statistics -n "$pnn" | egrep -q "$fpat" + elif [ -n "$mpat" ] ; then + ctdb getmonmode -n "$pnn" | egrep -q "$mpat" else echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' return 1 -- cgit From c9ca8ccc2312757f8c8a59c79997976796120417 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 8 Dec 2008 08:15:18 +1100 Subject: When running with local daemons, provided there is more than 2 of them, randomly pick a single node that will not have any public IPs assigned. This will make life a bit more interesting and will simulate what happens on real clusters with a management node. Some tests were disabling a node to implicitly trigger a ctdb restart - now use an explicit restart of ctdb when it is required. 17_ctdb_config_delete_ip.sh now randomly chooses a public IP on any node to disable - this works around a problem where the hardcoded node might not have any public addresses. Signed-off-by: Martin Schwenke (This used to be ctdb commit 3d59783c0e9478f4766c380945d6200fc654f5d9) --- ctdb/tests/scripts/ctdb_test_functions.bash | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 96a0f9f56e..08f0274f8f 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -397,6 +397,11 @@ start_daemons () local public_addresses=$var_dir/public_addresses.txt rm -f $nodes $public_addresses + # If there are (strictly) greater than 2 nodes then we'll randomly + # choose a node to have no public addresses. + local no_public_ips=-1 + [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes + 1)) + local i for i in $(seq 1 $num_nodes) ; do if [ "${CTDB_USE_IPV6}x" != "x" ]; then @@ -404,9 +409,11 @@ start_daemons () ip addr add ::$i/128 dev lo else echo 127.0.0.$i >> $nodes - # 2 public addresses per node, just to make things interesting. - echo "192.0.2.$i/24 lo" >> $public_addresses - echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses + # 2 public addresses on most nodes, just to make things interesting. + if [ $i -ne $no_public_ips ] ; then + echo "192.0.2.$i/24 lo" >> $public_addresses + echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses + fi fi done -- cgit From 60f86400acfa33407d989639f7d674fd5dffec59 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 9 Dec 2008 18:20:11 +1100 Subject: Added use of $ctdb_test_exit_hook to function ctdb_test_exit. Removed sleeps from ban/unban tests. Now expect "ctdb ping" to return false if it fails, so made relevant change to 09_ctdb_ping.sh. New functions install_eventscript and uninstall_eventscript. New setup/cleanup tests 00_ctdb_install_eventscript.sh and 99_ctdb_uninstall_eventscript.sh. New test 21_ctdb_disablemonitor.sh, which is incredibly complex. Signed-off-by: Martin Schwenke (This used to be ctdb commit 23bffef2295772f5b795236d60b7fb6ea754b7fb) --- ctdb/tests/scripts/ctdb_test_functions.bash | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 08f0274f8f..0c2c053903 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -63,6 +63,8 @@ ctdb_test_exit () [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + eval "$ctdb_test_exit_hook" + if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then echo "Restarting ctdb on all nodes to get back into known state..." restart_ctdb @@ -280,6 +282,8 @@ node_has_status () local bits fpat mpat case "$status" in + (unhealthy) bits="?:?:?:1" ;; + (healthy) bits="?:?:?:0" ;; (disconnected) bits="1:?:?:?" ;; (connected) bits="0:?:?:?" ;; (banned) bits="?:1:?:?" ;; @@ -462,3 +466,31 @@ restart_ctdb () echo "ctdb is ready" } +install_eventscript () +{ + local script_name="$1" + local script_contents="$2" + + if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + # The quoting here is *very* fragile. However, we do + # experience the joy of installing a short script using + # onnode, and without needing to know the IP addresses of the + # nodes. + onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" + else + f="${CTDB_DIR}/tests/events.d/${script_name}" + echo "$script_contents" >"$f" + chmod 755 "$f" + fi +} + +uninstall_eventscript () +{ + local script_name="$1" + + if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" + else + rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" + fi +} -- cgit From ceefb6a02905f0fd4daee55b1b3bc63c6f3ae1d4 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 10 Dec 2008 16:13:42 +1100 Subject: With local daemons the sockets are now numbered starting from 0. Fix setup of local daemons so that it correctly assigns no public IPs to a single node each time. Separate out daemon_setup so that the selection of the node with no public IPs is only done once at the beginning of testing. Clean up all current tests, mostly with a view to ensuring that a node selected for testing some kind of failover actually has public addresses assigned. Reenabled 01_ctdb_version.sh - it now passes if rpm doesn't do anything useful on the node. Signed-off-by: Martin Schwenke (This used to be ctdb commit 86807e8b7b179cbe87e559fb3b1f02c8b1990dc4) --- ctdb/tests/scripts/ctdb_test_env | 2 +- ctdb/tests/scripts/ctdb_test_functions.bash | 63 +++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 13 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index 9c35fedd0f..bb7a14af67 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -24,7 +24,7 @@ export CTDB_TEST_WRAPPER if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then export CTDB_NODES_SOCKETS="" - for i in $(seq 1 $CTDB_NUM_NODES) ; do + for i in $(seq 0 $(($CTDB_NUM_NODES -1))) ; do CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" done diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 0c2c053903..02fbf986a1 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -162,7 +162,7 @@ try_command_on_node () echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" echo "$out" - exit 1 + return 1 } if $verbose ; then @@ -366,7 +366,9 @@ wait_until_ips_are_on_nodeglob () wait_until 60 ips_are_on_nodeglob "$@" } -stop_daemons () +####################################### + +daemons_stop () { echo "Attempting to politely shutdown daemons..." onnode 1 ctdb shutdown -n all || true @@ -388,10 +390,9 @@ stop_daemons () rm -rf $var_dir/test.db } -start_daemons () +daemons_setup () { local num_nodes="${1:-2}" # default is 2 nodes - shift # "$@" gets passed to ctdbd local var_dir=$CTDB_DIR/tests/var @@ -399,12 +400,14 @@ start_daemons () local nodes=$var_dir/nodes.txt local public_addresses=$var_dir/public_addresses.txt - rm -f $nodes $public_addresses + local no_public_addresses=$var_dir/no_public_addresses.txt + rm -f $nodes $public_addresses $no_public_addresses # If there are (strictly) greater than 2 nodes then we'll randomly # choose a node to have no public addresses. local no_public_ips=-1 - [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes + 1)) + [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes)) + echo "$no_public_ips" >$no_public_addresses local i for i in $(seq 1 $num_nodes) ; do @@ -414,29 +417,56 @@ start_daemons () else echo 127.0.0.$i >> $nodes # 2 public addresses on most nodes, just to make things interesting. - if [ $i -ne $no_public_ips ] ; then + if [ $(($i - 1)) -ne $no_public_ips ] ; then echo "192.0.2.$i/24 lo" >> $public_addresses echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses fi fi done +} + +daemons_start () +{ + local num_nodes="${1:-2}" # default is 2 nodes + shift # "$@" gets passed to ctdbd + + local var_dir=$CTDB_DIR/tests/var + + local nodes=$var_dir/nodes.txt + local public_addresses=$var_dir/public_addresses.txt + local no_public_addresses=$var_dir/no_public_addresses.txt + + local no_public_ips=-1 + [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --public-addresses $public_addresses --nopublicipcheck --event-script-dir=tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" echo "Starting $num_nodes ctdb daemons..." - for i in $(seq 1 $num_nodes) ; do + if [ "$no_public_ips" != -1 ] ; then + echo "Node $no_public_ips will have no public IPs." + fi + + for i in $(seq 0 $(($num_nodes - 1))) ; do if [ $(id -u) -eq 0 ]; then ctdb_options="$ctdb_options --public-interface=lo" fi + if [ $i -eq $no_public_ips ] ; then + ctdb_options="$ctdb_options --public-addresses=/dev/null" + else + ctdb_options="$ctdb_options --public-addresses=$public_addresses" + fi + $VALGRIND bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" || return 1 done if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then - ln -sf $var_dir/sock.1 /tmp/ctdb.socket || return 1 + ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 fi } +####################################### + _restart_ctdb () { if [ -e /etc/redhat-release ] ; then @@ -446,11 +476,18 @@ _restart_ctdb () fi } +setup_ctdb () +{ + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_setup $CTDB_NUM_NODES + fi +} + restart_ctdb () { if [ -n "$CTDB_NODES_SOCKETS" ] ; then - stop_daemons - start_daemons $CTDB_NUM_NODES + daemons_stop + daemons_start $CTDB_NUM_NODES else onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb fi || return 1 @@ -466,6 +503,8 @@ restart_ctdb () echo "ctdb is ready" } +####################################### + install_eventscript () { local script_name="$1" -- cgit From 6c6fd475282300ad870eaeeb0bc98befc7901cf1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 12 Dec 2008 17:25:38 +1100 Subject: Rename $CTDB_NUM_NODES to $CTDB_TEST_NUM_DAEMONS and only set it if $CTDB_TEST_REAL_CLUSTER is not set. After a ctdb restart, force a recovery to attempt to help tests that follows. Signed-off-by: Martin Schwenke (This used to be ctdb commit 497c40f69e06776861a780500da1952eb7ea8fc1) --- ctdb/tests/scripts/ctdb_test_env | 6 +++--- ctdb/tests/scripts/ctdb_test_functions.bash | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index bb7a14af67..82aa3da9f5 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -3,8 +3,6 @@ export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd) var_dir=$CTDB_DIR/tests/var -export CTDB_NUM_NODES=3 - ###################################################################### ctdb_bin_dir=$CTDB_DIR/bin @@ -23,8 +21,10 @@ fi export CTDB_TEST_WRAPPER if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + export CTDB_TEST_NUM_DAEMONS=3 + export CTDB_NODES_SOCKETS="" - for i in $(seq 0 $(($CTDB_NUM_NODES -1))) ; do + for i in $(seq 0 $(($CTDB_TEST_NUM_DAEMONS -1))) ; do CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" done diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 02fbf986a1..4e9e7316bb 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -479,7 +479,7 @@ _restart_ctdb () setup_ctdb () { if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_setup $CTDB_NUM_NODES + daemons_setup $CTDB_TEST_NUM_DAEMONS fi } @@ -487,7 +487,7 @@ restart_ctdb () { if [ -n "$CTDB_NODES_SOCKETS" ] ; then daemons_stop - daemons_start $CTDB_NUM_NODES + daemons_start $CTDB_TEST_NUM_DAEMONS else onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb fi || return 1 @@ -497,6 +497,12 @@ restart_ctdb () echo "Setting RerecoveryTimeout to 1" onnode -pq all "ctdb setvar RerecoveryTimeout 1" + # In recent versions of CTDB, forcing a recovery like this blocks + # until the recovery is complete. Hopefully this will help the + # cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 ctdb recover + #echo "Sleeping to allow ctdb to settle..." #sleep_for 10 -- cgit From 3e1e589e58d7f5ff4a014cc5ee5d0311f21317f5 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 12 Dec 2008 18:44:21 +1100 Subject: Add a recovery to ctdb_test_exit to improve test stability. Signed-off-by: Martin Schwenke (This used to be ctdb commit b7be3de004cb05dad05704096e2a75e128952b18) --- ctdb/tests/scripts/ctdb_test_functions.bash | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 4e9e7316bb..60b8e9de77 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -68,6 +68,13 @@ ctdb_test_exit () if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then echo "Restarting ctdb on all nodes to get back into known state..." restart_ctdb + else + # This could be made unconditional but then we might get + # duplication from the recovery in restart_ctdb. We want to + # leave the recovery in restart_ctdb so that future tests that + # might do a manual restart mid-test will benefit. + echo "Forcing a recovery..." + onnode 0 ctdb recover fi test_exit -- cgit From b3dbc3f3aa0cb9e2ae338ea44dee0d4c7b4e51b5 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 15 Dec 2008 17:52:12 +1100 Subject: 3 new tests. 24_ctdb_getdbmap.sh is only 1/2 implemented but does something vaguely useful. ctdb_test_exit unsets $ctdb_test_exit_hook. Fix bug in 17_ctdb_config_delete_ip.sh. Signed-off-by: Martin Schwenke (This used to be ctdb commit f90f6e19952d58b8590fe550ecf2308bd9b065dc) --- ctdb/tests/scripts/ctdb_test_functions.bash | 1 + 1 file changed, 1 insertion(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 60b8e9de77..b086b7fb5c 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -64,6 +64,7 @@ ctdb_test_exit () [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status eval "$ctdb_test_exit_hook" + unset ctdb_test_exit_hook if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then echo "Restarting ctdb on all nodes to get back into known state..." -- cgit From 12c16dd9b0a669c63f550f81c7da0c2699df21e9 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 17 Dec 2008 16:59:23 +1100 Subject: ctdb_test_env now sets $CTDB_TIMEOUT. Fixed 26_ctdb_config_check_error_on_unreachable_ctdb.sh to work with new generic error message when trying to talk to disconnected node. Signed-off-by: Martin Schwenke (This used to be ctdb commit 33afe9bae732e62994e5957ee143a9c49571898b) --- ctdb/tests/scripts/ctdb_test_env | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index 82aa3da9f5..398caf36c8 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -11,6 +11,8 @@ ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}" +export CTDB_TIMEOUT=60 + ###################################################################### if [ -n "$CTDB_TEST_REMOTE_SCRIPTS_DIR" ] ; then -- cgit From c572c11d3a55258786029ad1f14056ca05581afd Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 18 Dec 2008 11:28:43 +1100 Subject: Remove bench.sh test and replace with new test 51_ctdb_bench.sh. Function try_command_on_node node passes any options other than -v to onnode. Minor update to 14_ctdb_statistics.sh. Signed-off-by: Martin Schwenke (This used to be ctdb commit e627f8e5fe8e2e3ea423b7dbd12d74597fb4983b) --- ctdb/tests/scripts/ctdb_test_functions.bash | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index b086b7fb5c..450eb48dc4 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -160,13 +160,20 @@ try_command_on_node () local nodespec="$1" ; shift local verbose=false - if [ "$nodespec" = "-v" ] ; then - verbose=true + local onnode_opts="" + + while [ "${nodespec#-}" != "$nodespec" ] ; do + if [ "$nodespec" = "-v" ] ; then + verbose=true + else + onnode_opts="$nodespec" + fi nodespec="$1" ; shift - fi + done + local cmd="$*" - out=$(onnode -q "$nodespec" "$cmd" 2>&1) || { + out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || { echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" echo "$out" -- cgit From a70327ce1a801afa05ccbace618255f320824288 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 8 Jan 2009 17:12:03 +1100 Subject: Delete some unstructured tests ({fetch,peristent,transaction}.sh) and replace them with new simple tests (52_ctdb_fetch.sh, 53_ctdb_transaction.sh, 61_ctdb_persistent_safe.sh, 62_ctdb_persistent_unsafe.sh). Remove "_simple" from some test filenames in the simple subdirectory - that's redundant. Always run ctdb as $CTDB to allow $VALGRIND magic to be used. Use pgrep/pkill to detect/kill local daemons so those running under valgrind can be found too - to support this, always run local daemons with the full path to the executable. run_tests now supports -s option to print sumamry when done - with more and more tests, it is getting hard to follow progress. Sort the output of commands in 06_ctdb_getpid.sh to make sure they compare nicely and also allow the processes' executables to be called "memcheck" to catch those running under valgrind. Remove redundant calls to onnode in commands run from calls try_command_on_node in some tests. 41_ctdb_ban.sh and 42_ctdb_unban.sh avoid banning the recmaster, since this causes the recmaster to be reassigned and all nodes to be unbanned. Minor cleanups. Signed-off-by: Martin Schwenke (This used to be ctdb commit 33cdf3e4bcfadf8e20822ca352babf7acca16821) --- ctdb/tests/scripts/ctdb_test_env | 7 +++++++ ctdb/tests/scripts/ctdb_test_functions.bash | 15 +++++++-------- ctdb/tests/scripts/run_tests | 27 +++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 10 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index 398caf36c8..f4bb86c51a 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -22,6 +22,8 @@ else fi export CTDB_TEST_WRAPPER +# If we're not running on a real cluster then we need a local copy of +# ctdb (and other stuff) in $PATH and we will use local daemons. if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then export CTDB_TEST_NUM_DAEMONS=3 @@ -34,5 +36,10 @@ if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then PATH="${ctdb_bin_dir}:${PATH}" fi +# If $VALGRIND is set then use it whenever ctdb is called, but only if +# $CTDB is not already set. +[ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb" + +###################################################################### "$@" diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 450eb48dc4..017fc4c615 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -8,8 +8,6 @@ fail () ###################################################################### -#. /root/SOFS/autosofs/scripts/tester.bash - ctdb_test_begin () { local name="$1" @@ -391,17 +389,17 @@ daemons_stop () echo "Sleeping for a while..." sleep_for 1 - if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then + if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then echo "Killing remaining daemons..." - killall $CTDB_DIR/bin/ctdbd + pkill -f $CTDB_DIR/bin/ctdbd - if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then + if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then echo "Once more with feeling.." - killall -9 $CTDB_DIR/bin/ctdbd + pkill -9 $CTDB_DIR/bin/ctdbd fi fi - var_dir=$CTDB_DIR/tests/var + local var_dir=$CTDB_DIR/tests/var rm -rf $var_dir/test.db } @@ -472,7 +470,8 @@ daemons_start () ctdb_options="$ctdb_options --public-addresses=$public_addresses" fi - $VALGRIND bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" || return 1 + # Need the $PWD so we can use "pkill -f" to kill the daemons. + $VALGRIND $PWD/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1 done if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 9779bf336d..f337240772 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -17,7 +17,9 @@ EOF ###################################################################### -temp=$(getopt -n "$prog" -o "xh" -l help -- "$@") +with_summary=false + +temp=$(getopt -n "$prog" -o "xhs" -l help -- "$@") [ $? != 0 ] && usage @@ -26,6 +28,7 @@ eval set -- "$temp" while true ; do case "$1" in -x) set -x; shift ;; + -s) with_summary=true ; shift ;; --) shift ; break ;; -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable. esac @@ -33,9 +36,29 @@ done ###################################################################### +tests_total=0 +tests_passed=0 +summary="" + +rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@') +ww=$((rows - 7)) + for f; do [ -x $f ] || fail "test \"$f\" is not executable" - ctdb_test_run "$f" + tests_total=$(($tests_total + 1)) + if ctdb_test_run "$f" ; then + tests_passed=$(($tests_passed + 1)) + t="PASSED" + else + t="FAILED" + fi + summary=$(printf "%s\n%-${ww}s%s" "$summary" "$f" "$t") done +if $with_summary ; then + echo "$summary" + echo + echo "${tests_passed}/${tests_total} tests passed" +fi + test_exit -- cgit From 91972a6bfc18044298e77008a97ce58ecc6e6aa9 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 9 Jan 2009 17:52:38 +1100 Subject: Add tests/README. ctdb_test_env and, therefore, run_tests can now be run from the tests subdirectory... and most other sensible places. Rename $CTDB_TEST_REMOTE_SCRIPTS_DIR to $CTDB_TEST_REMOTE_DIR since it is now intented that this directory can contain test binaries too. daemons_start() now passes a full path to the events.d directory when starting ctdbd. Also fixed the path to ctdbd. 41_ctdb_ban.sh and 42_ctdb_unban.sh fail immediately if they fail to select a test node. Signed-off-by: Martin Schwenke (This used to be ctdb commit 3ecce31d3a3f72b9fa851ac99291865c119e551e) --- ctdb/tests/scripts/ctdb_test_env | 8 ++++---- ctdb/tests/scripts/ctdb_test_functions.bash | 6 +++--- ctdb/tests/scripts/run_tests | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index f4bb86c51a..a5030852a4 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -1,13 +1,13 @@ #!/bin/bash -export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd) +ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) +export CTDB_DIR=$(dirname $(dirname $ctdb_test_scripts_dir)) var_dir=$CTDB_DIR/tests/var ###################################################################### ctdb_bin_dir=$CTDB_DIR/bin ctdb_tools_dir=$CTDB_DIR/tools -ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}" @@ -15,8 +15,8 @@ export CTDB_TIMEOUT=60 ###################################################################### -if [ -n "$CTDB_TEST_REMOTE_SCRIPTS_DIR" ] ; then - CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_SCRIPTS_DIR}/test_wrap" +if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then + CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" else CTDB_TEST_WRAPPER="${ctdb_test_scripts_dir}/test_wrap" fi diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 017fc4c615..b0b930ef36 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -452,7 +452,7 @@ daemons_start () local no_public_ips=-1 [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" echo "Starting $num_nodes ctdb daemons..." if [ "$no_public_ips" != -1 ] ; then @@ -470,8 +470,8 @@ daemons_start () ctdb_options="$ctdb_options --public-addresses=$public_addresses" fi - # Need the $PWD so we can use "pkill -f" to kill the daemons. - $VALGRIND $PWD/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1 + # Need full path so we can use "pkill -f" to kill the daemons. + $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1 done if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index f337240772..8df1aecd25 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -30,7 +30,7 @@ while true ; do -x) set -x; shift ;; -s) with_summary=true ; shift ;; --) shift ; break ;; - -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable. + *) usage ;; esac done -- cgit From 2e87ed4e9d3921b47f6c6f96374733058658d198 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 9 Jan 2009 18:15:06 +1100 Subject: Test binaries now go in tests/bin and ctdb_test_env now adds this directory to $PATH if local daemons are being used. Signed-off-by: Martin Schwenke (This used to be ctdb commit a497010f67d6a8e68f4d6d7e516b88d2261b1062) --- ctdb/tests/scripts/ctdb_test_env | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index a5030852a4..5c3006e25d 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -6,7 +6,6 @@ var_dir=$CTDB_DIR/tests/var ###################################################################### -ctdb_bin_dir=$CTDB_DIR/bin ctdb_tools_dir=$CTDB_DIR/tools PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}" @@ -32,8 +31,7 @@ if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" done - - PATH="${ctdb_bin_dir}:${PATH}" + PATH="${CTDB_DIR}/bin:${CTDB_DIR}/tests/bin:${PATH}" fi # If $VALGRIND is set then use it whenever ctdb is called, but only if -- cgit From fc1f71ce0f9c4c2eec1cf098547dfe540dd055d1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 16 Jan 2009 15:12:40 +1100 Subject: 23_ctdb_moveip.sh sanity checks the list of IPs/nodes instead of looping forever on older versions of CTDB. Signed-off-by: Martin Schwenke (This used to be ctdb commit 03dfcf9f284c9926479a8dd4e5acf1f5d2b964aa) --- ctdb/tests/scripts/ctdb_test_functions.bash | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index b0b930ef36..f1e141745c 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -215,6 +215,28 @@ sanity_check_output () return $ret } +sanity_check_ips () +{ + local ips="$1" # Output of "ctdb ip -n all" + + echo "Sanity checking IPs..." + + local x ipp prev + prev="" + while read x ipp ; do + [ "$ipp" = "-1" ] && break + if [ -n "$prev" -a "$ipp" != "$prev" ] ; then + echo "OK" + return 0 + fi + prev="$ipp" + done <<<"$ips" + + echo "BAD: a node was -1 or IPs are only assigned to one node" + echo "Are you running an old version of CTDB?" + return 1 +} + ####################################### # Wait until either timeout expires or command succeeds. The command -- cgit From 7c7c5b3489f97154b2d25cefafb55e04c8846c5d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 1 May 2009 17:40:45 +1000 Subject: Bug fixes for tests: simple/12_ctdb_getdebug.sh and scripts/test_wrap. simple/12_ctdb_getdebug.sh now recognises output with multi-digit node numbers. Sharing the ctdb directory via NFS and testing on a real cluster by setting CTDB_TEST_REAL_CLUSTER didn't work by default. The fix is to hack scripts/test_wrap so that it tries to find a valid bin directory next to the directory containing it is in. Signed-off-by: Martin Schwenke (This used to be ctdb commit ea2ca769e1d1068fbbad843750b19acfd87360e0) --- ctdb/tests/scripts/test_wrap | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/test_wrap b/ctdb/tests/scripts/test_wrap index 35ad41830f..6c730ef768 100755 --- a/ctdb/tests/scripts/test_wrap +++ b/ctdb/tests/scripts/test_wrap @@ -5,6 +5,12 @@ PATH="$(dirname $0):${PATH}" +f="ctdb_bench" +if [ ! $(which $f >/dev/null 2>&1) ] ; then + d=$(dirname $(dirname $0))/bin + [ -x "$d/$f" ] && PATH="$d:$PATH" +fi + . ctdb_test_functions.bash "$@" -- cgit From ffff61c13bacbadba34b99eee929e1dc9f29706f Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 16 Jun 2009 12:47:59 +1000 Subject: New tests for NFS and CIFS tickles. New tests/complex/ subdirectory contains 2 new tests to ensure that NFS and CIFS connections are tracked by CTDB and that tickle resets are sent when a node is disabled. Changes to ctdb_test_functions.bash to support these tests. Signed-off-by: Martin Schwenke (This used to be ctdb commit 31cc46eb157ca1301312f14879e4fb4da7d81088) --- ctdb/tests/scripts/ctdb_test_functions.bash | 72 ++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index f1e141745c..1930ae163a 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -61,7 +61,7 @@ ctdb_test_exit () [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status - eval "$ctdb_test_exit_hook" + eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then @@ -79,6 +79,11 @@ ctdb_test_exit () test_exit } +ctdb_test_exit_hook_add () +{ + ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" +} + ctdb_test_run () { local name="$1" ; shift @@ -150,6 +155,14 @@ ctdb_test_init () trap "ctdb_test_exit" 0 } +ctdb_test_check_real_cluster () +{ + [ -n "$CTDB_TEST_REAL_CLUSTER" ] && return 0 + + echo "ERROR: This test must be run on a real/virtual cluster, not local daemons." + return 1 +} + ######################################## # Sets: $out @@ -401,6 +414,63 @@ wait_until_ips_are_on_nodeglob () wait_until 60 ips_are_on_nodeglob "$@" } +get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" + out=$(netstat -tanp | + egrep "$pat" | + awk '{ print $4 }') + + [ -n "$out" ] +} + +wait_until_get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + echo "Waiting for ${prog} to establish connection to ${dst_socket}..." + + wait_until 5 get_src_socket "$@" +} + +# filename will be in $tcpdump_filename, pid in $tcpdump_pid +# By default, wait for 1 matching packet on any interface. +tcpdump_start () +{ + local filter="$1" + local count="${2:-1}" + local iface="${3:-any}" + + echo "Running tcpdump to capture ${count} packet(s) on interface ${iface}." + tcpdump_filename=$(mktemp) + ctdb_test_exit_hook_add "rm -f $tcpdump_filename" + tcpdump -s 1500 -w $tcpdump_filename -c "$count" -i "$iface" "$filter" & + tcpdump_pid=$! + ctdb_test_exit_hook_add "kill $tcpdump_pid >/dev/null 2>&1" + echo "Waiting for tcpdump output file to be initialised..." + wait_until 10 test -f $tcpdump_filename + sleep_for 1 +} + +not () +{ + ! "$@" +} + +tcpdump_wait () +{ + echo "Waiting for tcpdump to complete..." + wait_until 5 not kill -0 $tcpdump_pid >/dev/null 2>&1 +} + ####################################### daemons_stop () -- cgit From 62871fbcd5dedd5b4a025e5ef9e93eb231afb14e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 19 Jun 2009 11:40:09 +1000 Subject: Clean up handling the of CTDB restarts in testcases. Glitches during restarts of the CTDB cluster have been causing some tests to fail. This is because restarts are initiated in the body of many tests. This adds a simple function ctdb_restart_when_done, which schedules a restart using an existing hook in the test exit code. This function is now used in tests that need to restart CTDB. Signed-off-by: Martin Schwenke (This used to be ctdb commit d440e83bb4f0c19c085915d0f0e87cc0dabbc569) --- ctdb/tests/scripts/ctdb_test_functions.bash | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 1930ae163a..210e6c4853 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -591,6 +591,10 @@ setup_ctdb () restart_ctdb () { + if [ "$1" = "-v" ] ; then + echo "Restarting CTDB (scheduled)..." + fi + if [ -n "$CTDB_NODES_SOCKETS" ] ; then daemons_stop daemons_start $CTDB_TEST_NUM_DAEMONS @@ -615,6 +619,13 @@ restart_ctdb () echo "ctdb is ready" } +ctdb_restart_when_done () +{ + ctdb_test_exit_hook_add restart_ctdb -v +} + + + ####################################### install_eventscript () -- cgit From 0d425a07d43ca4996164e7fcc4423231b95d216e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 3 Jul 2009 17:40:16 +1000 Subject: Separate test cleanup code in output and clean up ctdb restart code. * ctdb_restart_when_done() now schedules a restart by setting an explicit variable that is respected in ctdb_test_exit(), rather than adding a restart to $ctdb_test_exit_hook. This means that restarts are all done in one place. * ctdb_test_exit() turns off "set -e" to make sure that all cleanup happens. * ctdb_test_exit() now prints a clear message indicating where the test ends and the cleanup begins. This message also includes the return code of the test. * Add debug in cluster_is_healthy to try to capture information about unexpected unhealthiness when a test starts. * Simplify simple/07_ctdb_process_exists.sh so that the exit code is generated more obviously. * Remove redundant calls to ctdb_test_exit at the end of tests, since they're done automatically via a trap. Also remove any preceding warnings of restarts or final hints about test success/failure. * Allow multi-digit debug levels in simple/12_ctdb_getdebug.sh and simple/13_ctdb_setdebug.sh. Signed-off-by: Martin Schwenke (This used to be ctdb commit b6fa044a1364cbb3008085041453ee4885f7ced1) --- ctdb/tests/scripts/ctdb_test_functions.bash | 33 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 210e6c4853..6485868cc2 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -60,12 +60,20 @@ ctdb_test_exit () trap - 0 [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + status=$(($testfailures+0)) + + # Avoid making a test fail from this point onwards. The test is + # now complete. + set +e + + echo "*** TEST COMPLETE (RC=$status), CLEANING UP..." eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook - if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then - echo "Restarting ctdb on all nodes to get back into known state..." + if $ctdb_test_restart_scheduled || \ + ! onnode 0 CTDB_TEST_CLEANING_UP=1 $CTDB_TEST_WRAPPER cluster_is_healthy ; then + restart_ctdb else # This could be made unconditional but then we might get @@ -76,7 +84,7 @@ ctdb_test_exit () onnode 0 ctdb recover fi - test_exit + exit $status } ctdb_test_exit_hook_add () @@ -149,6 +157,7 @@ ctdb_test_init () { scriptname=$(basename "$0") testfailures=0 + ctdb_test_restart_scheduled=false ctdb_test_cmd_options $@ @@ -309,6 +318,14 @@ cluster_is_healthy () exit 0 else echo "Cluster is UNHEALTHY" + if [ -z "$CTDB_TEST_CLEANING_UP" ] ; then + echo "DEBUG:" + local i + for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do + echo "$i" + $i || true + done + fi exit 1 fi } @@ -591,9 +608,11 @@ setup_ctdb () restart_ctdb () { - if [ "$1" = "-v" ] ; then - echo "Restarting CTDB (scheduled)..." + echo -n "Restarting CTDB" + if $ctdb_test_restart_scheduled ; then + echo -n " (scheduled)" fi + echo "..." if [ -n "$CTDB_NODES_SOCKETS" ] ; then daemons_stop @@ -621,11 +640,9 @@ restart_ctdb () ctdb_restart_when_done () { - ctdb_test_exit_hook_add restart_ctdb -v + ctdb_test_restart_scheduled=true } - - ####################################### install_eventscript () -- cgit From dba6c1ca7724a61ddcfb5fa7cc0d4017cb2097e6 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 3 Jul 2009 17:58:38 +1000 Subject: Fix the run_tests script so that the number of columns is never 0. Sometimes "stty size" reports 0, for example when running in a shell under Emacs. In this case, we just change it to 80. Signed-off-by: Martin Schwenke (This used to be ctdb commit e309cb3f95efcf6cff7d7c19713d7b161a138383) --- ctdb/tests/scripts/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 8df1aecd25..8a87859352 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -40,7 +40,7 @@ tests_total=0 tests_passed=0 summary="" -rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@') +rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's@^0$@80@') ww=$((rows - 7)) for f; do -- cgit From 7b3abce6848e0f8ca0dbeb083d8cdefac421fe8a Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 3 Jul 2009 18:01:29 +1000 Subject: Add an extra ctdb recovery to test function restart_ctdb(). There are still very rare cases where IPs haven't been reallocated before the beginning of the next test, so this adds a sleep and an extra call to "ctdb recover" to restart_ctdb(). Signed-off-by: Martin Schwenke (This used to be ctdb commit c2bdb77d91761c003e2f0e6918a27c54150f6030) --- ctdb/tests/scripts/ctdb_test_functions.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 6485868cc2..40f1e4fa6f 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -631,9 +631,9 @@ restart_ctdb () # cluster to stabilise before a subsequent test. echo "Forcing a recovery..." onnode -q 0 ctdb recover - - #echo "Sleeping to allow ctdb to settle..." - #sleep_for 10 + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 ctdb recover echo "ctdb is ready" } -- cgit From 613341d150c5ab8450e6c6321fba258d5a3af10b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 3 Jul 2009 20:44:55 +1000 Subject: Updates to TCP tickle tests and supporting functions. * Removed a race from tpcdump_start(). It seems impossible to tell when tcpdump is actually ready to capture packets. So this function now generates some dummy ping packets and waits until it sees them in the output file. * tcpdump_start() sets $tcpdump_filter. This is the default filter for tcpdump_wait() and tcpdump_show(), but other filters may be passed to those functions. * New functions tcptickle_sniff_start() and tcptickle_sniff_wait_show() handle capturing TCP tickle packets. These are used by complex/31_nfs_tickle.sh and complex/32_cifs_tickle.sh. Signed-off-by: Martin Schwenke (This used to be ctdb commit 8e2a89935a969340bfead8ed040d74703947cb81) --- ctdb/tests/scripts/ctdb_test_functions.bash | 89 +++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 16 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 40f1e4fa6f..b271ddea95 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -458,36 +458,93 @@ wait_until_get_src_socket () wait_until 5 get_src_socket "$@" } +####################################### + # filename will be in $tcpdump_filename, pid in $tcpdump_pid -# By default, wait for 1 matching packet on any interface. tcpdump_start () { - local filter="$1" - local count="${2:-1}" - local iface="${3:-any}" + tcpdump_filter="$1" # global - echo "Running tcpdump to capture ${count} packet(s) on interface ${iface}." + echo "Running tcpdump..." tcpdump_filename=$(mktemp) ctdb_test_exit_hook_add "rm -f $tcpdump_filename" - tcpdump -s 1500 -w $tcpdump_filename -c "$count" -i "$iface" "$filter" & - tcpdump_pid=$! - ctdb_test_exit_hook_add "kill $tcpdump_pid >/dev/null 2>&1" - echo "Waiting for tcpdump output file to be initialised..." - wait_until 10 test -f $tcpdump_filename - sleep_for 1 + + # The only way of being sure that tcpdump is listening is to send + # some packets that it will see. So we use dummy pings - the -U + # option to tcpdump ensures that packets are flushed to the file + # as they are captured. + local dummy_addr="127.3.2.1" + local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" + tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + echo "Waiting for tcpdump output file to be ready..." + ping -q "$dummy_addr" >/dev/null 2>&1 & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + tcpdump_listen_for_dummy () + { + tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 + } + + wait_until 10 tcpdump_listen_for_dummy } -not () +# By default, wait for 1 matching packet. +tcpdump_wait () { - ! "$@" + local count="${1:-1}" + local filter="${2:-${tcpdump_filter}}" + + tcpdump_check () + { + local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) + [ $found -ge $count ] + } + + echo "Waiting for tcpdump to capture some packets..." + if ! wait_until 30 tcpdump_check ; then + echo "DEBUG:" + local i + for i in "ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + echo "$i" + $i || true + done + return 1 + fi } -tcpdump_wait () +tcpdump_show () +{ + local filter="${1:-${tcpdump_filter}}" + + tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null +} + +tcptickle_sniff_start () { - echo "Waiting for tcpdump to complete..." - wait_until 5 not kill -0 $tcpdump_pid >/dev/null 2>&1 + local src="$1" + local dst="$2" + + local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" + local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" + local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 + local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" + tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" + local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" + + tcpdump_start "$filter" } +tcptickle_sniff_wait_show () +{ + tcpdump_wait 1 "$tcptickle_reset" + + echo "GOOD: here are some TCP tickle packets:" + tcpdump_show +} + + ####################################### daemons_stop () -- cgit From 35f998346e9e321b40758479a36cb610169c1cbf Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 6 Jul 2009 16:39:08 +1000 Subject: When testing make the time taken for some operations more obvious. If wait_until() does not timeout, print the time taken for the command to succeed. Signed-off-by: Martin Schwenke (This used to be ctdb commit bdb856ee22816ae1f6b8d15856555f488054f489) --- ctdb/tests/scripts/ctdb_test_functions.bash | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index b271ddea95..c7aaf6a9c8 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -268,14 +268,15 @@ wait_until () local timeout="$1" ; shift # "$@" is the command... echo -n "<${timeout}|" - while [ $timeout -gt 0 ] ; do + local t=$timeout + while [ $t -gt 0 ] ; do if "$@" ; then - echo '|' + echo "|$(($timeout - $t))|" echo "OK" return 0 fi echo -n . - timeout=$(($timeout - 1)) + t=$(($t - 1)) sleep 1 done -- cgit From d90d54ea3e7123ae973d915961d80563e24e1fb1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 6 Jul 2009 16:40:31 +1000 Subject: Make ctdbd restarts in tests more reliable. This works around potential race conditions in the init script where the restart operation is not necessarily reliable. It just wraps the actual restart in a loop and tries for a successful restart up to 5 times. Signed-off-by: Martin Schwenke (This used to be ctdb commit 1cac8a0ad429f29d1508158c7f7c42a2f1a22945) --- ctdb/tests/scripts/ctdb_test_functions.bash | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index c7aaf6a9c8..396c848034 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -672,12 +672,18 @@ restart_ctdb () fi echo "..." - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_stop - daemons_start $CTDB_TEST_NUM_DAEMONS - else - onnode -pq all $CTDB_TEST_WRAPPER _restart_ctdb - fi || return 1 + local i + for i in $(seq 1 5) ; do + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_stop + daemons_start $CTDB_TEST_NUM_DAEMONS + else + onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb + fi && break + + echo "That didn't seem to work - sleeping a while and trying again..." + sleep_for 5 + done onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 -- cgit From 96b351735671668a6aee9faf012e6a6918c2ba4d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 6 Jul 2009 17:52:11 +1000 Subject: Test suite: better debug info when the cluster is unexpectedly unhealthy. cluster_is_healthy() is now run locally in tests and internally causes _cluster_is_healthy() to be run on node 0. When it detects that the cluster is unhealthy and $ctdb_test_restart_scheduled is not true, debug information is printed. This replaces the previous use of $CTDB_TEST_CLEANING_UP. To avoid spurious debug on expected restarts, added scheduled restarts to several tests. Signed-off-by: Martin Schwenke (This used to be ctdb commit ee7caae3a55a64fb50cd28fa2fd4663c5dd83b4f) --- ctdb/tests/scripts/ctdb_test_functions.bash | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 396c848034..cc82d28bce 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -71,8 +71,7 @@ ctdb_test_exit () eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook - if $ctdb_test_restart_scheduled || \ - ! onnode 0 CTDB_TEST_CLEANING_UP=1 $CTDB_TEST_WRAPPER cluster_is_healthy ; then + if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then restart_ctdb else @@ -314,12 +313,12 @@ _cluster_is_healthy () cluster_is_healthy () { - if _cluster_is_healthy ; then + if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then echo "Cluster is HEALTHY" - exit 0 + return 0 else echo "Cluster is UNHEALTHY" - if [ -z "$CTDB_TEST_CLEANING_UP" ] ; then + if ! ${ctdb_test_restart_scheduled:-false} ; then echo "DEBUG:" local i for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do @@ -327,7 +326,7 @@ cluster_is_healthy () $i || true done fi - exit 1 + return 1 fi } -- cgit From 168ec02adf9d0dd6bf3983878637e8c9f01a2640 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 8 Jul 2009 13:37:52 +1000 Subject: Test suite: new tests and code factoring. * 2 new tests for NFS failover. * Factor repeated code from tests into new functions select_test_node_and_ips(), gratarp_sniff_start() and gratarp_sniff_wait_show(). Use these new functions in existing and new tests. Signed-off-by: Martin Schwenke (This used to be ctdb commit de0b58e18fcc0f90075fca74077ab62ae8dab5da) --- ctdb/tests/scripts/ctdb_test_functions.bash | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index cc82d28bce..b84fe7224e 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -258,6 +258,27 @@ sanity_check_ips () return 1 } +select_test_node_and_ips () +{ + try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'" + + # When selecting test_node we just want a node that has public + # IPs. This will work and is economically semi-random. :-) + local x + read x test_node <<<"$out" + + test_node_ips="" + local ip pnn + while read ip pnn ; do + if [ "$pnn" = "$test_node" ] ; then + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + echo "Selected node ${test_node} with IPs: ${test_node_ips}." + test_ip="${test_node_ips%% *}" +} + ####################################### # Wait until either timeout expires or command succeeds. The command @@ -544,6 +565,19 @@ tcptickle_sniff_wait_show () tcpdump_show } +gratarp_sniff_start () +{ + tcpdump_start "arp host ${test_ip}" +} + +gratarp_sniff_wait_show () +{ + tcpdump_wait 2 + + echo "GOOD: this should be the some gratuitous ARPs:" + tcpdump_show +} + ####################################### -- cgit From d846eb78db51ca09543b388a7867f1c212a5fadd Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 16 Jul 2009 14:04:06 +1000 Subject: Test suite: Fix debug code for unexpectedly unhealthy cluster. The debug code should run "ctdb status" on a cluster node, not on the test client. Signed-off-by: Martin Schwenke (This used to be ctdb commit 34e6f8a04b12f8879eb42d417f9741502ccccf0f) --- ctdb/tests/scripts/ctdb_test_functions.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index b84fe7224e..6506730f74 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -342,7 +342,7 @@ cluster_is_healthy () if ! ${ctdb_test_restart_scheduled:-false} ; then echo "DEBUG:" local i - for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do + for i in "onnode -q 0 ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do echo "$i" $i || true done @@ -527,7 +527,7 @@ tcpdump_wait () if ! wait_until 30 tcpdump_check ; then echo "DEBUG:" local i - for i in "ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + for i in "onnode -q 0 ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do echo "$i" $i || true done -- cgit From 366d413f2b227d074f654dc376151588ec1720ce Mon Sep 17 00:00:00 2001 From: Ronnie Sahlberg Date: Fri, 17 Jul 2009 13:01:11 +1000 Subject: new version 1.0.87 (This used to be ctdb commit d187eb8507f35a650ff3ffc50fa49110eebca0bd) --- ctdb/tests/scripts/ctdb_test_functions.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 6506730f74..b84fe7224e 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -342,7 +342,7 @@ cluster_is_healthy () if ! ${ctdb_test_restart_scheduled:-false} ; then echo "DEBUG:" local i - for i in "onnode -q 0 ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do + for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do echo "$i" $i || true done @@ -527,7 +527,7 @@ tcpdump_wait () if ! wait_until 30 tcpdump_check ; then echo "DEBUG:" local i - for i in "onnode -q 0 ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + for i in "ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do echo "$i" $i || true done -- cgit From 48078dd24fdf87b6524f4052694e0a6e16dc86fb Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 29 Jul 2009 18:01:07 +1000 Subject: Test suite: Fixes for node state parsing plus new stop/continue tests. The parsing of "ctdb status -Y" output to determine various node states was implemented very strictly. Therefore, the parsing broke due to the addition of the new "stopped" state to the output of "ctdb status -Y". This relaxes the parsing so that it should work for versions prior to the introduction of the "stopped" state, as well as future versions that add new states to the end of the list of bits in output of "ctdb status -Y". Similarly the check for cluster unhealthy (in _cluster_is_healthy()) now just checks for a single 1 in any bit in the "ctdb status -Y" output, rather than checking for a particular number of 0s. New tests tests/simple/{41_ctdb_stop.sh,42_ctdb_continue.sh,43_stop_recmaster_yield.sh} do rudimentary testing of the stop and continue functions. Remove tests tests/simple/41_ctdb_ban.sh and tests/simple/42_ctdb_unban.sh. They were both unreliable. tests/simple/21_ctdb_disablemonitor.sh now schedules a restart, since one will be required. Signed-off-by: Martin Schwenke (This used to be ctdb commit 67c5bfb5f02c9d45a32d976021ede4fb2174dfe9) --- ctdb/tests/scripts/ctdb_test_functions.bash | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index b84fe7224e..38324d7a9e 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -326,7 +326,7 @@ _cluster_is_healthy () count=0 while read line ; do count=$(($count + 1)) - [ "${line#:*:*:}" != "0:0:0:0:" ] && return 1 + [ "${line##:*:*:*1:}" != "$line" ] && return 1 done [ $count -gt 0 ] && return $? } <<<"$out" # Yay bash! @@ -368,14 +368,16 @@ node_has_status () local bits fpat mpat case "$status" in - (unhealthy) bits="?:?:?:1" ;; - (healthy) bits="?:?:?:0" ;; - (disconnected) bits="1:?:?:?" ;; - (connected) bits="0:?:?:?" ;; - (banned) bits="?:1:?:?" ;; - (unbanned) bits="?:0:?:?" ;; - (disabled) bits="?:?:1:?" ;; - (enabled) bits="?:?:0:?" ;; + (unhealthy) bits="?:?:?:1:*" ;; + (healthy) bits="?:?:?:0:*" ;; + (disconnected) bits="1:*" ;; + (connected) bits="0:*" ;; + (banned) bits="?:1:*" ;; + (unbanned) bits="?:0:*" ;; + (disabled) bits="?:?:1:*" ;; + (enabled) bits="?:?:0:*" ;; + (stopped) bits="?:?:?:?:1:*" ;; + (notstopped) bits="?:?:?:?:0:*" ;; (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; @@ -393,7 +395,7 @@ node_has_status () { read x while read line ; do - [ "${line#:${pnn}:*:${bits}:}" = "" ] && return 0 + [ "${line#:${pnn}:*:${bits}}" != "$line" ] && return 0 done return 1 } <<<"$out" # Yay bash! -- cgit From 1bde47212d919eac3c08342279c9df41e5d8bb2f Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 16 Jul 2009 14:04:06 +1000 Subject: Test suite: Fix debug code for unexpectedly unhealthy cluster. The debug code should run "ctdb status" on a cluster node, not on the test client. Signed-off-by: Martin Schwenke (This used to be ctdb commit 448cd8db1305c1e6dfab323f92eac4a576596e4e) --- ctdb/tests/scripts/ctdb_test_functions.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 38324d7a9e..8fd6141ecf 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -342,7 +342,7 @@ cluster_is_healthy () if ! ${ctdb_test_restart_scheduled:-false} ; then echo "DEBUG:" local i - for i in "ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do + for i in "onnode -q 0 ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do echo "$i" $i || true done @@ -529,7 +529,7 @@ tcpdump_wait () if ! wait_until 30 tcpdump_check ; then echo "DEBUG:" local i - for i in "ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + for i in "onnode -q 0 ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do echo "$i" $i || true done -- cgit From 7e09c07a340b81c6cbf0c8c867128aaa438f12c9 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 11 Sep 2009 14:06:12 +1000 Subject: Test suite: Rework the cluster (re)start code. Make it possible to start on only 1 node - for tests that need to restart a particular node. _ctdb_hack_options() attempts to see what options are being passed to a daemon that is being run via the initscript. It then sets a corresponding environment variable that the initscript knows about. Currently only the --start-as-stopped option is supported. This is extremely ugly but it seems like the only way... :-( Signed-off-by: Martin Schwenke (This used to be ctdb commit 407b3117dfc1072117abf681ec98b9e252d8744c) --- ctdb/tests/scripts/ctdb_test_functions.bash | 126 +++++++++++++++++++++++----- 1 file changed, 103 insertions(+), 23 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 8fd6141ecf..fa0e739b1d 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -607,7 +607,7 @@ daemons_stop () daemons_setup () { - local num_nodes="${1:-2}" # default is 2 nodes + local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes local var_dir=$CTDB_DIR/tests/var @@ -640,9 +640,9 @@ daemons_setup () done } -daemons_start () +daemons_start_1 () { - local num_nodes="${1:-2}" # default is 2 nodes + local pnn="$1" shift # "$@" gets passed to ctdbd local var_dir=$CTDB_DIR/tests/var @@ -654,28 +654,40 @@ daemons_start () local no_public_ips=-1 [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses + if [ "$no_public_ips" = $pnn ] ; then + echo "Node $no_public_ips will have no public IPs." + fi + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" - echo "Starting $num_nodes ctdb daemons..." - if [ "$no_public_ips" != -1 ] ; then - echo "Node $no_public_ips will have no public IPs." + if [ $(id -u) -eq 0 ]; then + ctdb_options="$ctdb_options --public-interface=lo" fi - for i in $(seq 0 $(($num_nodes - 1))) ; do - if [ $(id -u) -eq 0 ]; then - ctdb_options="$ctdb_options --public-interface=lo" - fi + if [ $pnn -eq $no_public_ips ] ; then + ctdb_options="$ctdb_options --public-addresses=/dev/null" + else + ctdb_options="$ctdb_options --public-addresses=$public_addresses" + fi - if [ $i -eq $no_public_ips ] ; then - ctdb_options="$ctdb_options --public-addresses=/dev/null" - else - ctdb_options="$ctdb_options --public-addresses=$public_addresses" - fi + # Need full path so we can use "pkill -f" to kill the daemons. + $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$pnn $ctdb_options "$@" ||return 1 +} + +daemons_start () +{ + # "$@" gets passed to ctdbd - # Need full path so we can use "pkill -f" to kill the daemons. - $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" ||return 1 + local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes + + echo "Starting $num_nodes ctdb daemons..." + + for i in $(seq 0 $(($num_nodes - 1))) ; do + daemons_start_1 $i "$@" done + local var_dir=$CTDB_DIR/tests/var + if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 fi @@ -683,8 +695,26 @@ daemons_start () ####################################### +_ctdb_hack_options () +{ + local ctdb_options="$*" + + # We really just want to pass CTDB_OPTIONS but on RH + # /etc/sysconfig/ctdb can, and frequently does, set that variable. + # So instead, we hack badly. We'll add these as we use them. + # Note that these may still be overridden by the above file... but + # we tend to use the exotic options here... so that is unlikely. + + case "$ctdb_options" in + *--start-as-stopped*) + export CTDB_START_AS_STOPPED="yes" + esac +} + _restart_ctdb () { + _ctdb_hack_options "$@" + if [ -e /etc/redhat-release ] ; then service ctdb restart else @@ -692,31 +722,81 @@ _restart_ctdb () fi } +_ctdb_start () +{ + _ctdb_hack_options "$@" + + /etc/init.d/ctdb start +} + setup_ctdb () { if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_setup $CTDB_TEST_NUM_DAEMONS + daemons_setup + fi +} + +# Common things to do after starting one or more nodes. +_ctdb_start_post () +{ + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "ctdb setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this blocks + # until the recovery is complete. Hopefully this will help the + # cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 ctdb recover + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 ctdb recover + + echo "ctdb is ready" +} + +# This assumes that ctdbd is not running on the given node. +ctdb_start_1 () +{ + local pnn="$1" + shift # "$@" is passed to ctdbd start. + + echo -n "Starting CTDB on node ${pnn}..." + + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_start_1 $pnn "$@" + else + onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@" fi + + # If we're starting only 1 node then we're doing something weird. + ctdb_restart_when_done } restart_ctdb () { + # "$@" is passed to ctdbd start. + echo -n "Restarting CTDB" if $ctdb_test_restart_scheduled ; then echo -n " (scheduled)" fi echo "..." - local i - for i in $(seq 1 5) ; do + local i=0 + while : ; do if [ -n "$CTDB_NODES_SOCKETS" ] ; then daemons_stop - daemons_start $CTDB_TEST_NUM_DAEMONS + daemons_start "$@" else - onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb + onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" fi && break - echo "That didn't seem to work - sleeping a while and trying again..." + i=$(($i + 1)) + [ $i -lt 5 ] || break + + echo "That didn't seem to work - sleeping for a while..." sleep_for 5 done -- cgit From b8b28cb567bf4e729220bace9122506ebc61a9b0 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 11 Sep 2009 15:55:53 +1000 Subject: Test suite: wait_until_node_has_status() now uses "onnode any". Many tests currently do this sort of thing: onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status 1 disconnected In fact, they all use exactly the same "onnode 0 $CTDB_TEST_WRAPPER" idiom. This is both repetitious and dangerous, since node 0 might be shutdown during a test. Instead, we push "onnode any $CTDB_TEST_WRAPPER" (which selects a connected node) into wait_until_node_has_status() and just call that function directly in tests, like this: wait_until_node_has_status 1 disconnected Signed-off-by: Martin Schwenke (This used to be ctdb commit a2aaef03d4d6bbd4b42f50f732254935d4d3469c) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index fa0e739b1d..57cd7816df 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -417,7 +417,7 @@ wait_until_node_has_status () echo "Waiting until node $pnn has status \"$status\"..." - wait_until $timeout node_has_status "$pnn" "$status" + onnode any $CTDB_TEST_WRAPPER wait_until $timeout node_has_status "$pnn" "$status" } # Useful for superficially testing IP failover. -- cgit From 4948051bf4b77cf5a04e2741e922dd152703bad4 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 25 Sep 2009 18:00:17 +1000 Subject: Test suite: Print debug info on node status timeouts. Signed-off-by: Martin Schwenke (This used to be ctdb commit a083a1976d621c76121f1fa2c2f484cfa47267bd) --- ctdb/tests/scripts/ctdb_test_functions.bash | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 57cd7816df..3da0f0effd 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -417,7 +417,15 @@ wait_until_node_has_status () echo "Waiting until node $pnn has status \"$status\"..." - onnode any $CTDB_TEST_WRAPPER wait_until $timeout node_has_status "$pnn" "$status" + if ! onnode any $CTDB_TEST_WRAPPER wait_until $timeout node_has_status "$pnn" "$status" ; then + for i in "onnode -q any ctdb status" "onnode -q any onnode all ctdb scriptstatus" ; do + echo "$i" + $i || true + done + + return 1 + fi + } # Useful for superficially testing IP failover. -- cgit From 469ee69363db47ca4e0a307e85c69a06b05a5e33 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 20 Oct 2009 16:44:44 +1100 Subject: Test suite: add -x option to ctdb_init() function. This facilitates tracing of tests. Signed-off-by: Martin Schwenke (This used to be ctdb commit 1f906bd3476e7cebf217e35b5477d6a7bb615a0c) --- ctdb/tests/scripts/ctdb_test_functions.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 3da0f0effd..40d8453e56 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -120,6 +120,7 @@ Options: --category show the test category (ACL, CTDB, Samba ...). -d, --description show test case description. --summary show short test case summary. + -x trace test using set -x EOF exit $status @@ -141,7 +142,7 @@ ctdb_test_cmd_options() -v|--version) ctdb_test_version ;; --category) echo "CTDB" ;; -d|--description) test_info ;; - --summary) test_info | head -1 ;; + -x) set -x ; return 0 ;; *) echo "Error: Unknown parameter = $1" echo -- cgit From d79f7647e7fa397a18f9f372e0eda24cc294311f Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 20 Oct 2009 16:45:29 +1100 Subject: Test suite: Fix bug in node_has_status(). This function has been broken since it was updated to work with the "stopped" state (probably commit 67c5bfb5f02c9d45a32d976021ede4fb2174dfe9). Although ${var#:*:0} removes the shortest matching prefix of $var, '*' can match substrings that include ':' if '0' isn't where you expect. So we were making unexpected matches and incorrectly returning true for some cases. Signed-off-by: Martin Schwenke (This used to be ctdb commit 11137bc2d492a62a26ec9f9f62ff362e81643f66) --- ctdb/tests/scripts/ctdb_test_functions.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 40d8453e56..8de0d4b2b7 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -396,7 +396,10 @@ node_has_status () { read x while read line ; do - [ "${line#:${pnn}:*:${bits}}" != "$line" ] && return 0 + # This needs to be done in 2 steps to avoid false matches. + local line_bits="${line#:${pnn}:*:}" + [ "$line_bits" = "$line" ] && continue + [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 done return 1 } <<<"$out" # Yay bash! -- cgit From f2a9ba6976bb5d7adb279728e01fc6c55796732f Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 21 Oct 2009 21:36:39 +1100 Subject: Test suite: Fix the timeouts on the skip share check tests. The timeout for waiting for state changes isn't very predictable. It is "about" MonitorInterval seconds... but can be longer given the duration of eventscript runs and other things. So, we change the timeout to MonitorInterval + EventScriptTimeout, hoping it never takes that long. Move the eventscript installation/removal from the old fake-tests into a function in the functions file. Implement supporting functions to create/remove/check-for various files that it handles. Also add a function that uses all of this that waits for the next monitor event (but only if all other monitor events pass). The final check in the skip share check tests uses the above and waits for a monitor event, and then checks that the node is still healthy. Also enhance the wait_until function to handle a command starting with '!' (as a separate word) to make it easy to wait for a file not to exist. Signed-off-by: Martin Schwenke (This used to be ctdb commit 25e82a8a667a54c6921ef076c63fdd738dd75d19) --- ctdb/tests/scripts/ctdb_test_functions.bash | 120 +++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 8de0d4b2b7..e554680f0a 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -288,10 +288,19 @@ wait_until () { local timeout="$1" ; shift # "$@" is the command... + local negate=false + if [ "$1" = "!" ] ; then + negate=true + shift + fi + echo -n "<${timeout}|" local t=$timeout while [ $t -gt 0 ] ; do - if "$@" ; then + ("$@") + local rc=$? + if { ! $negate && [ $rc -eq 0 ] ; } || \ + { $negate && [ $rc -ne 0 ] ; } ; then echo "|$(($timeout - $t))|" echo "OK" return 0 @@ -864,3 +873,112 @@ uninstall_eventscript () rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" fi } + +####################################### + +# This section deals with the 99.ctdb_test eventscript. + +# Metafunctions: Handle a ctdb-test file on a node. +# given event. +ctdb_test_eventscript_file_create () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_remove () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_exists () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1 +} + + +# Handle a flag file on a node that is removed by 99.ctdb_test on the +# given event. +ctdb_test_eventscript_flag () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}" +} + + +# Handle a trigger that causes 99.ctdb_test to fail it's monitor +# event. +ctdb_test_eventscript_unhealthy_trigger () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger" +} + +# Handle the file that 99.ctdb_test created to show that it has marked +# a node unhealthy because it detected the above trigger. +ctdb_test_eventscript_unhealthy_detected () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" +} + +# Note that the eventscript can't use the above functions! +ctdb_test_eventscript_install () +{ + + local script='#!/bin/sh +out=$(ctdb pnn) +pnn="${out#PNN:}" + +rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" + +trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" +detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" +if [ "$1" = "monitor" ] ; then + if [ -e "$trigger" ] ; then + echo "${0}: Unhealthy because \"$trigger\" detected" + touch "$detected" + exit 1 + elif [ -e "$detected" -a ! -e "$trigger" ] ; then + echo "${0}: Healthy again, \"$trigger\" no longer detected" + rm "$detected" + fi +fi + +exit 0 +' + install_eventscript "99.ctdb_test" "$script" +} + +ctdb_test_eventscript_uninstall () +{ + uninstall_eventscript "99.ctdb_test" +} + +# Note that this only works if you know all other monitor events will +# succeed. You also need to install the eventscript before using it. +wait_for_monitor_event () +{ + local pnn="$1" + + echo "Waiting for a monitor event on node $pnn to complete..." + ctdb_test_eventscript_flag create $pnn "monitor" + + wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" + +} -- cgit From 8767c894a0388f576a6bdd625d2917628f49150d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 28 Oct 2009 13:02:18 +1100 Subject: Test suite: Regression fix - wait_until should not run command in sub-shell. Commit 25e82a8a667a54c6921ef076c63fdd738dd75d19 changed wait_until() to protect the command it runs from "set -e" by running it in a subshell. This breaks uses where the command is expected to set global variables. For example, wait_until_get_src_socket lost the value of $out from its call to get_src_socket(). The fix is to not be lazy and use a sub-shell! Signed-off-by: Martin Schwenke (This used to be ctdb commit 39642e745254d93d74dde907787503854fe6ca4a) --- ctdb/tests/scripts/ctdb_test_functions.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index e554680f0a..832a42bf16 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -297,8 +297,8 @@ wait_until () echo -n "<${timeout}|" local t=$timeout while [ $t -gt 0 ] ; do - ("$@") - local rc=$? + local rc=0 + "$@" || rc=$? if { ! $negate && [ $rc -eq 0 ] ; } || \ { $negate && [ $rc -ne 0 ] ; } ; then echo "|$(($timeout - $t))|" -- cgit From 9a96ae0c978c553ebd7d1dc5e1abe1f65e093627 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 29 Nov 2009 12:39:23 +0100 Subject: server: only do the mkdir() calls for db_directory* once at the start metze (This used to be ctdb commit f30f33685db50860b6cd6fd1b6bdc3066620a78f) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 832a42bf16..d8a0de94aa 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -679,7 +679,7 @@ daemons_start_1 () echo "Node $no_public_ips will have no public IPs." fi - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent" + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" if [ $(id -u) -eq 0 ]; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From 5e92afeb336088a68ce712c01a217495417497ee Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 18 Dec 2009 14:42:58 +1100 Subject: Test suite: allow settign of timeout triggers for all events not just monitor. Signed-off-by: Martin Schwenke (This used to be ctdb commit f319bd54369a2bc7d32c3bda7fc22f2ef1a51c3a) --- ctdb/tests/scripts/ctdb_test_functions.bash | 45 ++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 11 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 832a42bf16..1bc540716c 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -937,6 +937,17 @@ ctdb_test_eventscript_unhealthy_detected () ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" } +# Handle a trigger that causes 99.ctdb_test to timeout it's monitor +# event. This should cause the node to be banned. +ctdb_test_eventscript_timeout_trigger () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout" +} + # Note that the eventscript can't use the above functions! ctdb_test_eventscript_install () { @@ -949,16 +960,28 @@ rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" -if [ "$1" = "monitor" ] ; then - if [ -e "$trigger" ] ; then - echo "${0}: Unhealthy because \"$trigger\" detected" - touch "$detected" - exit 1 - elif [ -e "$detected" -a ! -e "$trigger" ] ; then - echo "${0}: Healthy again, \"$trigger\" no longer detected" - rm "$detected" - fi -fi +timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}" +case "$1" in + monitor) + if [ -e "$trigger" ] ; then + echo "${0}: Unhealthy because \"$trigger\" detected" + touch "$detected" + exit 1 + elif [ -e "$detected" -a ! -e "$trigger" ] ; then + echo "${0}: Healthy again, \"$trigger\" no longer detected" + rm "$detected" + fi + + ;; + *) + if [ -e "$timeout_trigger" ] ; then + echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected" + sleep 9999 + fi + ;; + *) + +esac exit 0 ' @@ -976,7 +999,7 @@ wait_for_monitor_event () { local pnn="$1" - echo "Waiting for a monitor event on node $pnn to complete..." + echo "Waiting for a monitor event on node ${pnn}..." ctdb_test_eventscript_flag create $pnn "monitor" wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" -- cgit From 0054f0c54fcbed8f80f04f4f0264b4647e59db3f Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sat, 16 Jan 2010 10:35:41 +0100 Subject: tests: add a all_ips_on_node() helper function that wraps ctdb ip -Y metze (This used to be ctdb commit c24fbea156dfdc9154e94eace725526e44cbcdac) --- ctdb/tests/scripts/ctdb_test_functions.bash | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 7c1b46c91d..4f05888d4a 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -239,7 +239,7 @@ sanity_check_output () sanity_check_ips () { - local ips="$1" # Output of "ctdb ip -n all" + local ips="$1" # list of "ip node" lines echo "Sanity checking IPs..." @@ -259,9 +259,16 @@ sanity_check_ips () return 1 } +# This returns a list of "ip node" lines in $out +all_ips_on_node() +{ + local node=$@ + try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" +} + select_test_node_and_ips () { - try_command_on_node 0 "$CTDB ip -n all | sed -e '1d'" + all_ips_on_node 0 # When selecting test_node we just want a node that has public # IPs. This will work and is economically semi-random. :-) @@ -450,7 +457,7 @@ ips_are_on_nodeglob () local out - try_command_on_node 1 ctdb ip -n all + all_ips_on_node 1 while read ip pnn ; do for check in $ips ; do -- cgit From 77ad2be488c859795f721f87f04def8488dfbce5 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 5 Aug 2010 15:58:56 +1000 Subject: Test suite: weaken ctdb continue/enable tests for non-deterministic IPs. These tests currently wait for the old IPs to fail back to the test node. This isn't guaranteed with DeterministicIPs disabled. This changes those tests to wait until the test node gets at least 1 IP assigned. Signed-off-by: Martin Schwenke (This used to be ctdb commit e9b3f5b1b51d541a911a27eb4348b368f28d185e) --- ctdb/tests/scripts/ctdb_test_functions.bash | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 4f05888d4a..42053c0486 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -482,6 +482,30 @@ wait_until_ips_are_on_nodeglob () wait_until 60 ips_are_on_nodeglob "$@" } +node_has_some_ips () +{ + local node="$1" + + local out + + all_ips_on_node 1 + + while read ip pnn ; do + if [ "$node" = "$pnn" ] ; then + return 0 + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + return 1 +} + +wait_until_node_has_some_ips () +{ + echo "Waiting for node to have some IPs..." + + wait_until 60 node_has_some_ips "$@" +} + get_src_socket () { local proto="$1" -- cgit From 00fec0e76c1186fc115910be106c62a0e97778d7 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 12 Aug 2010 13:48:33 +1000 Subject: Test suite: improve wait_until_node_has_status() This currently does "onnode any ... wait_until ...". If ctdbd is being shutdown on a node then that node might be chosen anyway, if it is asked early enough. Then we'll loop on that node but our ctdb client command may always fail, causing a timeout rather than the expected behaviour. This puts the loop on the outside of the "onnode any" so that if the "wrong" node is chosen initially then on the next iteration the choice can be remade. Signed-off-by: Martin Schwenke (This used to be ctdb commit a88ee78686bd5aa2b789f5959e0562315a13525d) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 42053c0486..fae49473cc 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -437,7 +437,7 @@ wait_until_node_has_status () echo "Waiting until node $pnn has status \"$status\"..." - if ! onnode any $CTDB_TEST_WRAPPER wait_until $timeout node_has_status "$pnn" "$status" ; then + if ! wait_until $timeout onnode any $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then for i in "onnode -q any ctdb status" "onnode -q any onnode all ctdb scriptstatus" ; do echo "$i" $i || true -- cgit From a9fb1e318b2d41da7f9eb777c6d5f821cf7fbaf6 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 12 Aug 2010 14:13:07 +1000 Subject: Test suite: use $CTDB rather than ctdb everywhere in ctdb_test_functions.sh. Also ensure that $CTDB is set by default it to "ctdb". Signed-off-by: Martin Schwenke (This used to be ctdb commit 8222fef1e61836b9bfd406205f9ffb9396aa7480) --- ctdb/tests/scripts/ctdb_test_functions.bash | 35 ++++++++++++++++------------- 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index fae49473cc..40a9642afb 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -80,7 +80,7 @@ ctdb_test_exit () # leave the recovery in restart_ctdb so that future tests that # might do a manual restart mid-test will benefit. echo "Forcing a recovery..." - onnode 0 ctdb recover + onnode 0 $CTDB recover fi exit $status @@ -336,7 +336,7 @@ _cluster_is_healthy () { local out x count line - out=$(ctdb -Y status 2>&1) || return 1 + out=$($CTDB -Y status 2>&1) || return 1 { read x @@ -359,7 +359,7 @@ cluster_is_healthy () if ! ${ctdb_test_restart_scheduled:-false} ; then echo "DEBUG:" local i - for i in "onnode -q 0 ctdb status" "onnode -q 0 onnode all ctdb scriptstatus" ; do + for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do echo "$i" $i || true done @@ -407,7 +407,7 @@ node_has_status () if [ -n "$bits" ] ; then local out x line - out=$(ctdb -Y status 2>&1) || return 1 + out=$($CTDB -Y status 2>&1) || return 1 { read x @@ -420,9 +420,9 @@ node_has_status () return 1 } <<<"$out" # Yay bash! elif [ -n "$fpat" ] ; then - ctdb statistics -n "$pnn" | egrep -q "$fpat" + $CTDB statistics -n "$pnn" | egrep -q "$fpat" elif [ -n "$mpat" ] ; then - ctdb getmonmode -n "$pnn" | egrep -q "$mpat" + $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" else echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' return 1 @@ -438,7 +438,7 @@ wait_until_node_has_status () echo "Waiting until node $pnn has status \"$status\"..." if ! wait_until $timeout onnode any $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then - for i in "onnode -q any ctdb status" "onnode -q any onnode all ctdb scriptstatus" ; do + for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do echo "$i" $i || true done @@ -581,7 +581,7 @@ tcpdump_wait () if ! wait_until 30 tcpdump_check ; then echo "DEBUG:" local i - for i in "onnode -q 0 ctdb status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do echo "$i" $i || true done @@ -638,7 +638,7 @@ gratarp_sniff_wait_show () daemons_stop () { echo "Attempting to politely shutdown daemons..." - onnode 1 ctdb shutdown -n all || true + onnode 1 $CTDB shutdown -n all || true echo "Sleeping for a while..." sleep_for 1 @@ -794,16 +794,16 @@ _ctdb_start_post () onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 echo "Setting RerecoveryTimeout to 1" - onnode -pq all "ctdb setvar RerecoveryTimeout 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" # In recent versions of CTDB, forcing a recovery like this blocks # until the recovery is complete. Hopefully this will help the # cluster to stabilise before a subsequent test. echo "Forcing a recovery..." - onnode -q 0 ctdb recover + onnode -q 0 $CTDB recover sleep_for 1 echo "Forcing a recovery..." - onnode -q 0 ctdb recover + onnode -q 0 $CTDB recover echo "ctdb is ready" } @@ -855,16 +855,16 @@ restart_ctdb () onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 echo "Setting RerecoveryTimeout to 1" - onnode -pq all "ctdb setvar RerecoveryTimeout 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" # In recent versions of CTDB, forcing a recovery like this blocks # until the recovery is complete. Hopefully this will help the # cluster to stabilise before a subsequent test. echo "Forcing a recovery..." - onnode -q 0 ctdb recover + onnode -q 0 $CTDB recover sleep_for 1 echo "Forcing a recovery..." - onnode -q 0 ctdb recover + onnode -q 0 $CTDB recover echo "ctdb is ready" } @@ -984,7 +984,7 @@ ctdb_test_eventscript_install () { local script='#!/bin/sh -out=$(ctdb pnn) +out=$($CTDB pnn) pnn="${out#PNN:}" rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" @@ -1036,3 +1036,6 @@ wait_for_monitor_event () wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" } + +# Make sure that $CTDB is set. +: ${CTDB:=ctdb} -- cgit From 03aa9ee702540da9b44034cd34ecba7777dbbf27 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 13 Aug 2010 17:01:54 +1000 Subject: Test suite: strengthen function _cluster_is_healthy(). If there's a chance that "ctdb status -Y" can return 0 but print garbage then this function might return a false positive. So, we do 2 things: * Redirect stderr to >/dev/null rather than looking at it. This minimises the chance that we will see garbage. * Since we need at least 1 good line to decide the cluster is healthy, we sanity check each line to esnure it starts with :[0-9]. Signed-off-by: Martin Schwenke (This used to be ctdb commit d4189c7c3fceaa833f9f0446a2b06af6fed714ec) --- ctdb/tests/scripts/ctdb_test_functions.bash | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 40a9642afb..d2b069c95b 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -336,13 +336,15 @@ _cluster_is_healthy () { local out x count line - out=$($CTDB -Y status 2>&1) || return 1 + out=$($CTDB -Y status 2>/dev/null) || return 1 { read x count=0 while read line ; do - count=$(($count + 1)) + # We need to see valid lines if we're going to be healthy. + [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1)) + # A line indicating a node is unhealthy causes failure. [ "${line##:*:*:*1:}" != "$line" ] && return 1 done [ $count -gt 0 ] && return $? -- cgit From e28d2b8f22680c97ca1e612dd2d778d7da656114 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 17 Aug 2010 09:52:15 +1000 Subject: Test suite: print date/time at test completion. This should help with log cross-checking. Signed-off-by: Martin Schwenke (This used to be ctdb commit c0a916c40c623c0aa8245526283a064dbeea4b57) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index d2b069c95b..a7e1baf2d6 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -66,7 +66,7 @@ ctdb_test_exit () # now complete. set +e - echo "*** TEST COMPLETE (RC=$status), CLEANING UP..." + echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook -- cgit From a3e9fe20583848625bcdb59db5df277b8c024281 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 17 Aug 2010 09:55:48 +1000 Subject: Test suite: Add more timestamping of debugging information. Signed-off-by: Martin Schwenke (This used to be ctdb commit 4cdf3b9adc7edfd80a2901ef8457ae67aab0829a) --- ctdb/tests/scripts/ctdb_test_functions.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index a7e1baf2d6..68f7994da0 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -359,7 +359,7 @@ cluster_is_healthy () else echo "Cluster is UNHEALTHY" if ! ${ctdb_test_restart_scheduled:-false} ; then - echo "DEBUG:" + echo "DEBUG AT $(date '+%F %T'):" local i for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do echo "$i" @@ -581,7 +581,7 @@ tcpdump_wait () echo "Waiting for tcpdump to capture some packets..." if ! wait_until 30 tcpdump_check ; then - echo "DEBUG:" + echo "DEBUG AT $(date '+%F %T'):" local i for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do echo "$i" -- cgit From 0d2c554d5fd2e783b74d086543559157023660eb Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 26 Aug 2010 14:04:03 +1000 Subject: Test suite: in the test eventscript, run "ctdb" not "$CTDB". It is too hard to do anything else... Signed-off-by: Martin Schwenke (This used to be ctdb commit 08b636b500855e38e708e6963d8e63ded97c25ec) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 68f7994da0..1433a46ec3 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -986,7 +986,7 @@ ctdb_test_eventscript_install () { local script='#!/bin/sh -out=$($CTDB pnn) +out=$(ctdb pnn) pnn="${out#PNN:}" rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" -- cgit From eae91c959e487eb3513e98cbd5476d2107cd22f8 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 23 May 2011 15:41:59 +1000 Subject: Test suite: add a -d option to the run_tests script. This causes summary lines (when used with -s) to be pretty printed and include the test description. This is the 4th line of the test output - that is, immediately after the header. Signed-off-by: Martin Schwenke (This used to be ctdb commit 0e5cc2a58b0d38e10a2ef9e81dc887c20f3fbdcb) --- ctdb/tests/scripts/run_tests | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 8a87859352..5906c42814 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -18,8 +18,9 @@ EOF ###################################################################### with_summary=false +with_desc=false -temp=$(getopt -n "$prog" -o "xhs" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdhs" -l help -- "$@") [ $? != 0 ] && usage @@ -28,6 +29,7 @@ eval set -- "$temp" while true ; do case "$1" in -x) set -x; shift ;; + -d) with_desc=true ; shift ;; # 4th line of output is description -s) with_summary=true ; shift ;; --) shift ; break ;; *) usage ;; @@ -43,18 +45,31 @@ summary="" rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's@^0$@80@') ww=$((rows - 7)) +tf=$(mktemp) + +set -o pipefail + for f; do [ -x $f ] || fail "test \"$f\" is not executable" tests_total=$(($tests_total + 1)) - if ctdb_test_run "$f" ; then + ctdb_test_run "$f" | tee "$tf" + status=$? + if [ $status -eq 0 ] ; then tests_passed=$(($tests_passed + 1)) t="PASSED" else t="FAILED" fi + if $with_desc ; then + f="${f#./}" ; f="${f%%[./]*}" + desc=$(tail -n +4 $tf | head -n 1) + f="${f} ${desc}" + fi summary=$(printf "%s\n%-${ww}s%s" "$summary" "$f" "$t") done +rm -f "$tf" + if $with_summary ; then echo "$summary" echo -- cgit From 8d2c726debc34e9cad3c482a65100525840a3bc3 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 25 May 2011 13:39:11 +1000 Subject: Tests: change output format of run_tests script and add -q option Putting PASSED/FAILED on the left makes it easier to scan the results and simplifies the code. Also put starts around the word "*FAILED*" to make it more obvious. Also add a -q option to throw away test output and only display the summary (if -s is also specified). Signed-off-by: Martin Schwenke (This used to be ctdb commit c44b632b010b7d57007f3c8f294271c7e0217e0d) --- ctdb/tests/scripts/run_tests | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 5906c42814..e46d8d8e6c 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -19,8 +19,9 @@ EOF with_summary=false with_desc=false +quiet=false -temp=$(getopt -n "$prog" -o "xdhs" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdhqs" -l help -- "$@") [ $? != 0 ] && usage @@ -30,12 +31,19 @@ while true ; do case "$1" in -x) set -x; shift ;; -d) with_desc=true ; shift ;; # 4th line of output is description + -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; --) shift ; break ;; *) usage ;; esac done +if $quiet ; then + show_progress() { cat >/dev/null ; } +else + show_progress() { cat ; } +fi + ###################################################################### tests_total=0 @@ -46,34 +54,40 @@ rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's ww=$((rows - 7)) tf=$(mktemp) +sf=$(mktemp) set -o pipefail for f; do [ -x $f ] || fail "test \"$f\" is not executable" tests_total=$(($tests_total + 1)) - ctdb_test_run "$f" | tee "$tf" + ctdb_test_run "$f" | tee "$tf" | show_progress status=$? - if [ $status -eq 0 ] ; then - tests_passed=$(($tests_passed + 1)) - t="PASSED" - else - t="FAILED" - fi - if $with_desc ; then - f="${f#./}" ; f="${f%%[./]*}" - desc=$(tail -n +4 $tf | head -n 1) - f="${f} ${desc}" + if $with_summary ; then + if [ $status -eq 0 ] ; then + tests_passed=$(($tests_passed + 1)) + t=" PASSED " + else + t="*FAILED*" + fi + if $with_desc ; then + f="${f#./}" ; f="${f%%[./]*}" + desc=$(tail -n +4 $tf | head -n 1) + f="${f} ${desc}" + fi + echo "$t $f" >>"$sf" fi - summary=$(printf "%s\n%-${ww}s%s" "$summary" "$f" "$t") done rm -f "$tf" if $with_summary ; then - echo "$summary" + echo + cat "$sf" echo echo "${tests_passed}/${tests_total} tests passed" fi +rm -f "$sf" + test_exit -- cgit From 8006aec7b1d5687fb8776405c5e5ff031899107a Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 7 Jun 2011 16:06:42 +1000 Subject: Tests: run_tests script no longer prints filename in summary descriptions. If filenames should be printed in descriptions in the summary then the descriptions should include the filename. A better option is to include something more human-readable that makes the test just as easily identifiable. Signed-off-by: Martin Schwenke (This used to be ctdb commit 0efdbd61bdc2343e5459959b300bccc9986b1d78) --- ctdb/tests/scripts/run_tests | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index e46d8d8e6c..1ce089d852 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -71,9 +71,8 @@ for f; do t="*FAILED*" fi if $with_desc ; then - f="${f#./}" ; f="${f%%[./]*}" desc=$(tail -n +4 $tf | head -n 1) - f="${f} ${desc}" + f="$desc" fi echo "$t $f" >>"$sf" fi -- cgit From e05b902f99af5f97571208bc4c8f425dad336335 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 9 Sep 2010 17:21:11 +1000 Subject: Test suite: add automated checking of time logs. This depends on the format of onnode output and also depends on simple/00_ctdb_onnode.sh having been run. Signed-off-by: Martin Schwenke (This used to be ctdb commit 93b53b186df55942bf4d9e90cae329f47889af72) --- ctdb/tests/scripts/ctdb_test_functions.bash | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 1433a46ec3..6444a3eefc 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -53,6 +53,49 @@ test_exit () exit $(($testfailures+0)) } +ctdb_check_time_logs () +{ + local threshold=20 + + local jump=false + local prev="" + local ds_prev="" + local node="" + + out=$(onnode all tail -n 20 /var/log/ctdb.test.time.log 2>&1) + + if [ $? -eq 0 ] ; then + local line + while read line ; do + case "$line" in + \>\>\ NODE:\ *\ \<\<) + node="${line#>> NODE: }" + node=${node% <<*} + ds_prev="" + ;; + *\ *) + set -- $line + ds_curr="$1${2:0:1}" + if [ -n "$ds_prev" ] && \ + [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then + echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})" + jump=true + fi + prev="$line" + ds_prev="$ds_curr" + ;; + esac + done <<<"$out" + else + echo Error getting time logs + fi + if $jump ; then + echo "Check time sync (test client first):" + date + onnode -p all date + fi +} + ctdb_test_exit () { local status=$? @@ -68,6 +111,10 @@ ctdb_test_exit () echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." + if [ -n "$CTDB_TEST_REAL_CLUSTER" -a $status -ne 0 ] ; then + ctdb_check_time_logs + fi + eval "$ctdb_test_exit_hook" || true unset ctdb_test_exit_hook -- cgit From 659f54e61a5d5b399810f6677f765caf918a2692 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 21 Sep 2010 13:34:22 +1000 Subject: Test suite: add more debug to time jump post mortem. Signed-off-by: Martin Schwenke (This used to be ctdb commit fed3c2b80b8add8d1cf33abdd5dd8d8001af44d4) --- ctdb/tests/scripts/ctdb_test_functions.bash | 3 +++ 1 file changed, 3 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 6444a3eefc..5efe9a6bb6 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -93,6 +93,9 @@ ctdb_check_time_logs () echo "Check time sync (test client first):" date onnode -p all date + echo "Information from test client:" + hostname + top -b -n 1 fi } -- cgit From 372f0a1bff538ed3fab74ed0a0b979338c68c7af Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 23 Sep 2010 15:01:01 +1000 Subject: Test suite: Add debug for cluster (un)healthy flip-flop after restart. We're seeing the cluster become healthy after a restart and then revert to being unhealthy. It looks like there's a race and the cluster shouldn't have been healthy, given that we seem to see that the monitor cycle hasn't yet been run. This collects some state debug info from all nodes after the cluster becomes healthy. This is printed if the cluster is then unexpectedly unhealthy a short time later. Signed-off-by: Martin Schwenke (This used to be ctdb commit c2efb5897e4258df649149f9904d7ac47322e1b4) --- ctdb/tests/scripts/ctdb_test_functions.bash | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 5efe9a6bb6..ecb0ca1d55 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -906,6 +906,8 @@ restart_ctdb () onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 + local debug_out=$(onnode -p all ctdb status 2>&1; onnode -p all ctdb scriptstatus 2>&1) + echo "Setting RerecoveryTimeout to 1" onnode -pq all "$CTDB setvar RerecoveryTimeout 1" @@ -919,6 +921,13 @@ restart_ctdb () onnode -q 0 $CTDB recover echo "ctdb is ready" + + if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "OUCH! Cluster is UNHEALTHY again..." + echo "$debug_out" + # Try to make the calling test fail + status=1 + fi } ctdb_restart_when_done () -- cgit From 7e48ba58c6a1b78dde5fcc743f4a62761d3afc79 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 24 Sep 2010 14:34:46 +1000 Subject: Test suite: Print debug info from cluster nodes when time jumps occur. Signed-off-by: Martin Schwenke (This used to be ctdb commit 21cdc7ed6942238faeb42983c862d4abc3f54ffb) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index ecb0ca1d55..d347cf2016 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -96,6 +96,8 @@ ctdb_check_time_logs () echo "Information from test client:" hostname top -b -n 1 + echo "Information from cluster nodes:" + onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo" fi } -- cgit From 000fbb607ee039f415d8d9772fc79e1f219425e6 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 28 Sep 2010 14:37:02 +1000 Subject: Test suite: when the cluster flip-flops (un)healthy, using "ctdb status -Y". Signed-off-by: Martin Schwenke (This used to be ctdb commit d3dc9410501767c07d9b0106bb73c979d869c127) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index d347cf2016..1a7935d9f5 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -908,7 +908,7 @@ restart_ctdb () onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 - local debug_out=$(onnode -p all ctdb status 2>&1; onnode -p all ctdb scriptstatus 2>&1) + local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) echo "Setting RerecoveryTimeout to 1" onnode -pq all "$CTDB setvar RerecoveryTimeout 1" -- cgit From 94f0fd9cd547ffd458090453e7499520c3e65d03 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 12 Oct 2010 15:10:43 +1100 Subject: Test suite: Try much harder to get a healthy cluster when it is restarted. Signed-off-by: Martin Schwenke (This used to be ctdb commit 91e74cb01a11012e41ef9633c98f13ddbb2e5908) --- ctdb/tests/scripts/ctdb_test_functions.bash | 64 ++++++++++++++++------------- 1 file changed, 35 insertions(+), 29 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 1a7935d9f5..9d70db165e 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -889,47 +889,53 @@ restart_ctdb () echo -n " (scheduled)" fi echo "..." - - local i=0 - while : ; do + + local i + for i in $(seq 1 5) ; do if [ -n "$CTDB_NODES_SOCKETS" ] ; then daemons_stop daemons_start "$@" else onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" - fi && break + fi || { + echo "Restart failed. Trying again in a few seconds..." + sleep_for 5 + continue + } - i=$(($i + 1)) - [ $i -lt 5 ] || break + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { + echo "Cluster didn't become healthy. Restarting..." + continue + } - echo "That didn't seem to work - sleeping for a while..." - sleep_for 5 - done - - onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 + local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) - local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" - echo "Setting RerecoveryTimeout to 1" - onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + # In recent versions of CTDB, forcing a recovery like this + # blocks until the recovery is complete. Hopefully this will + # help the cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover - # In recent versions of CTDB, forcing a recovery like this blocks - # until the recovery is complete. Hopefully this will help the - # cluster to stabilise before a subsequent test. - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover + # Cluster is still healthy. Good, we're done! + if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster become UNHEALTHY again. Restarting..." + continue + fi - echo "ctdb is ready" + echo "ctdb is ready" + return 0 + done - if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then - echo "OUCH! Cluster is UNHEALTHY again..." - echo "$debug_out" - # Try to make the calling test fail - status=1 - fi + echo "Cluster UNHEALTHY... too many attempts..." + # Try to make the calling test fail + status=1 + return 1 } ctdb_restart_when_done () -- cgit From f5115a9ea63894f36a4556f3741e386a4d336a71 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 11 Feb 2011 15:13:37 +1100 Subject: Allow proxy_node to be specified for wait_until_node_has_status(). Default to "any"... but allow specification because sometimes it matters... Signed-off-by: Martin Schwenke (This used to be ctdb commit c12c97598afcd07ce4876b26e0b734bc825e54c1) --- ctdb/tests/scripts/ctdb_test_functions.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 9d70db165e..108331bbc5 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -488,10 +488,11 @@ wait_until_node_has_status () local pnn="$1" local status="$2" local timeout="${3:-30}" + local proxy_pnn="${4:-any}" echo "Waiting until node $pnn has status \"$status\"..." - if ! wait_until $timeout onnode any $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then + if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do echo "$i" $i || true -- cgit From 2042931ae6b808db62f0e4d6c3896d07fff2b0de Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 12 Oct 2010 16:49:42 +1100 Subject: Test suite: print debug output after cluster (un)healthy flip-flop on restart. Signed-off-by: Martin Schwenke (This used to be ctdb commit 0e14213dfa841080c07fa6fce23b192493adb926) --- ctdb/tests/scripts/ctdb_test_functions.bash | 1 + 1 file changed, 1 insertion(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 108331bbc5..13797dc89f 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -934,6 +934,7 @@ restart_ctdb () done echo "Cluster UNHEALTHY... too many attempts..." + echo "$debug_out" # Try to make the calling test fail status=1 return 1 -- cgit From 4b606fbb56d53c0050cf206e2c6b0f24b96483fc Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 2 Sep 2011 13:20:50 +1000 Subject: Tests - simple integration - do a "ctdb sync" after restarting the cluster There looks to be a minor race where IPs haven't yet been reallocated but the cluster is healthy. This should fix it. Signed-off-by: Martin Schwenke (This used to be ctdb commit 2d6a800a789ca59fdab92422f98a4e05ba55f34c) --- ctdb/tests/scripts/ctdb_test_functions.bash | 3 +++ 1 file changed, 3 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 13797dc89f..670b91e6f1 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -929,6 +929,9 @@ restart_ctdb () continue fi + echo "Doing a sync..." + onnode -q 0 $CTDB sync + echo "ctdb is ready" return 0 done -- cgit From 88c74987dc90fa09bb8501f20a509c12480fba2e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 13 Feb 2012 13:34:57 +1100 Subject: Tests - functions/environment - set and use $CTDB_NODES "ctdb listnodes" changed so that it never tries to contact the daemon but reads the local nodes file instead. This fails if the nodes file is in a non-default place but $CTDB_NODES isn't set. Signed-off-by: Martin Schwenke (This used to be ctdb commit a7ad2fb75f06791508dd928d2a0c305fc7f7b814) --- ctdb/tests/scripts/ctdb_test_env | 2 ++ ctdb/tests/scripts/ctdb_test_functions.bash | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index 5c3006e25d..f550d7945f 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -32,6 +32,8 @@ if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then done PATH="${CTDB_DIR}/bin:${CTDB_DIR}/tests/bin:${PATH}" + + export CTDB_NODES="$var_dir/nodes.txt" fi # If $VALGRIND is set then use it whenever ctdb is called, but only if diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 670b91e6f1..256ad99223 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -720,10 +720,9 @@ daemons_setup () mkdir -p $var_dir/test.db/persistent - local nodes=$var_dir/nodes.txt local public_addresses=$var_dir/public_addresses.txt local no_public_addresses=$var_dir/no_public_addresses.txt - rm -f $nodes $public_addresses $no_public_addresses + rm -f $CTDB_NODES $public_addresses $no_public_addresses # If there are (strictly) greater than 2 nodes then we'll randomly # choose a node to have no public addresses. @@ -737,7 +736,7 @@ daemons_setup () echo ::$i >> $nodes ip addr add ::$i/128 dev lo else - echo 127.0.0.$i >> $nodes + echo 127.0.0.$i >> $CTDB_NODES # 2 public addresses on most nodes, just to make things interesting. if [ $(($i - 1)) -ne $no_public_ips ] ; then echo "192.0.2.$i/24 lo" >> $public_addresses @@ -754,7 +753,6 @@ daemons_start_1 () local var_dir=$CTDB_DIR/tests/var - local nodes=$var_dir/nodes.txt local public_addresses=$var_dir/public_addresses.txt local no_public_addresses=$var_dir/no_public_addresses.txt @@ -765,7 +763,7 @@ daemons_start_1 () echo "Node $no_public_ips will have no public IPs." fi - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" if [ $(id -u) -eq 0 ]; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From 68ae5f7a3e9ab20a1c73f34ee4cea3cb1f4a8d15 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 28 Mar 2012 13:56:03 +1100 Subject: tests: Set the debug level = 3 when running local tests Signed-off-by: Amitay Isaacs (This used to be ctdb commit b86b947797c51e3576c6b34f547434c3f0aa36f3) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 256ad99223..c09a0672d5 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -763,7 +763,7 @@ daemons_start_1 () echo "Node $no_public_ips will have no public IPs." fi - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" if [ $(id -u) -eq 0 ]; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From ad4d5263e7211eb63587121b11dc1c24fda816f3 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 28 Mar 2012 14:08:08 +1100 Subject: tests: Use CTDB_TEST_REAL_CLUSTER to decide if tests use local daemons Signed-off-by: Amitay Isaacs (This used to be ctdb commit d5b2ad651495f32091bd33d30871638de0de633a) --- ctdb/tests/scripts/ctdb_test_functions.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index c09a0672d5..063f1c3a3e 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -765,7 +765,7 @@ daemons_start_1 () local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" - if [ $(id -u) -eq 0 ]; then + if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then ctdb_options="$ctdb_options --public-interface=lo" fi -- cgit From d4785e813aa5c2abf651c105edb1a8874afd1137 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 29 Mar 2012 14:54:22 +1100 Subject: Tests - turn of time logging by default We haven't seen problems related to time jumps for a long time. Turn this off by default. To switch it back on set $CTDB_TEST_TIME_LOGGING to any non-null value. Signed-off-by: Martin Schwenke (This used to be ctdb commit 2aa9bbf3a52dde0707eb06acd91e57c8da5c717f) --- ctdb/tests/scripts/ctdb_test_functions.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 256ad99223..95a9cd1f87 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -116,7 +116,8 @@ ctdb_test_exit () echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." - if [ -n "$CTDB_TEST_REAL_CLUSTER" -a $status -ne 0 ] ; then + if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ + $status -ne 0 ] ; then ctdb_check_time_logs fi -- cgit From a530358b2f0d1db145ddef002c678b548211c5a1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:27:22 +1000 Subject: tests - export new variable TEST_SCRIPTS_DIR This replaces previous script-local variable ctdb_test_scripts_dir. Signed-off-by: Martin Schwenke (This used to be ctdb commit 107b465172205cb304549fcffaf36b9416696c15) --- ctdb/tests/scripts/ctdb_test_env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index f550d7945f..18d100f3d8 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -1,14 +1,14 @@ #!/bin/bash -ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) -export CTDB_DIR=$(dirname $(dirname $ctdb_test_scripts_dir)) +export TEST_SCRIPTS_DIR=$(cd $(dirname $0) ; pwd) +export CTDB_DIR=$(dirname $(dirname $TEST_SCRIPTS_DIR)) var_dir=$CTDB_DIR/tests/var ###################################################################### ctdb_tools_dir=$CTDB_DIR/tools -PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}" +PATH="${TEST_SCRIPTS_DIR}:${ctdb_tools_dir}:${PATH}" export CTDB_TIMEOUT=60 @@ -17,7 +17,7 @@ export CTDB_TIMEOUT=60 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" else - CTDB_TEST_WRAPPER="${ctdb_test_scripts_dir}/test_wrap" + CTDB_TEST_WRAPPER="${TEST_SCRIPTS_DIR}/test_wrap" fi export CTDB_TEST_WRAPPER -- cgit From 2c141e6c05b7d512dd690cba5dd9a2c9c04ba472 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:32:55 +1000 Subject: tests - add scripts/common.sh Signed-off-by: Martin Schwenke (This used to be ctdb commit a7ad94fe9f2e773567dbb6500469dd2dd2f2f04b) --- ctdb/tests/scripts/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ctdb/tests/scripts/common.sh (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh new file mode 100644 index 0000000000..581663c438 --- /dev/null +++ b/ctdb/tests/scripts/common.sh @@ -0,0 +1,9 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Common variables and functions for all CTDB tests. + +# Print a message and exit. +die () +{ + echo "$1" >&2 ; exit ${2:-1} +} -- cgit From f6178fcc9db4e05bddba1af16fd5ef6c56e9a515 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:33:37 +1000 Subject: tests - add scripts/unit.sh This will be sourced by all unit tests. Signed-off-by: Martin Schwenke (This used to be ctdb commit afdaa5f032938d56ff315d9553cb285ebc413c4c) --- ctdb/tests/scripts/unit.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ctdb/tests/scripts/unit.sh (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/unit.sh b/ctdb/tests/scripts/unit.sh new file mode 100644 index 0000000000..977cc9ae9c --- /dev/null +++ b/ctdb/tests/scripts/unit.sh @@ -0,0 +1,16 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +. "${TEST_SCRIPTS_DIR}/common.sh" + +# Common variables and functions for CTDB unit tests. + +required_result () +{ + required_rc="${1:-0}" + required_output=$(cat) +} + +local="${TEST_SUBDIR}/scripts/local.sh" +if [ -r "$local" ] ; then + . "$local" +fi -- cgit From fec6fe628736c2b20ceeb53a43faa1ebff0a7ff0 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:48:49 +1000 Subject: tests - move functions only used by scripts/run_tests into that script Along with minor logic tweaks and removal of test_exit(). Signed-off-by: Martin Schwenke (This used to be ctdb commit 00713eb46cce638339845799bba2da041b3d02fb) --- ctdb/tests/scripts/ctdb_test_functions.bash | 61 ---------------------------- ctdb/tests/scripts/run_tests | 63 ++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 62 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 95a9cd1f87..e729486f55 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -8,51 +8,6 @@ fail () ###################################################################### -ctdb_test_begin () -{ - local name="$1" - - teststarttime=$(date '+%s') - testduration=0 - - echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" - echo "Running test $name ($(date '+%T'))" - echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" -} - -ctdb_test_end () -{ - local name="$1" ; shift - local status="$1" ; shift - # "$@" is command-line - - local interp="SKIPPED" - local statstr=" (reason $*)" - if [ -n "$status" ] ; then - if [ $status -eq 0 ] ; then - interp="PASSED" - statstr="" - echo "ALL OK: $*" - else - interp="FAILED" - statstr=" (status $status)" - testfailures=$(($testfailures+1)) - fi - fi - - testduration=$(($(date +%s)-$teststarttime)) - - echo "==========================================================================" - echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)" - echo "==========================================================================" - -} - -test_exit () -{ - exit $(($testfailures+0)) -} - ctdb_check_time_logs () { local threshold=20 @@ -144,22 +99,6 @@ ctdb_test_exit_hook_add () ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" } -ctdb_test_run () -{ - local name="$1" ; shift - - [ -n "$1" ] || set -- "$name" - - ctdb_test_begin "$name" - - local status=0 - "$@" || status=$? - - ctdb_test_end "$name" "$status" "$*" - - return $status -} - ctdb_test_usage() { local status=${1:-2} diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 1ce089d852..394b4150ac 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -46,8 +46,66 @@ fi ###################################################################### +ctdb_test_begin () +{ + local name="$1" + + teststarttime=$(date '+%s') + testduration=0 + + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" + echo "Running test $name ($(date '+%T'))" + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" +} + +ctdb_test_end () +{ + local name="$1" ; shift + local status="$1" ; shift + # "$@" is command-line + + local interp="SKIPPED" + local statstr=" (reason $*)" + if [ -n "$status" ] ; then + if [ $status -eq 0 ] ; then + interp="PASSED" + statstr="" + echo "ALL OK: $*" + else + interp="FAILED" + statstr=" (status $status)" + fi + fi + + testduration=$(($(date +%s)-$teststarttime)) + + echo "==========================================================================" + echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)" + echo "==========================================================================" + +} + +ctdb_test_run () +{ + local name="$1" ; shift + + [ -n "$1" ] || set -- "$name" + + ctdb_test_begin "$name" + + local status=0 + "$@" || status=$? + + ctdb_test_end "$name" "$status" "$*" + + return $status +} + +###################################################################### + tests_total=0 tests_passed=0 +tests_failed=0 summary="" rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's@^0$@80@') @@ -69,6 +127,7 @@ for f; do t=" PASSED " else t="*FAILED*" + tests_failed=$(($tests_failed + 1)) fi if $with_desc ; then desc=$(tail -n +4 $tf | head -n 1) @@ -89,4 +148,6 @@ fi rm -f "$sf" -test_exit +if [ $tests_failed -gt 0 ] ; then + exit 1 +fi -- cgit From 6eb8bf776ec0dc240a773ded0a7aaf99312dd271 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:51:22 +1000 Subject: tests - add -v option to set TEST_VERBOSE=true Signed-off-by: Martin Schwenke (This used to be ctdb commit 43badc5418b9f533398cd579607d9f1fc0f8f417) --- ctdb/tests/scripts/run_tests | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 394b4150ac..51a89bb3a9 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -21,7 +21,9 @@ with_summary=false with_desc=false quiet=false -temp=$(getopt -n "$prog" -o "xdhqs" -l help -- "$@") +export TEST_VERBOSE=false + +temp=$(getopt -n "$prog" -o "xdhqsv" -l help -- "$@") [ $? != 0 ] && usage @@ -33,6 +35,7 @@ while true ; do -d) with_desc=true ; shift ;; # 4th line of output is description -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; + -v) TEST_VERBOSE=true ; shift ;; --) shift ; break ;; *) usage ;; esac -- cgit From 9267727edb3c55665578a5a868127806ea39403e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:52:17 +1000 Subject: tests - run_tests can take a directory as an argument This makes it run all tests in the specified directory. Signed-off-by: Martin Schwenke (This used to be ctdb commit 89719384a74161ffa0c03602ecdd9e758d521d75) --- ctdb/tests/scripts/run_tests | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 51a89bb3a9..daa4b9c412 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -119,10 +119,16 @@ sf=$(mktemp) set -o pipefail -for f; do - [ -x $f ] || fail "test \"$f\" is not executable" +run_one_test () +{ + _f="$1" + + [ -x "$_f" ] || fail "test \"$_f\" is not executable" tests_total=$(($tests_total + 1)) - ctdb_test_run "$f" | tee "$tf" | show_progress + + export TEST_SUBDIR=$(dirname "$_f") + + ctdb_test_run "$_f" | tee "$tf" | show_progress status=$? if $with_summary ; then if [ $status -eq 0 ] ; then @@ -134,9 +140,21 @@ for f; do fi if $with_desc ; then desc=$(tail -n +4 $tf | head -n 1) - f="$desc" + _f="$desc" fi - echo "$t $f" >>"$sf" + echo "$t $_f" >>"$sf" + fi +} + +for f ; do + if [ -d "$f" ] ; then + for i in $(ls "$f/"*".sh" 2>/dev/null) ; do + run_one_test "$i" + done + elif [ -f "$f" ] ; then + run_one_test "$f" + else + fail "test \"$f\" is not recognised" fi done -- cgit From 8e71c5ec902544e4c1941098cd2864a56a03dc94 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:54:51 +1000 Subject: tests - run_tests includes common.sh, uses die() Signed-off-by: Martin Schwenke (This used to be ctdb commit 21df43c74bfcff420fdaf9df5440c25529c543d2) --- ctdb/tests/scripts/run_tests | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index daa4b9c412..c2ff15a69b 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -5,7 +5,7 @@ # the arguments that it sees. . $(dirname $0)/ctdb_test_env : -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/common.sh" usage() { cat < Date: Mon, 16 Apr 2012 15:58:44 +1000 Subject: tests - run_tests ignores trailing '/' on directories Signed-off-by: Martin Schwenke (This used to be ctdb commit dbec696930327ff07b39282e3084eef4ded064c0) --- ctdb/tests/scripts/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index c2ff15a69b..8fce3ae978 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -148,7 +148,7 @@ run_one_test () for f ; do if [ -d "$f" ] ; then - for i in $(ls "$f/"*".sh" 2>/dev/null) ; do + for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do run_one_test "$i" done elif [ -f "$f" ] ; then -- cgit From feff13ff6328b85fae1581c53ba4b0d617b799c2 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 16:27:20 +1000 Subject: tests - run_tests needs to expand directories like "." and "..". Signed-off-by: Martin Schwenke (This used to be ctdb commit fd08fc6c88cb80190ce87325867de0391cf1af51) --- ctdb/tests/scripts/run_tests | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 8fce3ae978..5e2de17c80 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -148,6 +148,10 @@ run_one_test () for f ; do if [ -d "$f" ] ; then + # This expands the most probable problem cases like "." and "..". + if [ $(dirname "$f") = "." ] ; then + f=$(cd "$f" ; pwd) + fi for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do run_one_test "$i" done -- cgit From 116f19b808ace4a74bcf56a638923dcd1e1e975d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 18 Apr 2012 10:37:45 +1000 Subject: tests: More unit test factoring/rationalisation and bug fixes Move some options from eventscripts/run_tests.sh to scripts/run_tests. Remove the former. Move some functions from eventscripts/scripts/local.sh to scripts/unit.sh. Both of these are modified during move so they are no longer eventscript-specific. Tweak */local.sh so that the new functions in unit.sh are used. Signed-off-by: Martin Schwenke (This used to be ctdb commit 7ff485687891732074c9fc9998502ca197663d02) --- ctdb/tests/scripts/run_tests | 17 +++++- ctdb/tests/scripts/unit.sh | 127 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 5e2de17c80..3a5ec96b74 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -11,6 +11,15 @@ usage() { cat <"$_outr" + + _outf=$(mktemp) + echo "$_out" >"$_outf" + + cat < Date: Wed, 18 Apr 2012 14:55:21 +1000 Subject: tests: Rename ctdb_test_functions.bash to integration.bash Signed-off-by: Martin Schwenke (This used to be ctdb commit 79adb50b3ce3873c3baf9e6715c1d1c3f181ce43) --- ctdb/tests/scripts/ctdb_test_functions.bash | 1053 --------------------------- ctdb/tests/scripts/integration.bash | 1053 +++++++++++++++++++++++++++ ctdb/tests/scripts/test_wrap | 4 +- 3 files changed, 1055 insertions(+), 1055 deletions(-) delete mode 100644 ctdb/tests/scripts/ctdb_test_functions.bash create mode 100644 ctdb/tests/scripts/integration.bash (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash deleted file mode 100644 index 70c5b5d10b..0000000000 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ /dev/null @@ -1,1053 +0,0 @@ -# Hey Emacs, this is a -*- shell-script -*- !!! :-) - -fail () -{ - echo "$*" - exit 1 -} - -###################################################################### - -ctdb_check_time_logs () -{ - local threshold=20 - - local jump=false - local prev="" - local ds_prev="" - local node="" - - out=$(onnode all tail -n 20 /var/log/ctdb.test.time.log 2>&1) - - if [ $? -eq 0 ] ; then - local line - while read line ; do - case "$line" in - \>\>\ NODE:\ *\ \<\<) - node="${line#>> NODE: }" - node=${node% <<*} - ds_prev="" - ;; - *\ *) - set -- $line - ds_curr="$1${2:0:1}" - if [ -n "$ds_prev" ] && \ - [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then - echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})" - jump=true - fi - prev="$line" - ds_prev="$ds_curr" - ;; - esac - done <<<"$out" - else - echo Error getting time logs - fi - if $jump ; then - echo "Check time sync (test client first):" - date - onnode -p all date - echo "Information from test client:" - hostname - top -b -n 1 - echo "Information from cluster nodes:" - onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo" - fi -} - -ctdb_test_exit () -{ - local status=$? - - trap - 0 - - [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status - status=$(($testfailures+0)) - - # Avoid making a test fail from this point onwards. The test is - # now complete. - set +e - - echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." - - if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ - $status -ne 0 ] ; then - ctdb_check_time_logs - fi - - eval "$ctdb_test_exit_hook" || true - unset ctdb_test_exit_hook - - if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then - - restart_ctdb - else - # This could be made unconditional but then we might get - # duplication from the recovery in restart_ctdb. We want to - # leave the recovery in restart_ctdb so that future tests that - # might do a manual restart mid-test will benefit. - echo "Forcing a recovery..." - onnode 0 $CTDB recover - fi - - exit $status -} - -ctdb_test_exit_hook_add () -{ - ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" -} - -ctdb_test_usage() -{ - local status=${1:-2} - - cat <&1) || { - - echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" - echo "$out" - return 1 - } - - if $verbose ; then - echo "Output of \"$cmd\":" - echo "$out" - fi -} - -sanity_check_output () -{ - local min_lines="$1" - local regexp="$2" # Should be anchored as necessary. - local output="$3" - - local ret=0 - - local num_lines=$(echo "$output" | wc -l) - echo "There are $num_lines lines of output" - if [ $num_lines -lt $min_lines ] ; then - echo "BAD: that's less than the required number (${min_lines})" - ret=1 - fi - - local status=0 - local unexpected # local doesn't pass through status of command on RHS. - unexpected=$(echo "$output" | egrep -v "$regexp") || status=$? - - # Note that this is reversed. - if [ $status -eq 0 ] ; then - echo "BAD: unexpected lines in output:" - echo "$unexpected" | cat -A - ret=1 - else - echo "Output lines look OK" - fi - - return $ret -} - -sanity_check_ips () -{ - local ips="$1" # list of "ip node" lines - - echo "Sanity checking IPs..." - - local x ipp prev - prev="" - while read x ipp ; do - [ "$ipp" = "-1" ] && break - if [ -n "$prev" -a "$ipp" != "$prev" ] ; then - echo "OK" - return 0 - fi - prev="$ipp" - done <<<"$ips" - - echo "BAD: a node was -1 or IPs are only assigned to one node" - echo "Are you running an old version of CTDB?" - return 1 -} - -# This returns a list of "ip node" lines in $out -all_ips_on_node() -{ - local node=$@ - try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" -} - -select_test_node_and_ips () -{ - all_ips_on_node 0 - - # When selecting test_node we just want a node that has public - # IPs. This will work and is economically semi-random. :-) - local x - read x test_node <<<"$out" - - test_node_ips="" - local ip pnn - while read ip pnn ; do - if [ "$pnn" = "$test_node" ] ; then - test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" - fi - done <<<"$out" # bashism to avoid problem setting variable in pipeline. - - echo "Selected node ${test_node} with IPs: ${test_node_ips}." - test_ip="${test_node_ips%% *}" -} - -####################################### - -# Wait until either timeout expires or command succeeds. The command -# will be tried once per second. -wait_until () -{ - local timeout="$1" ; shift # "$@" is the command... - - local negate=false - if [ "$1" = "!" ] ; then - negate=true - shift - fi - - echo -n "<${timeout}|" - local t=$timeout - while [ $t -gt 0 ] ; do - local rc=0 - "$@" || rc=$? - if { ! $negate && [ $rc -eq 0 ] ; } || \ - { $negate && [ $rc -ne 0 ] ; } ; then - echo "|$(($timeout - $t))|" - echo "OK" - return 0 - fi - echo -n . - t=$(($t - 1)) - sleep 1 - done - - echo "*TIMEOUT*" - - return 1 -} - -sleep_for () -{ - echo -n "=${1}|" - for i in $(seq 1 $1) ; do - echo -n '.' - sleep 1 - done - echo '|' -} - -_cluster_is_healthy () -{ - local out x count line - - out=$($CTDB -Y status 2>/dev/null) || return 1 - - { - read x - count=0 - while read line ; do - # We need to see valid lines if we're going to be healthy. - [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1)) - # A line indicating a node is unhealthy causes failure. - [ "${line##:*:*:*1:}" != "$line" ] && return 1 - done - [ $count -gt 0 ] && return $? - } <<<"$out" # Yay bash! -} - -cluster_is_healthy () -{ - if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then - echo "Cluster is HEALTHY" - return 0 - else - echo "Cluster is UNHEALTHY" - if ! ${ctdb_test_restart_scheduled:-false} ; then - echo "DEBUG AT $(date '+%F %T'):" - local i - for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do - echo "$i" - $i || true - done - fi - return 1 - fi -} - -wait_until_healthy () -{ - local timeout="${1:-120}" - - echo "Waiting for cluster to become healthy..." - - wait_until 120 _cluster_is_healthy -} - -# This function is becoming nicely overloaded. Soon it will collapse! :-) -node_has_status () -{ - local pnn="$1" - local status="$2" - - local bits fpat mpat - case "$status" in - (unhealthy) bits="?:?:?:1:*" ;; - (healthy) bits="?:?:?:0:*" ;; - (disconnected) bits="1:*" ;; - (connected) bits="0:*" ;; - (banned) bits="?:1:*" ;; - (unbanned) bits="?:0:*" ;; - (disabled) bits="?:?:1:*" ;; - (enabled) bits="?:?:0:*" ;; - (stopped) bits="?:?:?:?:1:*" ;; - (notstopped) bits="?:?:?:?:0:*" ;; - (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; - (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; - (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; - (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; - *) - echo "node_has_status: unknown status \"$status\"" - return 1 - esac - - if [ -n "$bits" ] ; then - local out x line - - out=$($CTDB -Y status 2>&1) || return 1 - - { - read x - while read line ; do - # This needs to be done in 2 steps to avoid false matches. - local line_bits="${line#:${pnn}:*:}" - [ "$line_bits" = "$line" ] && continue - [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 - done - return 1 - } <<<"$out" # Yay bash! - elif [ -n "$fpat" ] ; then - $CTDB statistics -n "$pnn" | egrep -q "$fpat" - elif [ -n "$mpat" ] ; then - $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" - else - echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' - return 1 - fi -} - -wait_until_node_has_status () -{ - local pnn="$1" - local status="$2" - local timeout="${3:-30}" - local proxy_pnn="${4:-any}" - - echo "Waiting until node $pnn has status \"$status\"..." - - if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then - for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do - echo "$i" - $i || true - done - - return 1 - fi - -} - -# Useful for superficially testing IP failover. -# IPs must be on nodes matching nodeglob. -ips_are_on_nodeglob () -{ - local nodeglob="$1" ; shift - local ips="$*" - - local out - - all_ips_on_node 1 - - while read ip pnn ; do - for check in $ips ; do - if [ "$check" = "$ip" ] ; then - case "$pnn" in - ($nodeglob) : ;; - (*) return 1 ;; - esac - ips="${ips/${ip}}" # Remove from list - fi - done - done <<<"$out" # bashism to avoid problem setting variable in pipeline. - - ips="${ips// }" # Remove any spaces. - [ -z "$ips" ] -} - -wait_until_ips_are_on_nodeglob () -{ - echo "Waiting for IPs to fail over..." - - wait_until 60 ips_are_on_nodeglob "$@" -} - -node_has_some_ips () -{ - local node="$1" - - local out - - all_ips_on_node 1 - - while read ip pnn ; do - if [ "$node" = "$pnn" ] ; then - return 0 - fi - done <<<"$out" # bashism to avoid problem setting variable in pipeline. - - return 1 -} - -wait_until_node_has_some_ips () -{ - echo "Waiting for node to have some IPs..." - - wait_until 60 node_has_some_ips "$@" -} - -get_src_socket () -{ - local proto="$1" - local dst_socket="$2" - local pid="$3" - local prog="$4" - - local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" - out=$(netstat -tanp | - egrep "$pat" | - awk '{ print $4 }') - - [ -n "$out" ] -} - -wait_until_get_src_socket () -{ - local proto="$1" - local dst_socket="$2" - local pid="$3" - local prog="$4" - - echo "Waiting for ${prog} to establish connection to ${dst_socket}..." - - wait_until 5 get_src_socket "$@" -} - -####################################### - -# filename will be in $tcpdump_filename, pid in $tcpdump_pid -tcpdump_start () -{ - tcpdump_filter="$1" # global - - echo "Running tcpdump..." - tcpdump_filename=$(mktemp) - ctdb_test_exit_hook_add "rm -f $tcpdump_filename" - - # The only way of being sure that tcpdump is listening is to send - # some packets that it will see. So we use dummy pings - the -U - # option to tcpdump ensures that packets are flushed to the file - # as they are captured. - local dummy_addr="127.3.2.1" - local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" - tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & - ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" - - echo "Waiting for tcpdump output file to be ready..." - ping -q "$dummy_addr" >/dev/null 2>&1 & - ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" - - tcpdump_listen_for_dummy () - { - tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 - } - - wait_until 10 tcpdump_listen_for_dummy -} - -# By default, wait for 1 matching packet. -tcpdump_wait () -{ - local count="${1:-1}" - local filter="${2:-${tcpdump_filter}}" - - tcpdump_check () - { - local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) - [ $found -ge $count ] - } - - echo "Waiting for tcpdump to capture some packets..." - if ! wait_until 30 tcpdump_check ; then - echo "DEBUG AT $(date '+%F %T'):" - local i - for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do - echo "$i" - $i || true - done - return 1 - fi -} - -tcpdump_show () -{ - local filter="${1:-${tcpdump_filter}}" - - tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null -} - -tcptickle_sniff_start () -{ - local src="$1" - local dst="$2" - - local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" - local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" - local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 - local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" - tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" - local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" - - tcpdump_start "$filter" -} - -tcptickle_sniff_wait_show () -{ - tcpdump_wait 1 "$tcptickle_reset" - - echo "GOOD: here are some TCP tickle packets:" - tcpdump_show -} - -gratarp_sniff_start () -{ - tcpdump_start "arp host ${test_ip}" -} - -gratarp_sniff_wait_show () -{ - tcpdump_wait 2 - - echo "GOOD: this should be the some gratuitous ARPs:" - tcpdump_show -} - - -####################################### - -daemons_stop () -{ - echo "Attempting to politely shutdown daemons..." - onnode 1 $CTDB shutdown -n all || true - - echo "Sleeping for a while..." - sleep_for 1 - - if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then - echo "Killing remaining daemons..." - pkill -f $CTDB_DIR/bin/ctdbd - - if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then - echo "Once more with feeling.." - pkill -9 $CTDB_DIR/bin/ctdbd - fi - fi - - local var_dir=$CTDB_DIR/tests/var - rm -rf $var_dir/test.db -} - -daemons_setup () -{ - local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes - - local var_dir=$CTDB_DIR/tests/var - - mkdir -p $var_dir/test.db/persistent - - local public_addresses=$var_dir/public_addresses.txt - local no_public_addresses=$var_dir/no_public_addresses.txt - rm -f $CTDB_NODES $public_addresses $no_public_addresses - - # If there are (strictly) greater than 2 nodes then we'll randomly - # choose a node to have no public addresses. - local no_public_ips=-1 - [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes)) - echo "$no_public_ips" >$no_public_addresses - - local i - for i in $(seq 1 $num_nodes) ; do - if [ "${CTDB_USE_IPV6}x" != "x" ]; then - echo ::$i >> $nodes - ip addr add ::$i/128 dev lo - else - echo 127.0.0.$i >> $CTDB_NODES - # 2 public addresses on most nodes, just to make things interesting. - if [ $(($i - 1)) -ne $no_public_ips ] ; then - echo "192.0.2.$i/24 lo" >> $public_addresses - echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses - fi - fi - done -} - -daemons_start_1 () -{ - local pnn="$1" - shift # "$@" gets passed to ctdbd - - local var_dir=$CTDB_DIR/tests/var - - local public_addresses=$var_dir/public_addresses.txt - local no_public_addresses=$var_dir/no_public_addresses.txt - - local no_public_ips=-1 - [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses - - if [ "$no_public_ips" = $pnn ] ; then - echo "Node $no_public_ips will have no public IPs." - fi - - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" - - if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then - ctdb_options="$ctdb_options --public-interface=lo" - fi - - if [ $pnn -eq $no_public_ips ] ; then - ctdb_options="$ctdb_options --public-addresses=/dev/null" - else - ctdb_options="$ctdb_options --public-addresses=$public_addresses" - fi - - # Need full path so we can use "pkill -f" to kill the daemons. - $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$pnn $ctdb_options "$@" ||return 1 -} - -daemons_start () -{ - # "$@" gets passed to ctdbd - - local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes - - echo "Starting $num_nodes ctdb daemons..." - - for i in $(seq 0 $(($num_nodes - 1))) ; do - daemons_start_1 $i "$@" - done - - local var_dir=$CTDB_DIR/tests/var - - if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then - ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 - fi -} - -####################################### - -_ctdb_hack_options () -{ - local ctdb_options="$*" - - # We really just want to pass CTDB_OPTIONS but on RH - # /etc/sysconfig/ctdb can, and frequently does, set that variable. - # So instead, we hack badly. We'll add these as we use them. - # Note that these may still be overridden by the above file... but - # we tend to use the exotic options here... so that is unlikely. - - case "$ctdb_options" in - *--start-as-stopped*) - export CTDB_START_AS_STOPPED="yes" - esac -} - -_restart_ctdb () -{ - _ctdb_hack_options "$@" - - if [ -e /etc/redhat-release ] ; then - service ctdb restart - else - /etc/init.d/ctdb restart - fi -} - -_ctdb_start () -{ - _ctdb_hack_options "$@" - - /etc/init.d/ctdb start -} - -setup_ctdb () -{ - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_setup - fi -} - -# Common things to do after starting one or more nodes. -_ctdb_start_post () -{ - onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 - - echo "Setting RerecoveryTimeout to 1" - onnode -pq all "$CTDB setvar RerecoveryTimeout 1" - - # In recent versions of CTDB, forcing a recovery like this blocks - # until the recovery is complete. Hopefully this will help the - # cluster to stabilise before a subsequent test. - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - - echo "ctdb is ready" -} - -# This assumes that ctdbd is not running on the given node. -ctdb_start_1 () -{ - local pnn="$1" - shift # "$@" is passed to ctdbd start. - - echo -n "Starting CTDB on node ${pnn}..." - - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_start_1 $pnn "$@" - else - onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@" - fi - - # If we're starting only 1 node then we're doing something weird. - ctdb_restart_when_done -} - -restart_ctdb () -{ - # "$@" is passed to ctdbd start. - - echo -n "Restarting CTDB" - if $ctdb_test_restart_scheduled ; then - echo -n " (scheduled)" - fi - echo "..." - - local i - for i in $(seq 1 5) ; do - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_stop - daemons_start "$@" - else - onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" - fi || { - echo "Restart failed. Trying again in a few seconds..." - sleep_for 5 - continue - } - - onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { - echo "Cluster didn't become healthy. Restarting..." - continue - } - - local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) - - echo "Setting RerecoveryTimeout to 1" - onnode -pq all "$CTDB setvar RerecoveryTimeout 1" - - # In recent versions of CTDB, forcing a recovery like this - # blocks until the recovery is complete. Hopefully this will - # help the cluster to stabilise before a subsequent test. - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - - # Cluster is still healthy. Good, we're done! - if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then - echo "Cluster become UNHEALTHY again. Restarting..." - continue - fi - - echo "Doing a sync..." - onnode -q 0 $CTDB sync - - echo "ctdb is ready" - return 0 - done - - echo "Cluster UNHEALTHY... too many attempts..." - echo "$debug_out" - # Try to make the calling test fail - status=1 - return 1 -} - -ctdb_restart_when_done () -{ - ctdb_test_restart_scheduled=true -} - -####################################### - -install_eventscript () -{ - local script_name="$1" - local script_contents="$2" - - if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then - # The quoting here is *very* fragile. However, we do - # experience the joy of installing a short script using - # onnode, and without needing to know the IP addresses of the - # nodes. - onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" - else - f="${CTDB_DIR}/tests/events.d/${script_name}" - echo "$script_contents" >"$f" - chmod 755 "$f" - fi -} - -uninstall_eventscript () -{ - local script_name="$1" - - if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then - onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" - else - rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" - fi -} - -####################################### - -# This section deals with the 99.ctdb_test eventscript. - -# Metafunctions: Handle a ctdb-test file on a node. -# given event. -ctdb_test_eventscript_file_create () -{ - local pnn="$1" - local type="$2" - - try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}" -} - -ctdb_test_eventscript_file_remove () -{ - local pnn="$1" - local type="$2" - - try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}" -} - -ctdb_test_eventscript_file_exists () -{ - local pnn="$1" - local type="$2" - - try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1 -} - - -# Handle a flag file on a node that is removed by 99.ctdb_test on the -# given event. -ctdb_test_eventscript_flag () -{ - local cmd="$1" - local pnn="$2" - local event="$3" - - ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}" -} - - -# Handle a trigger that causes 99.ctdb_test to fail it's monitor -# event. -ctdb_test_eventscript_unhealthy_trigger () -{ - local cmd="$1" - local pnn="$2" - - ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger" -} - -# Handle the file that 99.ctdb_test created to show that it has marked -# a node unhealthy because it detected the above trigger. -ctdb_test_eventscript_unhealthy_detected () -{ - local cmd="$1" - local pnn="$2" - - ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" -} - -# Handle a trigger that causes 99.ctdb_test to timeout it's monitor -# event. This should cause the node to be banned. -ctdb_test_eventscript_timeout_trigger () -{ - local cmd="$1" - local pnn="$2" - local event="$3" - - ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout" -} - -# Note that the eventscript can't use the above functions! -ctdb_test_eventscript_install () -{ - - local script='#!/bin/sh -out=$(ctdb pnn) -pnn="${out#PNN:}" - -rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" - -trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" -detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" -timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}" -case "$1" in - monitor) - if [ -e "$trigger" ] ; then - echo "${0}: Unhealthy because \"$trigger\" detected" - touch "$detected" - exit 1 - elif [ -e "$detected" -a ! -e "$trigger" ] ; then - echo "${0}: Healthy again, \"$trigger\" no longer detected" - rm "$detected" - fi - - ;; - *) - if [ -e "$timeout_trigger" ] ; then - echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected" - sleep 9999 - fi - ;; - *) - -esac - -exit 0 -' - install_eventscript "99.ctdb_test" "$script" -} - -ctdb_test_eventscript_uninstall () -{ - uninstall_eventscript "99.ctdb_test" -} - -# Note that this only works if you know all other monitor events will -# succeed. You also need to install the eventscript before using it. -wait_for_monitor_event () -{ - local pnn="$1" - - echo "Waiting for a monitor event on node ${pnn}..." - ctdb_test_eventscript_flag create $pnn "monitor" - - wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" - -} - -# Make sure that $CTDB is set. -: ${CTDB:=ctdb} diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash new file mode 100644 index 0000000000..70c5b5d10b --- /dev/null +++ b/ctdb/tests/scripts/integration.bash @@ -0,0 +1,1053 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +fail () +{ + echo "$*" + exit 1 +} + +###################################################################### + +ctdb_check_time_logs () +{ + local threshold=20 + + local jump=false + local prev="" + local ds_prev="" + local node="" + + out=$(onnode all tail -n 20 /var/log/ctdb.test.time.log 2>&1) + + if [ $? -eq 0 ] ; then + local line + while read line ; do + case "$line" in + \>\>\ NODE:\ *\ \<\<) + node="${line#>> NODE: }" + node=${node% <<*} + ds_prev="" + ;; + *\ *) + set -- $line + ds_curr="$1${2:0:1}" + if [ -n "$ds_prev" ] && \ + [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then + echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})" + jump=true + fi + prev="$line" + ds_prev="$ds_curr" + ;; + esac + done <<<"$out" + else + echo Error getting time logs + fi + if $jump ; then + echo "Check time sync (test client first):" + date + onnode -p all date + echo "Information from test client:" + hostname + top -b -n 1 + echo "Information from cluster nodes:" + onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo" + fi +} + +ctdb_test_exit () +{ + local status=$? + + trap - 0 + + [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + status=$(($testfailures+0)) + + # Avoid making a test fail from this point onwards. The test is + # now complete. + set +e + + echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." + + if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ + $status -ne 0 ] ; then + ctdb_check_time_logs + fi + + eval "$ctdb_test_exit_hook" || true + unset ctdb_test_exit_hook + + if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then + + restart_ctdb + else + # This could be made unconditional but then we might get + # duplication from the recovery in restart_ctdb. We want to + # leave the recovery in restart_ctdb so that future tests that + # might do a manual restart mid-test will benefit. + echo "Forcing a recovery..." + onnode 0 $CTDB recover + fi + + exit $status +} + +ctdb_test_exit_hook_add () +{ + ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" +} + +ctdb_test_usage() +{ + local status=${1:-2} + + cat <&1) || { + + echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" + echo "$out" + return 1 + } + + if $verbose ; then + echo "Output of \"$cmd\":" + echo "$out" + fi +} + +sanity_check_output () +{ + local min_lines="$1" + local regexp="$2" # Should be anchored as necessary. + local output="$3" + + local ret=0 + + local num_lines=$(echo "$output" | wc -l) + echo "There are $num_lines lines of output" + if [ $num_lines -lt $min_lines ] ; then + echo "BAD: that's less than the required number (${min_lines})" + ret=1 + fi + + local status=0 + local unexpected # local doesn't pass through status of command on RHS. + unexpected=$(echo "$output" | egrep -v "$regexp") || status=$? + + # Note that this is reversed. + if [ $status -eq 0 ] ; then + echo "BAD: unexpected lines in output:" + echo "$unexpected" | cat -A + ret=1 + else + echo "Output lines look OK" + fi + + return $ret +} + +sanity_check_ips () +{ + local ips="$1" # list of "ip node" lines + + echo "Sanity checking IPs..." + + local x ipp prev + prev="" + while read x ipp ; do + [ "$ipp" = "-1" ] && break + if [ -n "$prev" -a "$ipp" != "$prev" ] ; then + echo "OK" + return 0 + fi + prev="$ipp" + done <<<"$ips" + + echo "BAD: a node was -1 or IPs are only assigned to one node" + echo "Are you running an old version of CTDB?" + return 1 +} + +# This returns a list of "ip node" lines in $out +all_ips_on_node() +{ + local node=$@ + try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" +} + +select_test_node_and_ips () +{ + all_ips_on_node 0 + + # When selecting test_node we just want a node that has public + # IPs. This will work and is economically semi-random. :-) + local x + read x test_node <<<"$out" + + test_node_ips="" + local ip pnn + while read ip pnn ; do + if [ "$pnn" = "$test_node" ] ; then + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + echo "Selected node ${test_node} with IPs: ${test_node_ips}." + test_ip="${test_node_ips%% *}" +} + +####################################### + +# Wait until either timeout expires or command succeeds. The command +# will be tried once per second. +wait_until () +{ + local timeout="$1" ; shift # "$@" is the command... + + local negate=false + if [ "$1" = "!" ] ; then + negate=true + shift + fi + + echo -n "<${timeout}|" + local t=$timeout + while [ $t -gt 0 ] ; do + local rc=0 + "$@" || rc=$? + if { ! $negate && [ $rc -eq 0 ] ; } || \ + { $negate && [ $rc -ne 0 ] ; } ; then + echo "|$(($timeout - $t))|" + echo "OK" + return 0 + fi + echo -n . + t=$(($t - 1)) + sleep 1 + done + + echo "*TIMEOUT*" + + return 1 +} + +sleep_for () +{ + echo -n "=${1}|" + for i in $(seq 1 $1) ; do + echo -n '.' + sleep 1 + done + echo '|' +} + +_cluster_is_healthy () +{ + local out x count line + + out=$($CTDB -Y status 2>/dev/null) || return 1 + + { + read x + count=0 + while read line ; do + # We need to see valid lines if we're going to be healthy. + [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1)) + # A line indicating a node is unhealthy causes failure. + [ "${line##:*:*:*1:}" != "$line" ] && return 1 + done + [ $count -gt 0 ] && return $? + } <<<"$out" # Yay bash! +} + +cluster_is_healthy () +{ + if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster is HEALTHY" + return 0 + else + echo "Cluster is UNHEALTHY" + if ! ${ctdb_test_restart_scheduled:-false} ; then + echo "DEBUG AT $(date '+%F %T'):" + local i + for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do + echo "$i" + $i || true + done + fi + return 1 + fi +} + +wait_until_healthy () +{ + local timeout="${1:-120}" + + echo "Waiting for cluster to become healthy..." + + wait_until 120 _cluster_is_healthy +} + +# This function is becoming nicely overloaded. Soon it will collapse! :-) +node_has_status () +{ + local pnn="$1" + local status="$2" + + local bits fpat mpat + case "$status" in + (unhealthy) bits="?:?:?:1:*" ;; + (healthy) bits="?:?:?:0:*" ;; + (disconnected) bits="1:*" ;; + (connected) bits="0:*" ;; + (banned) bits="?:1:*" ;; + (unbanned) bits="?:0:*" ;; + (disabled) bits="?:?:1:*" ;; + (enabled) bits="?:?:0:*" ;; + (stopped) bits="?:?:?:?:1:*" ;; + (notstopped) bits="?:?:?:?:0:*" ;; + (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; + (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; + (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; + (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; + *) + echo "node_has_status: unknown status \"$status\"" + return 1 + esac + + if [ -n "$bits" ] ; then + local out x line + + out=$($CTDB -Y status 2>&1) || return 1 + + { + read x + while read line ; do + # This needs to be done in 2 steps to avoid false matches. + local line_bits="${line#:${pnn}:*:}" + [ "$line_bits" = "$line" ] && continue + [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 + done + return 1 + } <<<"$out" # Yay bash! + elif [ -n "$fpat" ] ; then + $CTDB statistics -n "$pnn" | egrep -q "$fpat" + elif [ -n "$mpat" ] ; then + $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" + else + echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' + return 1 + fi +} + +wait_until_node_has_status () +{ + local pnn="$1" + local status="$2" + local timeout="${3:-30}" + local proxy_pnn="${4:-any}" + + echo "Waiting until node $pnn has status \"$status\"..." + + if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then + for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do + echo "$i" + $i || true + done + + return 1 + fi + +} + +# Useful for superficially testing IP failover. +# IPs must be on nodes matching nodeglob. +ips_are_on_nodeglob () +{ + local nodeglob="$1" ; shift + local ips="$*" + + local out + + all_ips_on_node 1 + + while read ip pnn ; do + for check in $ips ; do + if [ "$check" = "$ip" ] ; then + case "$pnn" in + ($nodeglob) : ;; + (*) return 1 ;; + esac + ips="${ips/${ip}}" # Remove from list + fi + done + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + ips="${ips// }" # Remove any spaces. + [ -z "$ips" ] +} + +wait_until_ips_are_on_nodeglob () +{ + echo "Waiting for IPs to fail over..." + + wait_until 60 ips_are_on_nodeglob "$@" +} + +node_has_some_ips () +{ + local node="$1" + + local out + + all_ips_on_node 1 + + while read ip pnn ; do + if [ "$node" = "$pnn" ] ; then + return 0 + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + return 1 +} + +wait_until_node_has_some_ips () +{ + echo "Waiting for node to have some IPs..." + + wait_until 60 node_has_some_ips "$@" +} + +get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" + out=$(netstat -tanp | + egrep "$pat" | + awk '{ print $4 }') + + [ -n "$out" ] +} + +wait_until_get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + echo "Waiting for ${prog} to establish connection to ${dst_socket}..." + + wait_until 5 get_src_socket "$@" +} + +####################################### + +# filename will be in $tcpdump_filename, pid in $tcpdump_pid +tcpdump_start () +{ + tcpdump_filter="$1" # global + + echo "Running tcpdump..." + tcpdump_filename=$(mktemp) + ctdb_test_exit_hook_add "rm -f $tcpdump_filename" + + # The only way of being sure that tcpdump is listening is to send + # some packets that it will see. So we use dummy pings - the -U + # option to tcpdump ensures that packets are flushed to the file + # as they are captured. + local dummy_addr="127.3.2.1" + local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" + tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + echo "Waiting for tcpdump output file to be ready..." + ping -q "$dummy_addr" >/dev/null 2>&1 & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + tcpdump_listen_for_dummy () + { + tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 + } + + wait_until 10 tcpdump_listen_for_dummy +} + +# By default, wait for 1 matching packet. +tcpdump_wait () +{ + local count="${1:-1}" + local filter="${2:-${tcpdump_filter}}" + + tcpdump_check () + { + local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) + [ $found -ge $count ] + } + + echo "Waiting for tcpdump to capture some packets..." + if ! wait_until 30 tcpdump_check ; then + echo "DEBUG AT $(date '+%F %T'):" + local i + for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + echo "$i" + $i || true + done + return 1 + fi +} + +tcpdump_show () +{ + local filter="${1:-${tcpdump_filter}}" + + tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null +} + +tcptickle_sniff_start () +{ + local src="$1" + local dst="$2" + + local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" + local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" + local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 + local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" + tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" + local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" + + tcpdump_start "$filter" +} + +tcptickle_sniff_wait_show () +{ + tcpdump_wait 1 "$tcptickle_reset" + + echo "GOOD: here are some TCP tickle packets:" + tcpdump_show +} + +gratarp_sniff_start () +{ + tcpdump_start "arp host ${test_ip}" +} + +gratarp_sniff_wait_show () +{ + tcpdump_wait 2 + + echo "GOOD: this should be the some gratuitous ARPs:" + tcpdump_show +} + + +####################################### + +daemons_stop () +{ + echo "Attempting to politely shutdown daemons..." + onnode 1 $CTDB shutdown -n all || true + + echo "Sleeping for a while..." + sleep_for 1 + + if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then + echo "Killing remaining daemons..." + pkill -f $CTDB_DIR/bin/ctdbd + + if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then + echo "Once more with feeling.." + pkill -9 $CTDB_DIR/bin/ctdbd + fi + fi + + local var_dir=$CTDB_DIR/tests/var + rm -rf $var_dir/test.db +} + +daemons_setup () +{ + local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes + + local var_dir=$CTDB_DIR/tests/var + + mkdir -p $var_dir/test.db/persistent + + local public_addresses=$var_dir/public_addresses.txt + local no_public_addresses=$var_dir/no_public_addresses.txt + rm -f $CTDB_NODES $public_addresses $no_public_addresses + + # If there are (strictly) greater than 2 nodes then we'll randomly + # choose a node to have no public addresses. + local no_public_ips=-1 + [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes)) + echo "$no_public_ips" >$no_public_addresses + + local i + for i in $(seq 1 $num_nodes) ; do + if [ "${CTDB_USE_IPV6}x" != "x" ]; then + echo ::$i >> $nodes + ip addr add ::$i/128 dev lo + else + echo 127.0.0.$i >> $CTDB_NODES + # 2 public addresses on most nodes, just to make things interesting. + if [ $(($i - 1)) -ne $no_public_ips ] ; then + echo "192.0.2.$i/24 lo" >> $public_addresses + echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses + fi + fi + done +} + +daemons_start_1 () +{ + local pnn="$1" + shift # "$@" gets passed to ctdbd + + local var_dir=$CTDB_DIR/tests/var + + local public_addresses=$var_dir/public_addresses.txt + local no_public_addresses=$var_dir/no_public_addresses.txt + + local no_public_ips=-1 + [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses + + if [ "$no_public_ips" = $pnn ] ; then + echo "Node $no_public_ips will have no public IPs." + fi + + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" + + if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then + ctdb_options="$ctdb_options --public-interface=lo" + fi + + if [ $pnn -eq $no_public_ips ] ; then + ctdb_options="$ctdb_options --public-addresses=/dev/null" + else + ctdb_options="$ctdb_options --public-addresses=$public_addresses" + fi + + # Need full path so we can use "pkill -f" to kill the daemons. + $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$pnn $ctdb_options "$@" ||return 1 +} + +daemons_start () +{ + # "$@" gets passed to ctdbd + + local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes + + echo "Starting $num_nodes ctdb daemons..." + + for i in $(seq 0 $(($num_nodes - 1))) ; do + daemons_start_1 $i "$@" + done + + local var_dir=$CTDB_DIR/tests/var + + if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then + ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 + fi +} + +####################################### + +_ctdb_hack_options () +{ + local ctdb_options="$*" + + # We really just want to pass CTDB_OPTIONS but on RH + # /etc/sysconfig/ctdb can, and frequently does, set that variable. + # So instead, we hack badly. We'll add these as we use them. + # Note that these may still be overridden by the above file... but + # we tend to use the exotic options here... so that is unlikely. + + case "$ctdb_options" in + *--start-as-stopped*) + export CTDB_START_AS_STOPPED="yes" + esac +} + +_restart_ctdb () +{ + _ctdb_hack_options "$@" + + if [ -e /etc/redhat-release ] ; then + service ctdb restart + else + /etc/init.d/ctdb restart + fi +} + +_ctdb_start () +{ + _ctdb_hack_options "$@" + + /etc/init.d/ctdb start +} + +setup_ctdb () +{ + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_setup + fi +} + +# Common things to do after starting one or more nodes. +_ctdb_start_post () +{ + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this blocks + # until the recovery is complete. Hopefully this will help the + # cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + + echo "ctdb is ready" +} + +# This assumes that ctdbd is not running on the given node. +ctdb_start_1 () +{ + local pnn="$1" + shift # "$@" is passed to ctdbd start. + + echo -n "Starting CTDB on node ${pnn}..." + + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_start_1 $pnn "$@" + else + onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@" + fi + + # If we're starting only 1 node then we're doing something weird. + ctdb_restart_when_done +} + +restart_ctdb () +{ + # "$@" is passed to ctdbd start. + + echo -n "Restarting CTDB" + if $ctdb_test_restart_scheduled ; then + echo -n " (scheduled)" + fi + echo "..." + + local i + for i in $(seq 1 5) ; do + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_stop + daemons_start "$@" + else + onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" + fi || { + echo "Restart failed. Trying again in a few seconds..." + sleep_for 5 + continue + } + + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { + echo "Cluster didn't become healthy. Restarting..." + continue + } + + local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this + # blocks until the recovery is complete. Hopefully this will + # help the cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + + # Cluster is still healthy. Good, we're done! + if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster become UNHEALTHY again. Restarting..." + continue + fi + + echo "Doing a sync..." + onnode -q 0 $CTDB sync + + echo "ctdb is ready" + return 0 + done + + echo "Cluster UNHEALTHY... too many attempts..." + echo "$debug_out" + # Try to make the calling test fail + status=1 + return 1 +} + +ctdb_restart_when_done () +{ + ctdb_test_restart_scheduled=true +} + +####################################### + +install_eventscript () +{ + local script_name="$1" + local script_contents="$2" + + if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + # The quoting here is *very* fragile. However, we do + # experience the joy of installing a short script using + # onnode, and without needing to know the IP addresses of the + # nodes. + onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" + else + f="${CTDB_DIR}/tests/events.d/${script_name}" + echo "$script_contents" >"$f" + chmod 755 "$f" + fi +} + +uninstall_eventscript () +{ + local script_name="$1" + + if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" + else + rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" + fi +} + +####################################### + +# This section deals with the 99.ctdb_test eventscript. + +# Metafunctions: Handle a ctdb-test file on a node. +# given event. +ctdb_test_eventscript_file_create () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_remove () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_exists () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1 +} + + +# Handle a flag file on a node that is removed by 99.ctdb_test on the +# given event. +ctdb_test_eventscript_flag () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}" +} + + +# Handle a trigger that causes 99.ctdb_test to fail it's monitor +# event. +ctdb_test_eventscript_unhealthy_trigger () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger" +} + +# Handle the file that 99.ctdb_test created to show that it has marked +# a node unhealthy because it detected the above trigger. +ctdb_test_eventscript_unhealthy_detected () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" +} + +# Handle a trigger that causes 99.ctdb_test to timeout it's monitor +# event. This should cause the node to be banned. +ctdb_test_eventscript_timeout_trigger () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout" +} + +# Note that the eventscript can't use the above functions! +ctdb_test_eventscript_install () +{ + + local script='#!/bin/sh +out=$(ctdb pnn) +pnn="${out#PNN:}" + +rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" + +trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" +detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" +timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}" +case "$1" in + monitor) + if [ -e "$trigger" ] ; then + echo "${0}: Unhealthy because \"$trigger\" detected" + touch "$detected" + exit 1 + elif [ -e "$detected" -a ! -e "$trigger" ] ; then + echo "${0}: Healthy again, \"$trigger\" no longer detected" + rm "$detected" + fi + + ;; + *) + if [ -e "$timeout_trigger" ] ; then + echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected" + sleep 9999 + fi + ;; + *) + +esac + +exit 0 +' + install_eventscript "99.ctdb_test" "$script" +} + +ctdb_test_eventscript_uninstall () +{ + uninstall_eventscript "99.ctdb_test" +} + +# Note that this only works if you know all other monitor events will +# succeed. You also need to install the eventscript before using it. +wait_for_monitor_event () +{ + local pnn="$1" + + echo "Waiting for a monitor event on node ${pnn}..." + ctdb_test_eventscript_flag create $pnn "monitor" + + wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" + +} + +# Make sure that $CTDB is set. +: ${CTDB:=ctdb} diff --git a/ctdb/tests/scripts/test_wrap b/ctdb/tests/scripts/test_wrap index 6c730ef768..df12e4b731 100755 --- a/ctdb/tests/scripts/test_wrap +++ b/ctdb/tests/scripts/test_wrap @@ -1,7 +1,7 @@ #!/bin/bash # Execute the given command. The intention is that it is a function -# from ctdb_test_functions.bash. +# from "${TEST_SCRIPTS_DIR}/integration.bash". PATH="$(dirname $0):${PATH}" @@ -11,6 +11,6 @@ if [ ! $(which $f >/dev/null 2>&1) ] ; then [ -x "$d/$f" ] && PATH="$d:$PATH" fi -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" "$@" -- cgit From c2d1f8752c8b8c41238cf41ee94785199cea77ef Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 18 Apr 2012 15:04:50 +1000 Subject: tests: Rationalise integration test infrastructure * run_tests no longer includes common.sh, which is only to be included by test cases. Therefore, it defines its own die() function. * TEST_SUBDIR is now set in common.sh * Move complex-only functions to complex/scripts/local.bash Signed-off-by: Martin Schwenke (This used to be ctdb commit bfa1d6638d3e116640eb4e3bb71b21ba6ef8cae5) --- ctdb/tests/scripts/common.sh | 2 + ctdb/tests/scripts/integration.bash | 140 ++---------------------------------- ctdb/tests/scripts/run_tests | 10 +-- 3 files changed, 15 insertions(+), 137 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh index 581663c438..725b2a89fc 100644 --- a/ctdb/tests/scripts/common.sh +++ b/ctdb/tests/scripts/common.sh @@ -2,6 +2,8 @@ # Common variables and functions for all CTDB tests. +export TEST_SUBDIR=$(dirname $0) + # Print a message and exit. die () { diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 70c5b5d10b..ea8aeb8b5a 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -1,10 +1,6 @@ # Hey Emacs, this is a -*- shell-script -*- !!! :-) -fail () -{ - echo "$*" - exit 1 -} +. "${TEST_SCRIPTS_DIR}/common.sh" ###################################################################### @@ -120,7 +116,7 @@ EOF ctdb_test_version () { - [ -n "$CTDB_DIR" ] || fail "Can not determine version." + [ -n "$CTDB_DIR" ] || die "Can not determine version." (cd "$CTDB_DIR" && git describe) } @@ -501,133 +497,6 @@ wait_until_node_has_some_ips () wait_until 60 node_has_some_ips "$@" } -get_src_socket () -{ - local proto="$1" - local dst_socket="$2" - local pid="$3" - local prog="$4" - - local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" - out=$(netstat -tanp | - egrep "$pat" | - awk '{ print $4 }') - - [ -n "$out" ] -} - -wait_until_get_src_socket () -{ - local proto="$1" - local dst_socket="$2" - local pid="$3" - local prog="$4" - - echo "Waiting for ${prog} to establish connection to ${dst_socket}..." - - wait_until 5 get_src_socket "$@" -} - -####################################### - -# filename will be in $tcpdump_filename, pid in $tcpdump_pid -tcpdump_start () -{ - tcpdump_filter="$1" # global - - echo "Running tcpdump..." - tcpdump_filename=$(mktemp) - ctdb_test_exit_hook_add "rm -f $tcpdump_filename" - - # The only way of being sure that tcpdump is listening is to send - # some packets that it will see. So we use dummy pings - the -U - # option to tcpdump ensures that packets are flushed to the file - # as they are captured. - local dummy_addr="127.3.2.1" - local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" - tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & - ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" - - echo "Waiting for tcpdump output file to be ready..." - ping -q "$dummy_addr" >/dev/null 2>&1 & - ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" - - tcpdump_listen_for_dummy () - { - tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 - } - - wait_until 10 tcpdump_listen_for_dummy -} - -# By default, wait for 1 matching packet. -tcpdump_wait () -{ - local count="${1:-1}" - local filter="${2:-${tcpdump_filter}}" - - tcpdump_check () - { - local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) - [ $found -ge $count ] - } - - echo "Waiting for tcpdump to capture some packets..." - if ! wait_until 30 tcpdump_check ; then - echo "DEBUG AT $(date '+%F %T'):" - local i - for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do - echo "$i" - $i || true - done - return 1 - fi -} - -tcpdump_show () -{ - local filter="${1:-${tcpdump_filter}}" - - tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null -} - -tcptickle_sniff_start () -{ - local src="$1" - local dst="$2" - - local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" - local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" - local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 - local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" - tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" - local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" - - tcpdump_start "$filter" -} - -tcptickle_sniff_wait_show () -{ - tcpdump_wait 1 "$tcptickle_reset" - - echo "GOOD: here are some TCP tickle packets:" - tcpdump_show -} - -gratarp_sniff_start () -{ - tcpdump_start "arp host ${test_ip}" -} - -gratarp_sniff_wait_show () -{ - tcpdump_wait 2 - - echo "GOOD: this should be the some gratuitous ARPs:" - tcpdump_show -} - - ####################################### daemons_stop () @@ -1051,3 +920,8 @@ wait_for_monitor_event () # Make sure that $CTDB is set. : ${CTDB:=ctdb} + +local="${TEST_SUBDIR}/scripts/local.bash" +if [ -r "$local" ] ; then + . "$local" +fi diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 3a5ec96b74..a65ba878b1 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -5,8 +5,6 @@ # the arguments that it sees. . $(dirname $0)/ctdb_test_env : -. "${TEST_SCRIPTS_DIR}/common.sh" - usage() { cat <&2 ; exit ${2:-1} +} + ###################################################################### with_summary=false @@ -141,8 +145,6 @@ run_one_test () [ -x "$_f" ] || die "test \"$_f\" is not executable" tests_total=$(($tests_total + 1)) - export TEST_SUBDIR=$(dirname "$_f") - ctdb_test_run "$_f" | tee "$tf" | show_progress status=$? if $with_summary ; then -- cgit From f30d4c575edc9eddddb9445b5d4235d9abcad73a Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 19 Apr 2012 11:14:28 +1000 Subject: tests: Local daemons are no longer the default, now require run_tests -l Testing with local daemons is the current default but this is not the most common use case. Therefore, we make local daemons optional by using the -l switch with run_tests or by setting TEST_LOCAL_DAEMONS to the number of daemons to be used (-l sets this to 3). TEST_LOCAL_DAEMONS replaces CTDB_TEST_NUM_DAEMONS and CTDB_TEST_REAL_CLUSTER is removed. Most relevant logic is moved from ctdb_test_env to integration.bash. ctdb_test_check_real_cluster() is moved from integration.bash to complex/scripts/local.bash. Signed-off-by: Martin Schwenke (This used to be ctdb commit 72ecae61c43b318ec94b527a12cbb0a382e8c3db) --- ctdb/tests/scripts/common.sh | 2 ++ ctdb/tests/scripts/ctdb_test_env | 15 ------------- ctdb/tests/scripts/integration.bash | 43 ++++++++++++++++++++----------------- ctdb/tests/scripts/run_tests | 5 ++++- 4 files changed, 29 insertions(+), 36 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh index 725b2a89fc..481ef298c6 100644 --- a/ctdb/tests/scripts/common.sh +++ b/ctdb/tests/scripts/common.sh @@ -4,6 +4,8 @@ export TEST_SUBDIR=$(dirname $0) +CTDB_DIR=$(dirname $(dirname "$TEST_SUBDIR")) + # Print a message and exit. die () { diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index 18d100f3d8..dce419fc06 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -21,21 +21,6 @@ else fi export CTDB_TEST_WRAPPER -# If we're not running on a real cluster then we need a local copy of -# ctdb (and other stuff) in $PATH and we will use local daemons. -if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then - export CTDB_TEST_NUM_DAEMONS=3 - - export CTDB_NODES_SOCKETS="" - for i in $(seq 0 $(($CTDB_TEST_NUM_DAEMONS -1))) ; do - CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" - done - - PATH="${CTDB_DIR}/bin:${CTDB_DIR}/tests/bin:${PATH}" - - export CTDB_NODES="$var_dir/nodes.txt" -fi - # If $VALGRIND is set then use it whenever ctdb is called, but only if # $CTDB is not already set. [ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb" diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index ea8aeb8b5a..ae32e0b0f5 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -2,6 +2,21 @@ . "${TEST_SCRIPTS_DIR}/common.sh" +# If we're not running on a real cluster then we need a local copy of +# ctdb (and other stuff) in $PATH and we will use local daemons. +if [ -n "$TEST_LOCAL_DAEMONS" ] ; then + var_dir="${CTDB_DIR}/tests/var" + + export CTDB_NODES_SOCKETS="" + for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do + CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" + done + + PATH="${CTDB_DIR}/bin:${PATH}" + + export CTDB_NODES="$var_dir/nodes.txt" +fi + ###################################################################### ctdb_check_time_logs () @@ -67,7 +82,7 @@ ctdb_test_exit () echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." - if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ + if [ -z "$TEST_LOCAL_DAEMONS" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ $status -ne 0 ] ; then ctdb_check_time_logs fi @@ -152,14 +167,6 @@ ctdb_test_init () trap "ctdb_test_exit" 0 } -ctdb_test_check_real_cluster () -{ - [ -n "$CTDB_TEST_REAL_CLUSTER" ] && return 0 - - echo "ERROR: This test must be run on a real/virtual cluster, not local daemons." - return 1 -} - ######################################## # Sets: $out @@ -523,8 +530,6 @@ daemons_stop () daemons_setup () { - local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes - local var_dir=$CTDB_DIR/tests/var mkdir -p $var_dir/test.db/persistent @@ -536,11 +541,11 @@ daemons_setup () # If there are (strictly) greater than 2 nodes then we'll randomly # choose a node to have no public addresses. local no_public_ips=-1 - [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes)) + [ $TEST_LOCAL_DAEMONS -gt 2 ] && no_public_ips=$(($RANDOM % $TEST_LOCAL_DAEMONS)) echo "$no_public_ips" >$no_public_addresses local i - for i in $(seq 1 $num_nodes) ; do + for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do if [ "${CTDB_USE_IPV6}x" != "x" ]; then echo ::$i >> $nodes ip addr add ::$i/128 dev lo @@ -549,7 +554,7 @@ daemons_setup () # 2 public addresses on most nodes, just to make things interesting. if [ $(($i - 1)) -ne $no_public_ips ] ; then echo "192.0.2.$i/24 lo" >> $public_addresses - echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses + echo "192.0.2.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >> $public_addresses fi fi done @@ -592,11 +597,9 @@ daemons_start () { # "$@" gets passed to ctdbd - local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes - - echo "Starting $num_nodes ctdb daemons..." + echo "Starting $TEST_LOCAL_DAEMONS ctdb daemons..." - for i in $(seq 0 $(($num_nodes - 1))) ; do + for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do daemons_start_1 $i "$@" done @@ -762,7 +765,7 @@ install_eventscript () local script_name="$1" local script_contents="$2" - if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + if [ -z "$TEST_LOCAL_DAEMONS" ] ; then # The quoting here is *very* fragile. However, we do # experience the joy of installing a short script using # onnode, and without needing to know the IP addresses of the @@ -779,7 +782,7 @@ uninstall_eventscript () { local script_name="$1" - if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + if [ -z "$TEST_LOCAL_DAEMONS" ] ; then onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" else rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index a65ba878b1..5efe6e4147 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -11,6 +11,7 @@ Usage: run_tests [OPTIONS] [TESTS] Options: -s Print a summary of tests results after running all tests + -l Use local daemons for integration tests -v Verbose - print test output for non-failures (only some tests) -A Use "cat -A" to print test output (only some tests) -D Show diff between failed/expected test output (some tests only) @@ -38,8 +39,9 @@ export TEST_VERBOSE=false export TEST_COMMAND_TRACE="" export TEST_CAT_RESULTS_OPTS="" export TEST_DIFF_RESULTS=false +export TEST_LOCAL_DAEMONS # No default, developer can "override"! -temp=$(getopt -n "$prog" -o "xdhqsvXAD" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdhlqsvXAD" -l help -- "$@") [ $? != 0 ] && usage @@ -49,6 +51,7 @@ while true ; do case "$1" in -x) set -x; shift ;; -d) with_desc=true ; shift ;; # 4th line of output is description + -l) TEST_LOCAL_DAEMONS="3" ; shift ;; -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; -v) TEST_VERBOSE=true ; shift ;; -- cgit From 769407feabd4e6414da303b78a7b680f615a882b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 19 Apr 2012 11:23:07 +1000 Subject: tests: Move relative directory path hack from run_tests to common.sh Signed-off-by: Martin Schwenke (This used to be ctdb commit 66a7fece867966528689d2784a284e32d687a0f5) --- ctdb/tests/scripts/common.sh | 6 +++++- ctdb/tests/scripts/run_tests | 4 ---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh index 481ef298c6..2d7b2e292e 100644 --- a/ctdb/tests/scripts/common.sh +++ b/ctdb/tests/scripts/common.sh @@ -2,7 +2,11 @@ # Common variables and functions for all CTDB tests. -export TEST_SUBDIR=$(dirname $0) +# This expands the most probable problem cases like "." and "..". +TEST_SUBDIR=$(dirname "$0") +if [ $(dirname "$TEST_SUBDIR") = "." ] ; then + TEST_SUBDIR=$(cd "$TEST_SUBDIR" ; pwd) +fi CTDB_DIR=$(dirname $(dirname "$TEST_SUBDIR")) diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 5efe6e4147..2fc6979961 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -168,10 +168,6 @@ run_one_test () for f ; do if [ -d "$f" ] ; then - # This expands the most probable problem cases like "." and "..". - if [ $(dirname "$f") = "." ] ; then - f=$(cd "$f" ; pwd) - fi for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do run_one_test "$i" done -- cgit From 43f1209dc4484f456ccd47254955c0eb4d484383 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 19 Apr 2012 11:27:05 +1000 Subject: tests: Programs run by tests should be found in $PATH Signed-off-by: Martin Schwenke (This used to be ctdb commit b5308142d03332d6d4e0c3b77283c772462fbb23) --- ctdb/tests/scripts/common.sh | 5 +++++ ctdb/tests/scripts/ctdb_test_env | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh index 2d7b2e292e..4a51651f4c 100644 --- a/ctdb/tests/scripts/common.sh +++ b/ctdb/tests/scripts/common.sh @@ -10,6 +10,11 @@ fi CTDB_DIR=$(dirname $(dirname "$TEST_SUBDIR")) +_tests_dir=$(dirname "$TEST_SUBDIR") +[ -n "$TEST_BIN_DIR" ] || TEST_BIN_DIR="${_tests_dir}/bin" +[ -n "$CTDB_TOOLS_DIR" ] || CTDB_TOOLS_DIR="${CTDB_DIR}/tools" +PATH="${TEST_BIN_DIR}:${CTDB_TOOLS_DIR}:${PATH}" + # Print a message and exit. die () { diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index dce419fc06..2614d36207 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -6,9 +6,7 @@ var_dir=$CTDB_DIR/tests/var ###################################################################### -ctdb_tools_dir=$CTDB_DIR/tools - -PATH="${TEST_SCRIPTS_DIR}:${ctdb_tools_dir}:${PATH}" +PATH="${TEST_SCRIPTS_DIR}:${PATH}" export CTDB_TIMEOUT=60 -- cgit From 21b219075173e4e365d2fce271c1f4ea315455bf Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 19 Apr 2012 11:31:02 +1000 Subject: tests: Remove ctdb_test_env Move the contents to integration.bash and run_tests as appropriate. Signed-off-by: Martin Schwenke (This used to be ctdb commit 6136ab02db261b26a2a58b526c913e37e8146841) --- ctdb/tests/scripts/ctdb_test_env | 28 ---------------------------- ctdb/tests/scripts/integration.bash | 18 ++++++++++++++++++ ctdb/tests/scripts/run_tests | 7 ++----- 3 files changed, 20 insertions(+), 33 deletions(-) delete mode 100755 ctdb/tests/scripts/ctdb_test_env (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env deleted file mode 100755 index 2614d36207..0000000000 --- a/ctdb/tests/scripts/ctdb_test_env +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -export TEST_SCRIPTS_DIR=$(cd $(dirname $0) ; pwd) -export CTDB_DIR=$(dirname $(dirname $TEST_SCRIPTS_DIR)) -var_dir=$CTDB_DIR/tests/var - -###################################################################### - -PATH="${TEST_SCRIPTS_DIR}:${PATH}" - -export CTDB_TIMEOUT=60 - -###################################################################### - -if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then - CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" -else - CTDB_TEST_WRAPPER="${TEST_SCRIPTS_DIR}/test_wrap" -fi -export CTDB_TEST_WRAPPER - -# If $VALGRIND is set then use it whenever ctdb is called, but only if -# $CTDB is not already set. -[ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb" - -###################################################################### - -"$@" diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index ae32e0b0f5..ea4ef5cf7e 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -19,6 +19,24 @@ fi ###################################################################### +export CTDB_TIMEOUT=60 + +if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then + CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" +else + CTDB_TEST_WRAPPER="${TEST_SCRIPTS_DIR}/test_wrap" +fi +export CTDB_TEST_WRAPPER + +# If $VALGRIND is set then use it whenever ctdb is called, but only if +# $CTDB is not already set. +[ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb" + +# why??? +PATH="${TEST_SCRIPTS_DIR}:${PATH}" + +###################################################################### + ctdb_check_time_logs () { local threshold=20 diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 2fc6979961..f901ef2a7a 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -1,10 +1,5 @@ #!/bin/bash -# The ability of ctdb_test_env to take tests on the command-line is -# nice, but here we need to hack around it with that colon to reset -# the arguments that it sees. -. $(dirname $0)/ctdb_test_env : - usage() { cat < Date: Thu, 19 Apr 2012 11:50:32 +1000 Subject: tests: Add -e option to cause run_tests to exit on first test failure Signed-off-by: Martin Schwenke (This used to be ctdb commit f45295a3005474957852d0e7a5c3807e30ab519d) --- ctdb/tests/scripts/run_tests | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index f901ef2a7a..f45afc35a4 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -7,6 +7,7 @@ Usage: run_tests [OPTIONS] [TESTS] Options: -s Print a summary of tests results after running all tests -l Use local daemons for integration tests + -e Exit on the first test failure -v Verbose - print test output for non-failures (only some tests) -A Use "cat -A" to print test output (only some tests) -D Show diff between failed/expected test output (some tests only) @@ -29,6 +30,7 @@ die () with_summary=false with_desc=false quiet=false +exit_on_fail=false export TEST_VERBOSE=false export TEST_COMMAND_TRACE="" @@ -36,7 +38,7 @@ export TEST_CAT_RESULTS_OPTS="" export TEST_DIFF_RESULTS=false export TEST_LOCAL_DAEMONS # No default, developer can "override"! -temp=$(getopt -n "$prog" -o "xdhlqsvXAD" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdehlqsvXAD" -l help -- "$@") [ $? != 0 ] && usage @@ -46,6 +48,7 @@ while true ; do case "$1" in -x) set -x; shift ;; -d) with_desc=true ; shift ;; # 4th line of output is description + -e) exit_on_fail=true ; shift ;; -l) TEST_LOCAL_DAEMONS="3" ; shift ;; -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; @@ -167,9 +170,15 @@ for f ; do if [ -d "$f" ] ; then for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do run_one_test "$i" + if $exit_on_fail && [ $status -ne 0 ] ; then + break + fi done elif [ -f "$f" ] ; then run_one_test "$f" + if $exit_on_fail && [ $status -ne 0 ] ; then + break + fi else die "test \"$f\" is not recognised" fi -- cgit From 4d08afa0beabf2af378880a422d8e7b94780c8e9 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 19 Apr 2012 12:10:25 +1000 Subject: tests: Add new -H option for run_tests to avoid printing header/footer This is useful for using run_tests to run one test at a time within an alternative test framework. Signed-off-by: Martin Schwenke (This used to be ctdb commit dc71294a33e88baa4e85fa1fa66cab58a83c2607) --- ctdb/tests/scripts/run_tests | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index f45afc35a4..2ad215dad9 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -13,6 +13,7 @@ Options: -D Show diff between failed/expected test output (some tests only) -X Trace certain scripts run by tests using -x (only some tests) -d Print descriptions of tests instead of filenames (dodgy!) + -H No headers - for running single test with other wrapper -q Quiet - don't show tests being run (hint: use with -s) -x Trace this script with the -x option EOF @@ -31,6 +32,7 @@ with_summary=false with_desc=false quiet=false exit_on_fail=false +no_header=false export TEST_VERBOSE=false export TEST_COMMAND_TRACE="" @@ -38,7 +40,7 @@ export TEST_CAT_RESULTS_OPTS="" export TEST_DIFF_RESULTS=false export TEST_LOCAL_DAEMONS # No default, developer can "override"! -temp=$(getopt -n "$prog" -o "xdehlqsvXAD" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdehlqsvXADH" -l help -- "$@") [ $? != 0 ] && usage @@ -56,6 +58,7 @@ while true ; do -X) TEST_COMMAND_TRACE="sh -x" ; shift ;; -A) TEST_CAT_RESULTS_OPTS="-A" ; shift ;; -D) TEST_DIFF_RESULTS=true ; shift ;; + -H) no_header=true ; shift ;; --) shift ; break ;; *) usage ;; esac @@ -114,12 +117,12 @@ ctdb_test_run () [ -n "$1" ] || set -- "$name" - ctdb_test_begin "$name" + $no_header || ctdb_test_begin "$name" local status=0 "$@" || status=$? - ctdb_test_end "$name" "$status" "$*" + $no_header || ctdb_test_end "$name" "$status" "$*" return $status } -- cgit From 72ab31b024072cf73a587e9c0b4501c6f040de14 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 19 Apr 2012 12:19:02 +1000 Subject: tests: Fix trailing whitespace issues in integration.bash git complains... Signed-off-by: Martin Schwenke (This used to be ctdb commit d714cf7924674a7a0eb6d585eb74a6a4df26fc12) --- ctdb/tests/scripts/integration.bash | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index ea4ef5cf7e..d38d08d81d 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -131,11 +131,11 @@ ctdb_test_exit_hook_add () ctdb_test_usage() { local status=${1:-2} - + cat < Date: Thu, 19 Apr 2012 15:33:46 +1000 Subject: tests: CTDB_TEST_WRAPPER has to be an absolute path on a real cluster Signed-off-by: Amitay Isaacs (This used to be ctdb commit a0a7759d47ef5de4a8214273e39c50fb1f6e2e0c) --- ctdb/tests/scripts/integration.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index d38d08d81d..7355b46db8 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -24,7 +24,8 @@ export CTDB_TIMEOUT=60 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" else - CTDB_TEST_WRAPPER="${TEST_SCRIPTS_DIR}/test_wrap" + _d=$(cd ${TEST_SCRIPTS_DIR}; echo $PWD) + CTDB_TEST_WRAPPER="$_d/test_wrap" fi export CTDB_TEST_WRAPPER -- cgit From 2f2e341370640713ff9cb707b6a84ae78109e84e Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 19 Apr 2012 15:40:53 +1000 Subject: tests: test_wrap needs to set TEST_SCRIPTS_DIR Signed-off-by: Amitay Isaacs (This used to be ctdb commit 79e979d67e3a2a5a13ef8fd2ef8f56331ec51558) --- ctdb/tests/scripts/test_wrap | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/test_wrap b/ctdb/tests/scripts/test_wrap index df12e4b731..d5085fda81 100755 --- a/ctdb/tests/scripts/test_wrap +++ b/ctdb/tests/scripts/test_wrap @@ -5,6 +5,8 @@ PATH="$(dirname $0):${PATH}" +TEST_SCRIPTS_DIR=$(dirname $0) + f="ctdb_bench" if [ ! $(which $f >/dev/null 2>&1) ] ; then d=$(dirname $(dirname $0))/bin -- cgit From 05062d874a1becac026cefa78149aee1587e7da8 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 20 Apr 2012 13:57:01 +1000 Subject: tests: test_wrap can use $TEST_SCRIPTS_DIR for a little extra clarity Also add a comment to explain what is being added to $PATH. Signed-off-by: Martin Schwenke (This used to be ctdb commit 704a3e3b83aff63e8f7b0650c141776ed2c1f047) --- ctdb/tests/scripts/test_wrap | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/test_wrap b/ctdb/tests/scripts/test_wrap index d5085fda81..176310e9a1 100755 --- a/ctdb/tests/scripts/test_wrap +++ b/ctdb/tests/scripts/test_wrap @@ -7,9 +7,12 @@ PATH="$(dirname $0):${PATH}" TEST_SCRIPTS_DIR=$(dirname $0) +# We need the test binaries (i.e. tests/bin/) to be in $PATH. If they +# aren't already in $PATH then we know that tests/bin/ sits alongside +# tests/scripts/. f="ctdb_bench" if [ ! $(which $f >/dev/null 2>&1) ] ; then - d=$(dirname $(dirname $0))/bin + d=$(dirname "$TEST_SCRIPTS_DIR")/bin [ -x "$d/$f" ] && PATH="$d:$PATH" fi -- cgit From c4606275c9c29a4abb257efbe815e9791de63449 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 20 Apr 2012 14:09:43 +1000 Subject: tests: run_tests should exit with failed test status if running with -H Signed-off-by: Martin Schwenke (This used to be ctdb commit 6eb7fa572e7fc212332ddd68793e3f35161baf7c) --- ctdb/tests/scripts/run_tests | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 2ad215dad9..7a86301c95 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -199,5 +199,9 @@ fi rm -f "$sf" if [ $tests_failed -gt 0 ] ; then - exit 1 + if $no_header ; then + exit $status + else + exit 1 + fi fi -- cgit From 6757ad62f4cda14066789989fb002cee7c009cde Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 20 Apr 2012 16:43:17 +1000 Subject: tests: Make run_tests -X more flexible - it now works with onnode Hardcoding "sh -x" is suboptimal because some scripts, like onnode, require bash. Signed-off-by: Martin Schwenke (This used to be ctdb commit 246809af64c03d26288abff5907ed46614e72b15) --- ctdb/tests/scripts/run_tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 7a86301c95..d4eab4d847 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -35,7 +35,7 @@ exit_on_fail=false no_header=false export TEST_VERBOSE=false -export TEST_COMMAND_TRACE="" +export TEST_COMMAND_TRACE=false export TEST_CAT_RESULTS_OPTS="" export TEST_DIFF_RESULTS=false export TEST_LOCAL_DAEMONS # No default, developer can "override"! @@ -55,7 +55,7 @@ while true ; do -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; -v) TEST_VERBOSE=true ; shift ;; - -X) TEST_COMMAND_TRACE="sh -x" ; shift ;; + -X) TEST_COMMAND_TRACE=true ; shift ;; -A) TEST_CAT_RESULTS_OPTS="-A" ; shift ;; -D) TEST_DIFF_RESULTS=true ; shift ;; -H) no_header=true ; shift ;; -- cgit From 286d5aa5b10b25c900083ec29bf5f479729343b9 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 23 Apr 2012 14:45:04 +1000 Subject: tests/integration: Use absolute path for socket symlink Signed-off-by: Martin Schwenke (This used to be ctdb commit 709cd5dfeb630096a1cd2062da666fee9ddca715) --- ctdb/tests/scripts/integration.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 7355b46db8..363fecf5e8 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -622,7 +622,7 @@ daemons_start () daemons_start_1 $i "$@" done - local var_dir=$CTDB_DIR/tests/var + local var_dir=$(cd $CTDB_DIR/tests/var; echo $PWD) if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 -- cgit From 7ed438799044c186b55f9575415faa8fb702e521 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 24 Apr 2012 10:23:17 +1000 Subject: tests/integration: Remove cabability for testcase option This is unused and can't be used with the current test infrastructure. It may have been useful with ctdb_test_env but I don't think it was ever used. Signed-off-by: Martin Schwenke (This used to be ctdb commit 4aa879466dd46cb4e8710edbbaac1276521e475b) --- ctdb/tests/scripts/integration.bash | 48 ------------------------------------- 1 file changed, 48 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 363fecf5e8..3e3cc67616 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -129,60 +129,12 @@ ctdb_test_exit_hook_add () ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" } -ctdb_test_usage() -{ - local status=${1:-2} - - cat < Date: Tue, 24 Apr 2012 10:25:50 +1000 Subject: tests: Add a -V option to set new variable TEST_VAR_DIR Part of preparation to be able to install the tests. The current var/ subdirectories will end up somewhere in /usr/local/ or /usr/ and we don't want to put temporary files there. This creates a temporary TEST_VAR_DIR by default. If -V is specified then the given directory will be used as TEST_VAR_DIR. This allows the current behaviour where individual integration tests can be run against already-running daemons 9with sockets and other stuff already created) to save time and trouble. Yes, there are plans for a clean-up option... ;-) Signed-off-by: Martin Schwenke (This used to be ctdb commit 8e4ec9a40bb7d392d7474b067a74fa121c069007) --- ctdb/tests/scripts/run_tests | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index d4eab4d847..ced884837d 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -5,17 +5,18 @@ usage() { Usage: run_tests [OPTIONS] [TESTS] Options: - -s Print a summary of tests results after running all tests - -l Use local daemons for integration tests - -e Exit on the first test failure - -v Verbose - print test output for non-failures (only some tests) - -A Use "cat -A" to print test output (only some tests) - -D Show diff between failed/expected test output (some tests only) - -X Trace certain scripts run by tests using -x (only some tests) - -d Print descriptions of tests instead of filenames (dodgy!) - -H No headers - for running single test with other wrapper - -q Quiet - don't show tests being run (hint: use with -s) - -x Trace this script with the -x option + -s Print a summary of tests results after running all tests + -l Use local daemons for integration tests + -e Exit on the first test failure + -V Use as $TEST_VAR_DIR + -v Verbose - print test output for non-failures (only some tests) + -A Use "cat -A" to print test output (only some tests) + -D Show diff between failed/expected test output (some tests only) + -X Trace certain scripts run by tests using -x (only some tests) + -d Print descriptions of tests instead of filenames (dodgy!) + -H No headers - for running single test with other wrapper + -q Quiet - don't show tests being run (hint: use with -s) + -x Trace this script with the -x option EOF exit 1 } @@ -39,8 +40,9 @@ export TEST_COMMAND_TRACE=false export TEST_CAT_RESULTS_OPTS="" export TEST_DIFF_RESULTS=false export TEST_LOCAL_DAEMONS # No default, developer can "override"! +export TEST_VAR_DIR="" -temp=$(getopt -n "$prog" -o "xdehlqsvXADH" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdehlqsvV:XADH" -l help -- "$@") [ $? != 0 ] && usage @@ -55,6 +57,7 @@ while true ; do -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; -v) TEST_VERBOSE=true ; shift ;; + -V) TEST_VAR_DIR="$2" ; shift 2 ;; -X) TEST_COMMAND_TRACE=true ; shift ;; -A) TEST_CAT_RESULTS_OPTS="-A" ; shift ;; -D) TEST_DIFF_RESULTS=true ; shift ;; @@ -169,6 +172,12 @@ run_one_test () fi } +[ -n "$TEST_VAR_DIR" ] || TEST_VAR_DIR=$(mktemp -d) +mkdir -p "$TEST_VAR_DIR" +# Must be absolute +TEST_VAR_DIR=$(cd "$TEST_VAR_DIR"; echo "$PWD") +echo "TEST_VAR_DIR=$TEST_VAR_DIR" + for f ; do if [ -d "$f" ] ; then for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do -- cgit From d28e268ed6e165bcb2817d94e8feac763ca2e267 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 24 Apr 2012 11:39:07 +1000 Subject: tests: Update integration.sh to use TEST_VAR_DIR There are 2 subtleties: * Local daemons currently use the in-tree tests/events.d/ subdirectory for their eventscripts. However, some tests install/remove test-specific eventscripts so now $TEST_VAR_DIR/events.d/ is used and the standard eventscripts are copied there at setup time. * The pgrep/pkill logic for killing local daemons currently relies on ctdbd being run with a full path to tests/bin/ctdbd. If the tests are installed and run against an installed daemon then this won't work. Therefore, ctdbd is now expected to be in $PATH and is found/killed by matching: ctdbd --socket=.* --nlist .* --nopublicipcheck This is complex but should avoid killing a real ctdbd running on a node. Signed-off-by: Martin Schwenke (This used to be ctdb commit 301491802eec0e49e108f5aae7d7be379703d72c) --- ctdb/tests/scripts/integration.bash | 57 +++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 27 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 3e3cc67616..d13b56e7ea 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -5,16 +5,14 @@ # If we're not running on a real cluster then we need a local copy of # ctdb (and other stuff) in $PATH and we will use local daemons. if [ -n "$TEST_LOCAL_DAEMONS" ] ; then - var_dir="${CTDB_DIR}/tests/var" - export CTDB_NODES_SOCKETS="" for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do - CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}" + CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${TEST_VAR_DIR}/sock.${i}" done PATH="${CTDB_DIR}/bin:${PATH}" - export CTDB_NODES="$var_dir/nodes.txt" + export CTDB_NODES="${TEST_VAR_DIR}/nodes.txt" fi ###################################################################### @@ -485,28 +483,26 @@ daemons_stop () echo "Sleeping for a while..." sleep_for 1 - if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then + local pat="ctdbd --socket=.* --nlist .* --nopublicipcheck" + if pgrep -f "$pat" >/dev/null ; then echo "Killing remaining daemons..." - pkill -f $CTDB_DIR/bin/ctdbd + pkill -f "$pat" - if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then + if pgrep -f "$pat" >/dev/null ; then echo "Once more with feeling.." - pkill -9 $CTDB_DIR/bin/ctdbd + pkill -9 -f "$pat" fi fi - local var_dir=$CTDB_DIR/tests/var - rm -rf $var_dir/test.db + rm -rf "${TEST_VAR_DIR}/test.db" } daemons_setup () { - local var_dir=$CTDB_DIR/tests/var - - mkdir -p $var_dir/test.db/persistent + mkdir -p "${TEST_VAR_DIR}/test.db/persistent" - local public_addresses=$var_dir/public_addresses.txt - local no_public_addresses=$var_dir/no_public_addresses.txt + local public_addresses="${TEST_VAR_DIR}/public_addresses.txt" + local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt" rm -f $CTDB_NODES $public_addresses $no_public_addresses # If there are (strictly) greater than 2 nodes then we'll randomly @@ -515,6 +511,16 @@ daemons_setup () [ $TEST_LOCAL_DAEMONS -gt 2 ] && no_public_ips=$(($RANDOM % $TEST_LOCAL_DAEMONS)) echo "$no_public_ips" >$no_public_addresses + # When running certain tests we add and remove eventscripts, so we + # need to be able to modify the events.d/ directory. Therefore, + # we use a temporary events.d/ directory under $TEST_VAR_DIR. We + # copy the actual test eventscript(s) in there from the original + # events.d/ directory that sits alongside $TEST_SCRIPT_DIR. + local top=$(dirname "$TEST_SCRIPTS_DIR") + local events_d="${top}/events.d" + mkdir -p "${TEST_VAR_DIR}/events.d" + cp -p "${events_d}/"* "${TEST_VAR_DIR}/events.d/" + local i for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do if [ "${CTDB_USE_IPV6}x" != "x" ]; then @@ -536,10 +542,8 @@ daemons_start_1 () local pnn="$1" shift # "$@" gets passed to ctdbd - local var_dir=$CTDB_DIR/tests/var - - local public_addresses=$var_dir/public_addresses.txt - local no_public_addresses=$var_dir/no_public_addresses.txt + local public_addresses="${TEST_VAR_DIR}/public_addresses.txt" + local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt" local no_public_ips=-1 [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses @@ -548,7 +552,7 @@ daemons_start_1 () echo "Node $no_public_ips will have no public IPs." fi - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" + local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemons.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then ctdb_options="$ctdb_options --public-interface=lo" @@ -560,8 +564,9 @@ daemons_start_1 () ctdb_options="$ctdb_options --public-addresses=$public_addresses" fi - # Need full path so we can use "pkill -f" to kill the daemons. - $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$pnn $ctdb_options "$@" ||return 1 + # We'll use "pkill -f" to kill the daemons with + # "--socket=.* --nlist .* --nopublicipcheck" as context. + $VALGRIND ctdbd --socket="${TEST_VAR_DIR}/sock.$pnn" $ctdb_options "$@" ||return 1 } daemons_start () @@ -574,10 +579,8 @@ daemons_start () daemons_start_1 $i "$@" done - local var_dir=$(cd $CTDB_DIR/tests/var; echo $PWD) - if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then - ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 + ln -sf "${TEST_VAR_DIR}/sock.0" /tmp/ctdb.socket || return 1 fi } @@ -743,7 +746,7 @@ install_eventscript () # nodes. onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" else - f="${CTDB_DIR}/tests/events.d/${script_name}" + f="${TEST_VAR_DIR}/events.d/${script_name}" echo "$script_contents" >"$f" chmod 755 "$f" fi @@ -756,7 +759,7 @@ uninstall_eventscript () if [ -z "$TEST_LOCAL_DAEMONS" ] ; then onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" else - rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" + rm -vf "${TEST_VAR_DIR}/events.d/${script_name}" fi } -- cgit From b7b6f10e9be4a610866b3aee10491e3292ba09df Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 24 Apr 2012 11:50:36 +1000 Subject: tests: Improve the logic for adding directories to $PATH Signed-off-by: Martin Schwenke (This used to be ctdb commit e6bfd3bdb3a35b2e7e7c41a6f37976772a54e3ce) --- ctdb/tests/scripts/common.sh | 29 ++++++++++++++++++++++++----- ctdb/tests/scripts/integration.bash | 6 +++++- 2 files changed, 29 insertions(+), 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh index 4a51651f4c..64a176b917 100644 --- a/ctdb/tests/scripts/common.sh +++ b/ctdb/tests/scripts/common.sh @@ -8,12 +8,31 @@ if [ $(dirname "$TEST_SUBDIR") = "." ] ; then TEST_SUBDIR=$(cd "$TEST_SUBDIR" ; pwd) fi -CTDB_DIR=$(dirname $(dirname "$TEST_SUBDIR")) +_test_dir=$(dirname "$TEST_SUBDIR") -_tests_dir=$(dirname "$TEST_SUBDIR") -[ -n "$TEST_BIN_DIR" ] || TEST_BIN_DIR="${_tests_dir}/bin" -[ -n "$CTDB_TOOLS_DIR" ] || CTDB_TOOLS_DIR="${CTDB_DIR}/tools" -PATH="${TEST_BIN_DIR}:${CTDB_TOOLS_DIR}:${PATH}" +# If we are running from within the source tree then, depending on the +# tests that we're running, we may need to add the top level bin/ and +# tools/ subdirectories to $PATH. This means we need a way of +# determining if we're running from within the source tree. There is +# no use looking outside the tests/ subdirectory because anything +# above that level may be meaningless and outside our control. +# Therefore, we'll use existence of $_test_dir/run_tests.sh to +# indicate that we're running in-tree - on a system where the tests +# have been installed, this file will be absent (renamed and placed in +# some bin/ directory). +if [ -f "${_test_dir}/run_tests.sh" ] ; then + ctdb_dir=$(dirname "$_test_dir") + + _tools_dir="${ctdb_dir}/tools" + if [ -d "$_tools_dir" ] ; then + PATH="${_tools_dir}:$PATH" + fi +fi + +_test_bin_dir="${TEST_BIN_DIR:-${_test_dir}/bin}" +if [ -d "$_test_bin_dir" ] ; then + PATH="${_test_bin_dir}:$PATH" +fi # Print a message and exit. die () diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index d13b56e7ea..776ceec855 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -10,7 +10,11 @@ if [ -n "$TEST_LOCAL_DAEMONS" ] ; then CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${TEST_VAR_DIR}/sock.${i}" done - PATH="${CTDB_DIR}/bin:${PATH}" + # Use in-tree binaries if running against local daemons. + # Otherwise CTDB need to be installed on all nodes. + if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then + PATH="${ctdb_dir}/bin:${PATH}" + fi export CTDB_NODES="${TEST_VAR_DIR}/nodes.txt" fi -- cgit From 451e66033817ff013c7940bfa671dfbd497a077e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 24 Apr 2012 14:03:41 +1000 Subject: tests: Time logging in integration tests should use TEST_VAR_DIR Signed-off-by: Martin Schwenke (This used to be ctdb commit 734cbd7def23236d418ebc6d813a748a84900101) --- ctdb/tests/scripts/integration.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 776ceec855..2dab6d77e4 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -49,7 +49,7 @@ ctdb_check_time_logs () local ds_prev="" local node="" - out=$(onnode all tail -n 20 /var/log/ctdb.test.time.log 2>&1) + out=$(onnode all tail -n 20 "${TEST_VAR_DIR}/ctdb.test.time.log" 2>&1) if [ $? -eq 0 ] ; then local line -- cgit From 70270b6815fa3741f473bdf7262494a271844ca4 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 24 Apr 2012 14:39:24 +1000 Subject: tests: New run_tests -C (cleanup) option Sets TEST_CLEANUP=true and causes $TEST_VAR_DIR to be removed... and potentially other cleanup actions in testcases. Signed-off-by: Martin Schwenke (This used to be ctdb commit 3219f221a858e499f084b8beb44610537312602b) --- ctdb/tests/scripts/run_tests | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index ced884837d..0ae4cee151 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -9,6 +9,7 @@ Options: -l Use local daemons for integration tests -e Exit on the first test failure -V Use as $TEST_VAR_DIR + -C Clean up - kill daemons and remove $TEST_VAR_DIR when done -v Verbose - print test output for non-failures (only some tests) -A Use "cat -A" to print test output (only some tests) -D Show diff between failed/expected test output (some tests only) @@ -41,8 +42,9 @@ export TEST_CAT_RESULTS_OPTS="" export TEST_DIFF_RESULTS=false export TEST_LOCAL_DAEMONS # No default, developer can "override"! export TEST_VAR_DIR="" +export TEST_CLEANUP=false -temp=$(getopt -n "$prog" -o "xdehlqsvV:XADH" -l help -- "$@") +temp=$(getopt -n "$prog" -o "xdehlqsvV:XACDH" -l help -- "$@") [ $? != 0 ] && usage @@ -60,6 +62,7 @@ while true ; do -V) TEST_VAR_DIR="$2" ; shift 2 ;; -X) TEST_COMMAND_TRACE=true ; shift ;; -A) TEST_CAT_RESULTS_OPTS="-A" ; shift ;; + -C) TEST_CLEANUP=true ; shift ;; -D) TEST_DIFF_RESULTS=true ; shift ;; -H) no_header=true ; shift ;; --) shift ; break ;; @@ -207,6 +210,14 @@ fi rm -f "$sf" +echo +if $TEST_CLEANUP ; then + echo "Removing TEST_VAR_DIR=$TEST_VAR_DIR" + rm -rf "$TEST_VAR_DIR" +else + echo "Not cleaning up TEST_VAR_DIR=$TEST_VAR_DIR" +fi + if [ $tests_failed -gt 0 ] ; then if $no_header ; then exit $status -- cgit From f510ae64cb556e80af9512b1fb0107f68a696120 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 10 May 2012 16:58:16 +1000 Subject: tests: In integration tests, use --node-ip to avoid locking weirdness Signed-off-by: Martin Schwenke (This used to be ctdb commit a00e80c701a0f9695f41c24e0360c25c0873d49d) --- ctdb/tests/scripts/integration.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 2dab6d77e4..9d5affc568 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -556,7 +556,8 @@ daemons_start_1 () echo "Node $no_public_ips will have no public IPs." fi - local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemons.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" + local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") + local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemons.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From 594601bdad81f0211e5c9f844e81262ede06c3b6 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 14 May 2012 11:57:20 +1000 Subject: tests: Fix wrapper scripts to handle options and tests without breakage If the -V option is given and no tests are supplied, the "cd" command in run_tests.sh cause scripts/run_tests to interpret the argument to -V incorrectly. Therefore, the wrapper scripts can't use "cd" because they don't know what the options are doing! Instead scripts/run_tests searches for each test relative to the current directory and, if not previously found, then searches relative to the top-level tests directory. This is a much better way of doing things. Given that run_tests.sh and run_cluster_tests.sh were starting to contain duplicate complex logic, remove run_cluster_tests.sh and replace it with a symlink to run_tests.sh. Run_tests.sh checks $0 to see what options/defaults to use. Update INSTALL to deal with this. Signed-off-by: Martin Schwenke (This used to be ctdb commit ed2db1f4e8d2b222d7f912a4a007ce48a23e83b0) --- ctdb/tests/scripts/run_tests | 60 +++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 18 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 0ae4cee151..2d3417420e 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -155,23 +155,42 @@ run_one_test () [ -x "$_f" ] || die "test \"$_f\" is not executable" tests_total=$(($tests_total + 1)) - export TEST_SCRIPTS_DIR=$(dirname "$0") - ctdb_test_run "$_f" | tee "$tf" | show_progress status=$? if $with_summary ; then if [ $status -eq 0 ] ; then tests_passed=$(($tests_passed + 1)) - t=" PASSED " + _t=" PASSED " else - t="*FAILED*" + _t="*FAILED*" tests_failed=$(($tests_failed + 1)) fi if $with_desc ; then desc=$(tail -n +4 $tf | head -n 1) _f="$desc" fi - echo "$t $_f" >>"$sf" + echo "$_t $_f" >>"$sf" + fi +} + +find_and_run_one_test () +{ + _t="$1" + _dir="$2" + + _f="${_dir}${_dir:+/}${_t}" + + if [ -d "$_f" ] ; then + for _i in $(ls "${_f%/}/"*".sh" 2>/dev/null) ; do + run_one_test "$_i" + if $exit_on_fail && [ $status -ne 0 ] ; then + break + fi + done + elif [ -f "$_f" ] ; then + run_one_test "$_f" + else + status=127 fi } @@ -181,21 +200,26 @@ mkdir -p "$TEST_VAR_DIR" TEST_VAR_DIR=$(cd "$TEST_VAR_DIR"; echo "$PWD") echo "TEST_VAR_DIR=$TEST_VAR_DIR" +export TEST_SCRIPTS_DIR=$(dirname "$0") + for f ; do - if [ -d "$f" ] ; then - for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do - run_one_test "$i" - if $exit_on_fail && [ $status -ne 0 ] ; then - break - fi - done - elif [ -f "$f" ] ; then - run_one_test "$f" - if $exit_on_fail && [ $status -ne 0 ] ; then + find_and_run_one_test "$f" + + if [ $status -eq 127 ] ; then + # Find the the top-level tests directory + tests_dir=$(dirname $(cd $TEST_SCRIPTS_DIR; echo $PWD)) + # Strip off current directory from beginning, if there, just + # to make paths more friendly. + tests_dir=${tests_dir#$PWD/} + find_and_run_one_test "$f" "$tests_dir" + fi + + if [ $status -eq 127 ] ; then + die "test \"$f\" is not recognised" + fi + + if $exit_on_fail && [ $status -ne 0 ] ; then break - fi - else - die "test \"$f\" is not recognised" fi done -- cgit From e3d357b9c3d81808d0b8b475122b6600d999fb1a Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 14 May 2012 14:59:22 +1000 Subject: tests: Use per-daemon public_addresses file for local daemons This allows a node's public addresses file to be hacked for testing. Signed-off-by: Martin Schwenke (This used to be ctdb commit c7d6e4557d00de674737e2c8d6cbebaa2461c303) --- ctdb/tests/scripts/integration.bash | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 9d5affc568..7014910b6f 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -505,9 +505,9 @@ daemons_setup () { mkdir -p "${TEST_VAR_DIR}/test.db/persistent" - local public_addresses="${TEST_VAR_DIR}/public_addresses.txt" + local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all" local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt" - rm -f $CTDB_NODES $public_addresses $no_public_addresses + rm -f $CTDB_NODES $public_addresses_all $no_public_addresses # If there are (strictly) greater than 2 nodes then we'll randomly # choose a node to have no public addresses. @@ -534,8 +534,8 @@ daemons_setup () echo 127.0.0.$i >> $CTDB_NODES # 2 public addresses on most nodes, just to make things interesting. if [ $(($i - 1)) -ne $no_public_ips ] ; then - echo "192.0.2.$i/24 lo" >> $public_addresses - echo "192.0.2.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >> $public_addresses + echo "192.0.2.$i/24 lo" >>"$public_addresses_all" + echo "192.0.2.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >>"$public_addresses_all" fi fi done @@ -546,7 +546,8 @@ daemons_start_1 () local pnn="$1" shift # "$@" gets passed to ctdbd - local public_addresses="${TEST_VAR_DIR}/public_addresses.txt" + local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all" + local public_addresses_mine="${TEST_VAR_DIR}/public_addresses.${pnn}" local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt" local no_public_ips=-1 @@ -566,7 +567,8 @@ daemons_start_1 () if [ $pnn -eq $no_public_ips ] ; then ctdb_options="$ctdb_options --public-addresses=/dev/null" else - ctdb_options="$ctdb_options --public-addresses=$public_addresses" + cp "$public_addresses_all" "$public_addresses_mine" + ctdb_options="$ctdb_options --public-addresses=$public_addresses_mine" fi # We'll use "pkill -f" to kill the daemons with -- cgit From 888048465818732b48b18488387b4fc275048c3b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 14 May 2012 15:00:32 +1000 Subject: tests: New function get_ctdbd_command_line_option() for integration testing This allows, for example, the public addresses file used by a particular daemon to be known. Signed-off-by: Martin Schwenke (This used to be ctdb commit f4b7d14f2e3c7345e7a09abb27c32923fb78cbc4) --- ctdb/tests/scripts/integration.bash | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 7014910b6f..652e4cacb8 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -739,6 +739,27 @@ ctdb_restart_when_done () ctdb_test_restart_scheduled=true } +get_ctdbd_command_line_option () +{ + local pnn="$1" + local option="$2" + + try_command_on_node "$pnn" "$CTDB getpid" || \ + die "Unable to get PID of ctdbd on node $pnn" + + local pid="${out#*:}" + try_command_on_node "$pnn" "ps -p $pid -o args hww" || \ + die "Unable to get command-line of PID $pid" + + # Strip everything up to and including --option + local t="${out#*--${option}}" + # Strip leading '=' or space if present + t="${t#=}" + t="${t# }" + # Strip any following options and print + echo "${t%% -*}" +} + ####################################### install_eventscript () -- cgit From 4563865fb6bf7efcbcf9a9940bb6d285021ddcb1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 14 May 2012 15:01:44 +1000 Subject: tests: Fix a typo in daemons_setup() Signed-off-by: Martin Schwenke (This used to be ctdb commit 863ad337fa3c4effe1fd370d3ba414027c600bd6) --- ctdb/tests/scripts/integration.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 652e4cacb8..b156ca2070 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -528,10 +528,10 @@ daemons_setup () local i for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do if [ "${CTDB_USE_IPV6}x" != "x" ]; then - echo ::$i >> $nodes + echo ::$i >>"$CTDB_NODES" ip addr add ::$i/128 dev lo else - echo 127.0.0.$i >> $CTDB_NODES + echo 127.0.0.$i >>"$CTDB_NODES" # 2 public addresses on most nodes, just to make things interesting. if [ $(($i - 1)) -ne $no_public_ips ] ; then echo "192.0.2.$i/24 lo" >>"$public_addresses_all" -- cgit From cfe030e2971faddf397b062a7d403217f54f559f Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 10 May 2012 16:59:39 +1000 Subject: tests: Use per node log files when running tests with local daemons Signed-off-by: Amitay Isaacs (This used to be ctdb commit 61df417821762d87ed01a7b5e64c35079940344d) --- ctdb/tests/scripts/integration.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index b156ca2070..70308de618 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -558,7 +558,7 @@ daemons_start_1 () fi local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") - local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemons.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" + local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From 27ce9dc366614f04f030831c81fe308dd4285c58 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Tue, 3 Jul 2012 11:50:05 +0200 Subject: run_tests: improve spacing (This used to be ctdb commit a0a0f5588445aeabe07b0e4d65087db454dc09da) --- ctdb/tests/scripts/run_tests | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 2d3417420e..0d9577461b 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -234,7 +234,8 @@ fi rm -f "$sf" -echo +echo + if $TEST_CLEANUP ; then echo "Removing TEST_VAR_DIR=$TEST_VAR_DIR" rm -rf "$TEST_VAR_DIR" -- cgit From c075c2e5c9c28e851096a16b99d53801c340140b Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 14 Jun 2012 19:36:04 +1000 Subject: tests: select_test_node_and_ips() should never select non-node -1 Instead of selecting the 1st pnn found, select the 1st one that isn't -1. Signed-off-by: Martin Schwenke (This used to be ctdb commit f02e501342112aab67aee95f253e29a670b29273) --- ctdb/tests/scripts/integration.bash | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 70308de618..e3301f283c 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -238,14 +238,13 @@ select_test_node_and_ips () { all_ips_on_node 0 - # When selecting test_node we just want a node that has public - # IPs. This will work and is economically semi-random. :-) - local x - read x test_node <<<"$out" - + test_node="" # this matches no PNN test_node_ips="" local ip pnn while read ip pnn ; do + if [ -z "$test_node" -a "$pnn" != "-1" ] ; then + test_node="$pnn" + fi if [ "$pnn" = "$test_node" ] ; then test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" fi -- cgit From 053174c07bc0fb5e5563d5023b71007390c7762f Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 20 Jun 2012 15:57:48 +1000 Subject: tests: run_tests should exit with $status with -e option Signed-off-by: Martin Schwenke (This used to be ctdb commit 619af3e857c2ced3840abfd86135cc954796da97) --- ctdb/tests/scripts/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 0d9577461b..50a578b26c 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -244,7 +244,7 @@ else fi if [ $tests_failed -gt 0 ] ; then - if $no_header ; then + if $no_header || $exit_on_fail ; then exit $status else exit 1 -- cgit From 959e37da7bbdfa47f7e4fd6e9b92dbac1f9dea58 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 2 Jul 2012 14:05:21 +1000 Subject: tests: simple tests against local daemons should check $TEST_LOCAL_DEAMONS Note the old $CTDB_TEST_REAL_CLUSTER - it doesn't exist anymore... Signed-off-by: Martin Schwenke (This used to be ctdb commit 47180dc75d15f3d61470705603565b718491c9f8) --- ctdb/tests/scripts/integration.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index e3301f283c..5229184066 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -559,7 +559,7 @@ daemons_start_1 () local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" - if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then + if [ -n "$TEST_LOCAL_DAEMONS" ] ; then ctdb_options="$ctdb_options --public-interface=lo" fi -- cgit From 523be079ea82928b3772480842a75b7533a12d9d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 2 Jul 2012 14:06:35 +1000 Subject: tests: select_test_node_and_ips() should try to avoid failing Sometimes "ctdb sync" doesn't do its job, so we end up with unassigned IPs. If $test_node isn't set then this is bad. However, try a few times to ensure it is set. Signed-off-by: Martin Schwenke (This used to be ctdb commit 2fd0157382b42aa5c5212b8e743c6f589edc6662) --- ctdb/tests/scripts/integration.bash | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 5229184066..8b268b1aa0 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -234,7 +234,7 @@ all_ips_on_node() try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" } -select_test_node_and_ips () +_select_test_node_and_ips () { all_ips_on_node 0 @@ -252,6 +252,24 @@ select_test_node_and_ips () echo "Selected node ${test_node} with IPs: ${test_node_ips}." test_ip="${test_node_ips%% *}" + + [ -n "$test_node" ] || return 1 +} + +select_test_node_and_ips () +{ + local timeout=10 + while ! _select_test_node_and_ips ; do + echo "Unable to find a test node with IPs assigned" + if [ $timeout -le 0 ] ; then + echo "BAD: Too many attempts" + return 1 + fi + sleep_for 1 + timeout=$(($timeout - 1)) + done + + return 0 } ####################################### -- cgit From 2d719e5c84e3f898a992e97992c94a2ffb94e3a0 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 3 Sep 2012 15:37:01 +1000 Subject: eventscripts: Auto-start/stop services in background If $CTDB_SERVICE_AUTOSTARTSTOP="yes" then service start/stop is done in the background with logging. Fix some unit tests for samba and winbind. Signed-off-by: Martin Schwenke (This used to be ctdb commit 3a3dae4cb5ec8b4b8381a4013adda25b87641f3a) --- ctdb/tests/scripts/unit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/unit.sh b/ctdb/tests/scripts/unit.sh index 6f92d6cb16..b27df45e76 100644 --- a/ctdb/tests/scripts/unit.sh +++ b/ctdb/tests/scripts/unit.sh @@ -49,7 +49,7 @@ result_print () if [ -n "$_extra_header" ] ; then cat < Date: Thu, 18 Oct 2012 14:15:09 +1100 Subject: tests: Local daemons should use the logging ringbuffer Signed-off-by: Martin Schwenke (This used to be ctdb commit 7547e011005f0dd5bd38e67572280126cf16e229) --- ctdb/tests/scripts/integration.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 8b268b1aa0..38420bacee 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -575,7 +575,7 @@ daemons_start_1 () fi local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") - local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" + local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" if [ -n "$TEST_LOCAL_DAEMONS" ] ; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From 5cf98c80ff7bc3814e5a052bd21ee51172dcb02c Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 3 Dec 2012 15:32:49 +1100 Subject: tests: Local daemons should use --listen instead of --node-ip Signed-off-by: Martin Schwenke Pair-programmed-with: Amitay Isaacs (This used to be ctdb commit 3221fce9ee2f6fdd3bb17a5e1629ad52a32f90d6) --- ctdb/tests/scripts/integration.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 38420bacee..8813499856 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -575,7 +575,7 @@ daemons_start_1 () fi local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") - local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --node-ip=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" + local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --listen=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" if [ -n "$TEST_LOCAL_DAEMONS" ] ; then ctdb_options="$ctdb_options --public-interface=lo" -- cgit From f6be6f387a697406c976b46c7615b25fd74c98d1 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 30 Nov 2012 16:37:28 +1100 Subject: tests: new function ip2ipmask() for integration testing Signed-off-by: Martin Schwenke (This used to be ctdb commit 8164d9b29bf9080ccc76b1305fb6c07f1ed61d55) --- ctdb/tests/scripts/integration.bash | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 8813499856..07e764ea20 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -494,6 +494,13 @@ wait_until_node_has_some_ips () wait_until 60 node_has_some_ips "$@" } +ip2ipmask () +{ + _ip="$1" + + ip addr show to "$_ip" | awk '$1 == "inet" { print $2 }' +} + ####################################### daemons_stop () -- cgit From ef7329a415dc4127950e0bf428ed4b84117a04b7 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 12 Oct 2012 16:12:38 +1100 Subject: tests/simple: Add test to check recovery daemon IP verification Also update ips_are_on_nodeglob() to handle negation. Signed-off-by: Martin Schwenke (This used to be ctdb commit 13a5944f8a27d43006acfffba76958693cae7702) --- ctdb/tests/scripts/integration.bash | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 07e764ea20..2e5fb37d7a 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -438,8 +438,14 @@ wait_until_node_has_status () # Useful for superficially testing IP failover. # IPs must be on nodes matching nodeglob. +# If the first argument is '!' then the IPs must not be on nodes +# matching nodeglob. ips_are_on_nodeglob () { + local negating=false + if [ "$1" = "!" ] ; then + negating=true ; shift + fi local nodeglob="$1" ; shift local ips="$*" @@ -447,17 +453,23 @@ ips_are_on_nodeglob () all_ips_on_node 1 - while read ip pnn ; do - for check in $ips ; do + for check in $ips ; do + while read ip pnn ; do if [ "$check" = "$ip" ] ; then case "$pnn" in - ($nodeglob) : ;; - (*) return 1 ;; + ($nodeglob) if $negating ; then return 1 ; fi ;; + (*) if ! $negating ; then return 1 ; fi ;; esac ips="${ips/${ip}}" # Remove from list + break fi - done - done <<<"$out" # bashism to avoid problem setting variable in pipeline. + # If we're negating and we didn't see the address then it + # isn't hosted by anyone! + if $negating ; then + ips="${ips/${check}}" + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + done ips="${ips// }" # Remove any spaces. [ -z "$ips" ] -- cgit From 8cbf3211b7375ebc99980a6e58f7b5289c4070b9 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 11 Apr 2013 16:58:34 +1000 Subject: tests: Support waiting for "recovered" state in tests Signed-off-by: Amitay Isaacs Reviewed-by: Michael Adam (This used to be ctdb commit 2438f3a4944f7adbcae4cc1b9d5452714244afe7) --- ctdb/tests/scripts/integration.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 2e5fb37d7a..2ec827c14f 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -370,7 +370,7 @@ node_has_status () local pnn="$1" local status="$2" - local bits fpat mpat + local bits fpat mpat rpat case "$status" in (unhealthy) bits="?:?:?:1:*" ;; (healthy) bits="?:?:?:0:*" ;; @@ -386,6 +386,7 @@ node_has_status () (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; + (recovered) rpat='^Recovery mode:NORMAL \(0\)$' ;; *) echo "node_has_status: unknown status \"$status\"" return 1 @@ -410,6 +411,8 @@ node_has_status () $CTDB statistics -n "$pnn" | egrep -q "$fpat" elif [ -n "$mpat" ] ; then $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" + elif [ -n "$rpat" ] ; then + $CTDB status -n "$pnn" | egrep -q "$rpat" else echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' return 1 -- cgit From b416376e6861691f63a794a1b012e423598a35df Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 3 May 2013 15:53:13 +1000 Subject: tests: Unit test diff output should use filtered output Signed-off-by: Martin Schwenke (This used to be ctdb commit 9721aae001b3023e9c8b4af2b143c0db3442d623) --- ctdb/tests/scripts/unit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/unit.sh b/ctdb/tests/scripts/unit.sh index b27df45e76..c7c2b7a273 100644 --- a/ctdb/tests/scripts/unit.sh +++ b/ctdb/tests/scripts/unit.sh @@ -75,7 +75,7 @@ EOF echo "$required_output" >"$_outr" _outf=$(mktemp) - echo "$_out" >"$_outf" + echo "$_fout" >"$_outf" cat < Date: Tue, 30 Apr 2013 15:07:49 +1000 Subject: locking: Use separate locking helper binary for locking Signed-off-by: Amitay Isaacs (This used to be ctdb commit 7cde53a6cbe74b1e46f7e1bca298df82c08de866) --- ctdb/tests/scripts/integration.bash | 1 + 1 file changed, 1 insertion(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 2ec827c14f..429df69de1 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -14,6 +14,7 @@ if [ -n "$TEST_LOCAL_DAEMONS" ] ; then # Otherwise CTDB need to be installed on all nodes. if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then PATH="${ctdb_dir}/bin:${PATH}" + export CTDB_LOCK_HELPER="${ctdb_dir}/bin/ctdb_lock_helper" fi export CTDB_NODES="${TEST_VAR_DIR}/nodes.txt" -- cgit From e6673f2c46d9e5261c6a092ef55e685d6509c940 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Wed, 8 May 2013 16:25:30 +1000 Subject: tests: Fix output of run_tests usage (This used to be ctdb commit 29911fa44a480c17c701528ef46919b2a962a366) --- ctdb/tests/scripts/run_tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 50a578b26c..35db196361 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -8,8 +8,8 @@ Options: -s Print a summary of tests results after running all tests -l Use local daemons for integration tests -e Exit on the first test failure - -V Use as $TEST_VAR_DIR - -C Clean up - kill daemons and remove $TEST_VAR_DIR when done + -V Use as TEST_VAR_DIR + -C Clean up - kill daemons and remove TEST_VAR_DIR when done -v Verbose - print test output for non-failures (only some tests) -A Use "cat -A" to print test output (only some tests) -D Show diff between failed/expected test output (some tests only) -- cgit From 17d72884403d3926c840cd39bbaa1d47d1b6da55 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Sat, 25 May 2013 19:08:49 +1000 Subject: tests: Fix integration tests to use real private IPs 192.0.2.x was a typo. Signed-off-by: Martin Schwenke (This used to be ctdb commit c9e36f596c63c9af7f80d7cb8d7a5c6dcca4860a) --- ctdb/tests/scripts/integration.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 429df69de1..7d77139110 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -574,8 +574,8 @@ daemons_setup () echo 127.0.0.$i >>"$CTDB_NODES" # 2 public addresses on most nodes, just to make things interesting. if [ $(($i - 1)) -ne $no_public_ips ] ; then - echo "192.0.2.$i/24 lo" >>"$public_addresses_all" - echo "192.0.2.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >>"$public_addresses_all" + echo "192.168.234.$i/24 lo" >>"$public_addresses_all" + echo "192.168.234.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >>"$public_addresses_all" fi fi done -- cgit From 66019e32879a09094836d9a263a413397a5d2010 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Sat, 25 May 2013 19:57:24 +1000 Subject: scripts: Provide mktemp function for platforms without mktemp command This is needed for AIX and possibly others. Also provide a cheaper mktemp function is needed in the run_tests script. Signed-off-by: Martin Schwenke (This used to be ctdb commit b2b572e9049c7138bd223226475bef8fe3e01f10) --- ctdb/tests/scripts/run_tests | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 35db196361..13d211a7f0 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -143,6 +143,27 @@ summary="" rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's@^0$@80@') ww=$((rows - 7)) +if ! which mktemp >/dev/null 2>&1 ; then + # Not perfect, but it will do... + mktemp () + { + _dir=false + if [ "$1" = "-d" ] ; then + _dir=true + fi + _t="${TMPDIR:-/tmp}/tmp.$$.$RANDOM" + ( + umask 077 + if $_dir ; then + mkdir "$_t" + else + >"$_t" + fi + ) + echo "$_t" + } +fi + tf=$(mktemp) sf=$(mktemp) -- cgit From 7dca4420572ce43c6ca5a95c1eaa4d71c6a65198 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 27 May 2013 15:16:28 +1000 Subject: tests/scripts: Delete unused $rows and $ww variables from run_tests Signed-off-by: Martin Schwenke (This used to be ctdb commit 80b3cf2c652c6098390cdd0dbb3edc648f7df487) --- ctdb/tests/scripts/run_tests | 3 --- 1 file changed, 3 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 13d211a7f0..a3af5cbbc1 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -140,9 +140,6 @@ tests_passed=0 tests_failed=0 summary="" -rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's@^0$@80@') -ww=$((rows - 7)) - if ! which mktemp >/dev/null 2>&1 ; then # Not perfect, but it will do... mktemp () -- cgit From 529db4d52c36b33e923654c1051017eac90fce38 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 28 May 2013 11:26:17 +1000 Subject: tests/integration: Improve debug output for unhealthy cluster after restart Signed-off-by: Martin Schwenke (This used to be ctdb commit 25a6fd784cde96f3d20a79f70b5589b5c4aca675) --- ctdb/tests/scripts/integration.bash | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 7d77139110..45e0b99e87 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -740,8 +740,6 @@ restart_ctdb () continue } - local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) - echo "Setting RerecoveryTimeout to 1" onnode -pq all "$CTDB setvar RerecoveryTimeout 1" @@ -756,7 +754,10 @@ restart_ctdb () # Cluster is still healthy. Good, we're done! if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then - echo "Cluster become UNHEALTHY again. Restarting..." + echo "Cluster became UNHEALTHY again [$(date)]" + onnode -p all ctdb status -Y 2>&1 + onnode -p all ctdb scriptstatus 2>&1 + echo "Restarting..." continue fi @@ -768,7 +769,9 @@ restart_ctdb () done echo "Cluster UNHEALTHY... too many attempts..." - echo "$debug_out" + onnode -p all ctdb status -Y 2>&1 + onnode -p all ctdb scriptstatus 2>&1 + # Try to make the calling test fail status=1 return 1 -- cgit From 0a80d65c2efcb1a76bffb954d0fad218a9c62975 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 20 Jun 2013 16:42:30 +1000 Subject: tests: Integration test infrastructure should do only a single recovery No need for 2 recoveries after a restart. Signed-off-by: Martin Schwenke (This used to be ctdb commit b953524185632d7f96a76d8f3bbed7ac1d143d40) --- ctdb/tests/scripts/integration.bash | 4 ---- 1 file changed, 4 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 45e0b99e87..a43ffe8be0 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -688,8 +688,6 @@ _ctdb_start_post () echo "Forcing a recovery..." onnode -q 0 $CTDB recover sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover echo "ctdb is ready" } @@ -749,8 +747,6 @@ restart_ctdb () echo "Forcing a recovery..." onnode -q 0 $CTDB recover sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover # Cluster is still healthy. Good, we're done! if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then -- cgit From 16d374f75e9120a2d8ea3bb7b503d9d0f9f72a51 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 20 Jun 2013 16:43:10 +1000 Subject: tests: Integration tests use "ctdb nodestatus" for healthy cluster check Also check that we're not in recovery mode. Signed-off-by: Martin Schwenke (This used to be ctdb commit b7aaa28b3a6a2de923417f3d143f8d516447711e) --- ctdb/tests/scripts/integration.bash | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index a43ffe8be0..95d67c3a35 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -320,21 +320,8 @@ sleep_for () _cluster_is_healthy () { - local out x count line - - out=$($CTDB -Y status 2>/dev/null) || return 1 - - { - read x - count=0 - while read line ; do - # We need to see valid lines if we're going to be healthy. - [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1)) - # A line indicating a node is unhealthy causes failure. - [ "${line##:*:*:*1:}" != "$line" ] && return 1 - done - [ $count -gt 0 ] && return $? - } <<<"$out" # Yay bash! + $CTDB nodestatus all >/dev/null && \ + node_has_status 0 recovered } cluster_is_healthy () -- cgit From 1584f296b4d0a7292afa188998cbe1e3f4c75b75 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Fri, 19 Jul 2013 19:59:43 +1000 Subject: tests: Fix exit status of run_tests when a single test is run with -H Signed-off-by: Martin Schwenke (This used to be ctdb commit 9d6e1c147bd036d832b98c155f405ee2a5d6f57f) --- ctdb/tests/scripts/run_tests | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index a3af5cbbc1..719a16e52a 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -261,10 +261,10 @@ else echo "Not cleaning up TEST_VAR_DIR=$TEST_VAR_DIR" fi -if [ $tests_failed -gt 0 ] ; then - if $no_header || $exit_on_fail ; then - exit $status - else - exit 1 - fi +if $no_header || $exit_on_fail ; then + exit $status +elif [ $tests_failed -gt 0 ] ; then + exit 1 +else + exit 0 fi -- cgit From 6882625cfea795b6d5a5aaeef4bdec51a25baa65 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 22 Jul 2013 14:32:13 +1000 Subject: tests/complex: Fix NFS tests to work with root_squash Refactor the NFS test setup/cleanup code into new common functions. Signed-off-by: Martin Schwenke Pair-programmed-with: Amitay Isaacs (This used to be ctdb commit 29e98017221326bdc9b1c4f7c05b3b495c1de29b) --- ctdb/tests/scripts/integration.bash | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 95d67c3a35..df93dcf6bb 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -949,6 +949,42 @@ wait_for_monitor_event () } +####################################### + +nfs_test_setup () +{ + select_test_node_and_ips + + nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p') + + echo "Creating test subdirectory..." + try_command_on_node $test_node "mktemp -d --tmpdir=$nfs_first_export" + nfs_test_dir="$out" + try_command_on_node $test_node "chmod 777 $nfs_test_dir" + + nfs_mnt_d=$(mktemp -d) + nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE" + nfs_remote_file="${nfs_test_dir}/TEST_FILE" + + ctdb_test_exit_hook_add nfs_test_cleanup + + echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..." + mount -o timeo=1,hard,intr,vers=3 \ + ${test_ip}:${nfs_first_export} ${nfs_mnt_d} +} + +nfs_test_cleanup () +{ + rm -f "$nfs_local_file" + umount -f "$nfs_mnt_d" + rmdir "$nfs_mnt_d" + onnode -q $test_node rmdir "$nfs_test_dir" +} + + + +####################################### + # Make sure that $CTDB is set. : ${CTDB:=ctdb} -- cgit From 34d55048bc21005b7e5b15a8ceca1a0f6377a024 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 22 Jul 2013 20:11:58 +1000 Subject: tests: Always tally the number of passed/failed tests Regardless of whether a summary is being printed! Signed-off-by: Martin Schwenke (This used to be ctdb commit a69e03a5e4671e998d45b4fef8611a421bbdb3e1) --- ctdb/tests/scripts/run_tests | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 719a16e52a..171e8197f7 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -175,13 +175,16 @@ run_one_test () ctdb_test_run "$_f" | tee "$tf" | show_progress status=$? + if [ $status -eq 0 ] ; then + tests_passed=$(($tests_passed + 1)) + else + tests_failed=$(($tests_failed + 1)) + fi if $with_summary ; then if [ $status -eq 0 ] ; then - tests_passed=$(($tests_passed + 1)) _t=" PASSED " else _t="*FAILED*" - tests_failed=$(($tests_failed + 1)) fi if $with_desc ; then desc=$(tail -n +4 $tf | head -n 1) -- cgit From 9256010cfbd3d82ba2eba0311385eede91819490 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 23 Sep 2013 16:24:46 +1000 Subject: tests/integration: Tweak ctdbd startup options * --public-interface is not needed * Add --sloppy-start to speed up restarts Signed-off-by: Martin Schwenke (This used to be ctdb commit d0dec5b8e60316701fdd02150c4dd8f01aacbfda) --- ctdb/tests/scripts/integration.bash | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index df93dcf6bb..888528cb2b 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -585,11 +585,7 @@ daemons_start_1 () fi local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") - local ctdb_options="--reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --listen=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" - - if [ -n "$TEST_LOCAL_DAEMONS" ] ; then - ctdb_options="$ctdb_options --public-interface=lo" - fi + local ctdb_options="--sloppy-start --reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --listen=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" if [ $pnn -eq $no_public_ips ] ; then ctdb_options="$ctdb_options --public-addresses=/dev/null" -- cgit From 4526fdbbca4fbadc53ec354ceca7cc59ba3a31b6 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 15 Oct 2013 11:54:58 +1100 Subject: scripts: Remove support for CTDB_OPTIONS configuration variable Allowing people to put random options in CTDB_OPTIONS complicates some logic (particularly around use of syslog). If we're going to have variables for options then let's make sure we have a variable for each option and make people use them. Signed-off-by: Martin Schwenke (This used to be ctdb commit e55f3a1577eff0182802b0341d865d961aeae1c7) --- ctdb/tests/scripts/integration.bash | 6 ------ 1 file changed, 6 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 888528cb2b..59f37c919e 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -620,12 +620,6 @@ _ctdb_hack_options () { local ctdb_options="$*" - # We really just want to pass CTDB_OPTIONS but on RH - # /etc/sysconfig/ctdb can, and frequently does, set that variable. - # So instead, we hack badly. We'll add these as we use them. - # Note that these may still be overridden by the above file... but - # we tend to use the exotic options here... so that is unlikely. - case "$ctdb_options" in *--start-as-stopped*) export CTDB_START_AS_STOPPED="yes" -- cgit From b331fab51541b8460db797b662e03c618f93b21e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 22 Oct 2013 15:36:30 +1100 Subject: tests/integration: Pass --valgrinding option when running under valgrind Signed-off-by: Martin Schwenke (This used to be ctdb commit 913f229508302378212678d98c22606a4954b09c) --- ctdb/tests/scripts/integration.bash | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 59f37c919e..e5f1972a15 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -594,6 +594,10 @@ daemons_start_1 () ctdb_options="$ctdb_options --public-addresses=$public_addresses_mine" fi + if [ -n "$VALGRIND" ] ; then + ctdb_options="$ctdb_options --valgrinding" + fi + # We'll use "pkill -f" to kill the daemons with # "--socket=.* --nlist .* --nopublicipcheck" as context. $VALGRIND ctdbd --socket="${TEST_VAR_DIR}/sock.$pnn" $ctdb_options "$@" ||return 1 -- cgit From c07e3830b3c4f24dccacf9492b1e5e08bacab760 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 21 Oct 2013 19:08:52 +1100 Subject: common: New function mkdir_p() Behaves like mkdir -p. Signed-off-by: Martin Schwenke Pair-programmed-with: Amitay Isaacs (This used to be ctdb commit afe2145d91725daf1399f0a24f1cddcf65f0ec31) --- ctdb/tests/scripts/integration.bash | 4 ---- 1 file changed, 4 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index e5f1972a15..59f37c919e 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -594,10 +594,6 @@ daemons_start_1 () ctdb_options="$ctdb_options --public-addresses=$public_addresses_mine" fi - if [ -n "$VALGRIND" ] ; then - ctdb_options="$ctdb_options --valgrinding" - fi - # We'll use "pkill -f" to kill the daemons with # "--socket=.* --nlist .* --nopublicipcheck" as context. $VALGRIND ctdbd --socket="${TEST_VAR_DIR}/sock.$pnn" $ctdb_options "$@" ||return 1 -- cgit From 7eb680a95fcdacfd47d7ddab3a4df5b9ed0c837d Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Thu, 3 Oct 2013 15:19:05 +1000 Subject: build: Move the default CTDB socket from /tmp to /var/run/ctdb Use /var/run/ctdb/ctdbd.socket because there might be other daemons that need sockets in the future. The local daemons test code to create a link for the default convenience socket has to be removed because the link can't be created as a regular user in the new location. This should be OK since all calls to the ctdb tool in the test code should be wrapped in onnode. When debugging tests, a developer will have to set CTDB_SOCKET by hand. Signed-off-by: Amitay Isaacs Pair-programmed-with: Martin Schwenke (This used to be ctdb commit dc67a4e24af9d07aead2a1710eeaf5d6cc409201) --- ctdb/tests/scripts/integration.bash | 4 ---- 1 file changed, 4 deletions(-) (limited to 'ctdb/tests/scripts') diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 59f37c919e..040a36048d 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -608,10 +608,6 @@ daemons_start () for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do daemons_start_1 $i "$@" done - - if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then - ln -sf "${TEST_VAR_DIR}/sock.0" /tmp/ctdb.socket || return 1 - fi } ####################################### -- cgit