From bf197d097f86ed5c75622778332a7c85c4684a12 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 18 Apr 2012 14:55:21 +1000 Subject: tests: Rename ctdb_test_functions.bash to integration.bash Signed-off-by: Martin Schwenke (This used to be ctdb commit 79adb50b3ce3873c3baf9e6715c1d1c3f181ce43) --- ctdb/tests/complex/01_ctdb_nfs_skip_share_check.sh | 2 +- .../complex/02_ctdb_samba_skip_share_check.sh | 2 +- ctdb/tests/complex/11_ctdb_delip_removes_ip.sh | 2 +- ctdb/tests/complex/31_nfs_tickle.sh | 2 +- ctdb/tests/complex/32_cifs_tickle.sh | 2 +- ctdb/tests/complex/33_gratuitous_arp.sh | 2 +- ctdb/tests/complex/41_failover_ping_discrete.sh | 2 +- ctdb/tests/complex/42_failover_ssh_hostname.sh | 2 +- ctdb/tests/complex/43_failover_nfs_basic.sh | 2 +- ctdb/tests/complex/44_failover_nfs_oneway.sh | 2 +- ctdb/tests/scripts/ctdb_test_functions.bash | 1053 -------------------- ctdb/tests/scripts/integration.bash | 1053 ++++++++++++++++++++ ctdb/tests/scripts/test_wrap | 4 +- ctdb/tests/simple/00_ctdb_init.sh | 2 +- ctdb/tests/simple/00_ctdb_onnode.sh | 2 +- ctdb/tests/simple/01_ctdb_version.sh | 2 +- ctdb/tests/simple/02_ctdb_listvars.sh | 2 +- ctdb/tests/simple/03_ctdb_getvar.sh | 2 +- ctdb/tests/simple/04_ctdb_setvar.sh | 2 +- ctdb/tests/simple/05_ctdb_listnodes.sh | 2 +- ctdb/tests/simple/06_ctdb_getpid.sh | 2 +- ctdb/tests/simple/07_ctdb_process_exists.sh | 2 +- ctdb/tests/simple/08_ctdb_isnotrecmaster.sh | 2 +- ctdb/tests/simple/09_ctdb_ping.sh | 2 +- ctdb/tests/simple/11_ctdb_ip.sh | 2 +- ctdb/tests/simple/12_ctdb_getdebug.sh | 2 +- ctdb/tests/simple/13_ctdb_setdebug.sh | 2 +- ctdb/tests/simple/14_ctdb_statistics.sh | 2 +- ctdb/tests/simple/15_ctdb_statisticsreset.sh | 2 +- ctdb/tests/simple/16_ctdb_config_add_ip.sh | 2 +- ctdb/tests/simple/17_ctdb_config_delete_ip.sh | 2 +- ctdb/tests/simple/23_ctdb_moveip.sh | 2 +- ctdb/tests/simple/24_ctdb_getdbmap.sh | 2 +- ctdb/tests/simple/25_dumpmemory.sh | 2 +- ..._ctdb_config_check_error_on_unreachable_ctdb.sh | 2 +- ctdb/tests/simple/31_ctdb_disable.sh | 2 +- ctdb/tests/simple/32_ctdb_enable.sh | 2 +- ctdb/tests/simple/41_ctdb_stop.sh | 2 +- ctdb/tests/simple/42_ctdb_continue.sh | 2 +- ctdb/tests/simple/43_stop_recmaster_yield.sh | 2 +- ctdb/tests/simple/51_ctdb_bench.sh | 2 +- ctdb/tests/simple/52_ctdb_fetch.sh | 2 +- ctdb/tests/simple/53_ctdb_transaction.sh | 2 +- ctdb/tests/simple/54_ctdb_transaction_recovery.sh | 2 +- ctdb/tests/simple/70_recoverpdbbyseqnum.sh | 2 +- ctdb/tests/simple/71_ctdb_wipedb.sh | 2 +- ctdb/tests/simple/72_update_record_persistent.sh | 2 +- ctdb/tests/simple/73_tunable_NoIPTakeover.sh | 2 +- ctdb/tests/simple/75_readonly_records_basic.sh | 2 +- 49 files changed, 1101 insertions(+), 1101 deletions(-) delete mode 100644 ctdb/tests/scripts/ctdb_test_functions.bash create mode 100644 ctdb/tests/scripts/integration.bash diff --git a/ctdb/tests/complex/01_ctdb_nfs_skip_share_check.sh b/ctdb/tests/complex/01_ctdb_nfs_skip_share_check.sh index 969b445191..a7ad938fec 100755 --- a/ctdb/tests/complex/01_ctdb_nfs_skip_share_check.sh +++ b/ctdb/tests/complex/01_ctdb_nfs_skip_share_check.sh @@ -54,7 +54,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/02_ctdb_samba_skip_share_check.sh b/ctdb/tests/complex/02_ctdb_samba_skip_share_check.sh index c7eadc6b11..9097a78baf 100755 --- a/ctdb/tests/complex/02_ctdb_samba_skip_share_check.sh +++ b/ctdb/tests/complex/02_ctdb_samba_skip_share_check.sh @@ -56,7 +56,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh index f60b630096..043c345fac 100755 --- a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh +++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh @@ -38,7 +38,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/31_nfs_tickle.sh b/ctdb/tests/complex/31_nfs_tickle.sh index dcbd4d77b2..ce4ae81c9d 100755 --- a/ctdb/tests/complex/31_nfs_tickle.sh +++ b/ctdb/tests/complex/31_nfs_tickle.sh @@ -39,7 +39,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/32_cifs_tickle.sh b/ctdb/tests/complex/32_cifs_tickle.sh index 3a77381960..93634e72bd 100755 --- a/ctdb/tests/complex/32_cifs_tickle.sh +++ b/ctdb/tests/complex/32_cifs_tickle.sh @@ -38,7 +38,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/33_gratuitous_arp.sh b/ctdb/tests/complex/33_gratuitous_arp.sh index 8a3de03bed..721b0f2a34 100755 --- a/ctdb/tests/complex/33_gratuitous_arp.sh +++ b/ctdb/tests/complex/33_gratuitous_arp.sh @@ -38,7 +38,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/41_failover_ping_discrete.sh b/ctdb/tests/complex/41_failover_ping_discrete.sh index 73bc84e30d..88b2013f06 100755 --- a/ctdb/tests/complex/41_failover_ping_discrete.sh +++ b/ctdb/tests/complex/41_failover_ping_discrete.sh @@ -32,7 +32,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/42_failover_ssh_hostname.sh b/ctdb/tests/complex/42_failover_ssh_hostname.sh index f1a01f799d..defe15ad11 100755 --- a/ctdb/tests/complex/42_failover_ssh_hostname.sh +++ b/ctdb/tests/complex/42_failover_ssh_hostname.sh @@ -32,7 +32,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/43_failover_nfs_basic.sh b/ctdb/tests/complex/43_failover_nfs_basic.sh index 883a452190..71a822968f 100755 --- a/ctdb/tests/complex/43_failover_nfs_basic.sh +++ b/ctdb/tests/complex/43_failover_nfs_basic.sh @@ -36,7 +36,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/complex/44_failover_nfs_oneway.sh b/ctdb/tests/complex/44_failover_nfs_oneway.sh index 8fa12b99ca..7da8d019ec 100755 --- a/ctdb/tests/complex/44_failover_nfs_oneway.sh +++ b/ctdb/tests/complex/44_failover_nfs_oneway.sh @@ -38,7 +38,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" set -e diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash deleted file mode 100644 index 70c5b5d10b..0000000000 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ /dev/null @@ -1,1053 +0,0 @@ -# Hey Emacs, this is a -*- shell-script -*- !!! :-) - -fail () -{ - echo "$*" - exit 1 -} - -###################################################################### - -ctdb_check_time_logs () -{ - local threshold=20 - - local jump=false - local prev="" - local ds_prev="" - local node="" - - out=$(onnode all tail -n 20 /var/log/ctdb.test.time.log 2>&1) - - if [ $? -eq 0 ] ; then - local line - while read line ; do - case "$line" in - \>\>\ NODE:\ *\ \<\<) - node="${line#>> NODE: }" - node=${node% <<*} - ds_prev="" - ;; - *\ *) - set -- $line - ds_curr="$1${2:0:1}" - if [ -n "$ds_prev" ] && \ - [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then - echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})" - jump=true - fi - prev="$line" - ds_prev="$ds_curr" - ;; - esac - done <<<"$out" - else - echo Error getting time logs - fi - if $jump ; then - echo "Check time sync (test client first):" - date - onnode -p all date - echo "Information from test client:" - hostname - top -b -n 1 - echo "Information from cluster nodes:" - onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo" - fi -} - -ctdb_test_exit () -{ - local status=$? - - trap - 0 - - [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status - status=$(($testfailures+0)) - - # Avoid making a test fail from this point onwards. The test is - # now complete. - set +e - - echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." - - if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ - $status -ne 0 ] ; then - ctdb_check_time_logs - fi - - eval "$ctdb_test_exit_hook" || true - unset ctdb_test_exit_hook - - if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then - - restart_ctdb - else - # This could be made unconditional but then we might get - # duplication from the recovery in restart_ctdb. We want to - # leave the recovery in restart_ctdb so that future tests that - # might do a manual restart mid-test will benefit. - echo "Forcing a recovery..." - onnode 0 $CTDB recover - fi - - exit $status -} - -ctdb_test_exit_hook_add () -{ - ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" -} - -ctdb_test_usage() -{ - local status=${1:-2} - - cat <&1) || { - - echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" - echo "$out" - return 1 - } - - if $verbose ; then - echo "Output of \"$cmd\":" - echo "$out" - fi -} - -sanity_check_output () -{ - local min_lines="$1" - local regexp="$2" # Should be anchored as necessary. - local output="$3" - - local ret=0 - - local num_lines=$(echo "$output" | wc -l) - echo "There are $num_lines lines of output" - if [ $num_lines -lt $min_lines ] ; then - echo "BAD: that's less than the required number (${min_lines})" - ret=1 - fi - - local status=0 - local unexpected # local doesn't pass through status of command on RHS. - unexpected=$(echo "$output" | egrep -v "$regexp") || status=$? - - # Note that this is reversed. - if [ $status -eq 0 ] ; then - echo "BAD: unexpected lines in output:" - echo "$unexpected" | cat -A - ret=1 - else - echo "Output lines look OK" - fi - - return $ret -} - -sanity_check_ips () -{ - local ips="$1" # list of "ip node" lines - - echo "Sanity checking IPs..." - - local x ipp prev - prev="" - while read x ipp ; do - [ "$ipp" = "-1" ] && break - if [ -n "$prev" -a "$ipp" != "$prev" ] ; then - echo "OK" - return 0 - fi - prev="$ipp" - done <<<"$ips" - - echo "BAD: a node was -1 or IPs are only assigned to one node" - echo "Are you running an old version of CTDB?" - return 1 -} - -# This returns a list of "ip node" lines in $out -all_ips_on_node() -{ - local node=$@ - try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" -} - -select_test_node_and_ips () -{ - all_ips_on_node 0 - - # When selecting test_node we just want a node that has public - # IPs. This will work and is economically semi-random. :-) - local x - read x test_node <<<"$out" - - test_node_ips="" - local ip pnn - while read ip pnn ; do - if [ "$pnn" = "$test_node" ] ; then - test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" - fi - done <<<"$out" # bashism to avoid problem setting variable in pipeline. - - echo "Selected node ${test_node} with IPs: ${test_node_ips}." - test_ip="${test_node_ips%% *}" -} - -####################################### - -# Wait until either timeout expires or command succeeds. The command -# will be tried once per second. -wait_until () -{ - local timeout="$1" ; shift # "$@" is the command... - - local negate=false - if [ "$1" = "!" ] ; then - negate=true - shift - fi - - echo -n "<${timeout}|" - local t=$timeout - while [ $t -gt 0 ] ; do - local rc=0 - "$@" || rc=$? - if { ! $negate && [ $rc -eq 0 ] ; } || \ - { $negate && [ $rc -ne 0 ] ; } ; then - echo "|$(($timeout - $t))|" - echo "OK" - return 0 - fi - echo -n . - t=$(($t - 1)) - sleep 1 - done - - echo "*TIMEOUT*" - - return 1 -} - -sleep_for () -{ - echo -n "=${1}|" - for i in $(seq 1 $1) ; do - echo -n '.' - sleep 1 - done - echo '|' -} - -_cluster_is_healthy () -{ - local out x count line - - out=$($CTDB -Y status 2>/dev/null) || return 1 - - { - read x - count=0 - while read line ; do - # We need to see valid lines if we're going to be healthy. - [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1)) - # A line indicating a node is unhealthy causes failure. - [ "${line##:*:*:*1:}" != "$line" ] && return 1 - done - [ $count -gt 0 ] && return $? - } <<<"$out" # Yay bash! -} - -cluster_is_healthy () -{ - if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then - echo "Cluster is HEALTHY" - return 0 - else - echo "Cluster is UNHEALTHY" - if ! ${ctdb_test_restart_scheduled:-false} ; then - echo "DEBUG AT $(date '+%F %T'):" - local i - for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do - echo "$i" - $i || true - done - fi - return 1 - fi -} - -wait_until_healthy () -{ - local timeout="${1:-120}" - - echo "Waiting for cluster to become healthy..." - - wait_until 120 _cluster_is_healthy -} - -# This function is becoming nicely overloaded. Soon it will collapse! :-) -node_has_status () -{ - local pnn="$1" - local status="$2" - - local bits fpat mpat - case "$status" in - (unhealthy) bits="?:?:?:1:*" ;; - (healthy) bits="?:?:?:0:*" ;; - (disconnected) bits="1:*" ;; - (connected) bits="0:*" ;; - (banned) bits="?:1:*" ;; - (unbanned) bits="?:0:*" ;; - (disabled) bits="?:?:1:*" ;; - (enabled) bits="?:?:0:*" ;; - (stopped) bits="?:?:?:?:1:*" ;; - (notstopped) bits="?:?:?:?:0:*" ;; - (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; - (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; - (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; - (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; - *) - echo "node_has_status: unknown status \"$status\"" - return 1 - esac - - if [ -n "$bits" ] ; then - local out x line - - out=$($CTDB -Y status 2>&1) || return 1 - - { - read x - while read line ; do - # This needs to be done in 2 steps to avoid false matches. - local line_bits="${line#:${pnn}:*:}" - [ "$line_bits" = "$line" ] && continue - [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 - done - return 1 - } <<<"$out" # Yay bash! - elif [ -n "$fpat" ] ; then - $CTDB statistics -n "$pnn" | egrep -q "$fpat" - elif [ -n "$mpat" ] ; then - $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" - else - echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' - return 1 - fi -} - -wait_until_node_has_status () -{ - local pnn="$1" - local status="$2" - local timeout="${3:-30}" - local proxy_pnn="${4:-any}" - - echo "Waiting until node $pnn has status \"$status\"..." - - if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then - for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do - echo "$i" - $i || true - done - - return 1 - fi - -} - -# Useful for superficially testing IP failover. -# IPs must be on nodes matching nodeglob. -ips_are_on_nodeglob () -{ - local nodeglob="$1" ; shift - local ips="$*" - - local out - - all_ips_on_node 1 - - while read ip pnn ; do - for check in $ips ; do - if [ "$check" = "$ip" ] ; then - case "$pnn" in - ($nodeglob) : ;; - (*) return 1 ;; - esac - ips="${ips/${ip}}" # Remove from list - fi - done - done <<<"$out" # bashism to avoid problem setting variable in pipeline. - - ips="${ips// }" # Remove any spaces. - [ -z "$ips" ] -} - -wait_until_ips_are_on_nodeglob () -{ - echo "Waiting for IPs to fail over..." - - wait_until 60 ips_are_on_nodeglob "$@" -} - -node_has_some_ips () -{ - local node="$1" - - local out - - all_ips_on_node 1 - - while read ip pnn ; do - if [ "$node" = "$pnn" ] ; then - return 0 - fi - done <<<"$out" # bashism to avoid problem setting variable in pipeline. - - return 1 -} - -wait_until_node_has_some_ips () -{ - echo "Waiting for node to have some IPs..." - - wait_until 60 node_has_some_ips "$@" -} - -get_src_socket () -{ - local proto="$1" - local dst_socket="$2" - local pid="$3" - local prog="$4" - - local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" - out=$(netstat -tanp | - egrep "$pat" | - awk '{ print $4 }') - - [ -n "$out" ] -} - -wait_until_get_src_socket () -{ - local proto="$1" - local dst_socket="$2" - local pid="$3" - local prog="$4" - - echo "Waiting for ${prog} to establish connection to ${dst_socket}..." - - wait_until 5 get_src_socket "$@" -} - -####################################### - -# filename will be in $tcpdump_filename, pid in $tcpdump_pid -tcpdump_start () -{ - tcpdump_filter="$1" # global - - echo "Running tcpdump..." - tcpdump_filename=$(mktemp) - ctdb_test_exit_hook_add "rm -f $tcpdump_filename" - - # The only way of being sure that tcpdump is listening is to send - # some packets that it will see. So we use dummy pings - the -U - # option to tcpdump ensures that packets are flushed to the file - # as they are captured. - local dummy_addr="127.3.2.1" - local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" - tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & - ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" - - echo "Waiting for tcpdump output file to be ready..." - ping -q "$dummy_addr" >/dev/null 2>&1 & - ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" - - tcpdump_listen_for_dummy () - { - tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 - } - - wait_until 10 tcpdump_listen_for_dummy -} - -# By default, wait for 1 matching packet. -tcpdump_wait () -{ - local count="${1:-1}" - local filter="${2:-${tcpdump_filter}}" - - tcpdump_check () - { - local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) - [ $found -ge $count ] - } - - echo "Waiting for tcpdump to capture some packets..." - if ! wait_until 30 tcpdump_check ; then - echo "DEBUG AT $(date '+%F %T'):" - local i - for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do - echo "$i" - $i || true - done - return 1 - fi -} - -tcpdump_show () -{ - local filter="${1:-${tcpdump_filter}}" - - tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null -} - -tcptickle_sniff_start () -{ - local src="$1" - local dst="$2" - - local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" - local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" - local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 - local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" - tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" - local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" - - tcpdump_start "$filter" -} - -tcptickle_sniff_wait_show () -{ - tcpdump_wait 1 "$tcptickle_reset" - - echo "GOOD: here are some TCP tickle packets:" - tcpdump_show -} - -gratarp_sniff_start () -{ - tcpdump_start "arp host ${test_ip}" -} - -gratarp_sniff_wait_show () -{ - tcpdump_wait 2 - - echo "GOOD: this should be the some gratuitous ARPs:" - tcpdump_show -} - - -####################################### - -daemons_stop () -{ - echo "Attempting to politely shutdown daemons..." - onnode 1 $CTDB shutdown -n all || true - - echo "Sleeping for a while..." - sleep_for 1 - - if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then - echo "Killing remaining daemons..." - pkill -f $CTDB_DIR/bin/ctdbd - - if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then - echo "Once more with feeling.." - pkill -9 $CTDB_DIR/bin/ctdbd - fi - fi - - local var_dir=$CTDB_DIR/tests/var - rm -rf $var_dir/test.db -} - -daemons_setup () -{ - local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes - - local var_dir=$CTDB_DIR/tests/var - - mkdir -p $var_dir/test.db/persistent - - local public_addresses=$var_dir/public_addresses.txt - local no_public_addresses=$var_dir/no_public_addresses.txt - rm -f $CTDB_NODES $public_addresses $no_public_addresses - - # If there are (strictly) greater than 2 nodes then we'll randomly - # choose a node to have no public addresses. - local no_public_ips=-1 - [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes)) - echo "$no_public_ips" >$no_public_addresses - - local i - for i in $(seq 1 $num_nodes) ; do - if [ "${CTDB_USE_IPV6}x" != "x" ]; then - echo ::$i >> $nodes - ip addr add ::$i/128 dev lo - else - echo 127.0.0.$i >> $CTDB_NODES - # 2 public addresses on most nodes, just to make things interesting. - if [ $(($i - 1)) -ne $no_public_ips ] ; then - echo "192.0.2.$i/24 lo" >> $public_addresses - echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses - fi - fi - done -} - -daemons_start_1 () -{ - local pnn="$1" - shift # "$@" gets passed to ctdbd - - local var_dir=$CTDB_DIR/tests/var - - local public_addresses=$var_dir/public_addresses.txt - local no_public_addresses=$var_dir/no_public_addresses.txt - - local no_public_ips=-1 - [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses - - if [ "$no_public_ips" = $pnn ] ; then - echo "Node $no_public_ips will have no public IPs." - fi - - local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" - - if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then - ctdb_options="$ctdb_options --public-interface=lo" - fi - - if [ $pnn -eq $no_public_ips ] ; then - ctdb_options="$ctdb_options --public-addresses=/dev/null" - else - ctdb_options="$ctdb_options --public-addresses=$public_addresses" - fi - - # Need full path so we can use "pkill -f" to kill the daemons. - $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$pnn $ctdb_options "$@" ||return 1 -} - -daemons_start () -{ - # "$@" gets passed to ctdbd - - local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes - - echo "Starting $num_nodes ctdb daemons..." - - for i in $(seq 0 $(($num_nodes - 1))) ; do - daemons_start_1 $i "$@" - done - - local var_dir=$CTDB_DIR/tests/var - - if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then - ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 - fi -} - -####################################### - -_ctdb_hack_options () -{ - local ctdb_options="$*" - - # We really just want to pass CTDB_OPTIONS but on RH - # /etc/sysconfig/ctdb can, and frequently does, set that variable. - # So instead, we hack badly. We'll add these as we use them. - # Note that these may still be overridden by the above file... but - # we tend to use the exotic options here... so that is unlikely. - - case "$ctdb_options" in - *--start-as-stopped*) - export CTDB_START_AS_STOPPED="yes" - esac -} - -_restart_ctdb () -{ - _ctdb_hack_options "$@" - - if [ -e /etc/redhat-release ] ; then - service ctdb restart - else - /etc/init.d/ctdb restart - fi -} - -_ctdb_start () -{ - _ctdb_hack_options "$@" - - /etc/init.d/ctdb start -} - -setup_ctdb () -{ - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_setup - fi -} - -# Common things to do after starting one or more nodes. -_ctdb_start_post () -{ - onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 - - echo "Setting RerecoveryTimeout to 1" - onnode -pq all "$CTDB setvar RerecoveryTimeout 1" - - # In recent versions of CTDB, forcing a recovery like this blocks - # until the recovery is complete. Hopefully this will help the - # cluster to stabilise before a subsequent test. - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - - echo "ctdb is ready" -} - -# This assumes that ctdbd is not running on the given node. -ctdb_start_1 () -{ - local pnn="$1" - shift # "$@" is passed to ctdbd start. - - echo -n "Starting CTDB on node ${pnn}..." - - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_start_1 $pnn "$@" - else - onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@" - fi - - # If we're starting only 1 node then we're doing something weird. - ctdb_restart_when_done -} - -restart_ctdb () -{ - # "$@" is passed to ctdbd start. - - echo -n "Restarting CTDB" - if $ctdb_test_restart_scheduled ; then - echo -n " (scheduled)" - fi - echo "..." - - local i - for i in $(seq 1 5) ; do - if [ -n "$CTDB_NODES_SOCKETS" ] ; then - daemons_stop - daemons_start "$@" - else - onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" - fi || { - echo "Restart failed. Trying again in a few seconds..." - sleep_for 5 - continue - } - - onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { - echo "Cluster didn't become healthy. Restarting..." - continue - } - - local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) - - echo "Setting RerecoveryTimeout to 1" - onnode -pq all "$CTDB setvar RerecoveryTimeout 1" - - # In recent versions of CTDB, forcing a recovery like this - # blocks until the recovery is complete. Hopefully this will - # help the cluster to stabilise before a subsequent test. - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - sleep_for 1 - echo "Forcing a recovery..." - onnode -q 0 $CTDB recover - - # Cluster is still healthy. Good, we're done! - if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then - echo "Cluster become UNHEALTHY again. Restarting..." - continue - fi - - echo "Doing a sync..." - onnode -q 0 $CTDB sync - - echo "ctdb is ready" - return 0 - done - - echo "Cluster UNHEALTHY... too many attempts..." - echo "$debug_out" - # Try to make the calling test fail - status=1 - return 1 -} - -ctdb_restart_when_done () -{ - ctdb_test_restart_scheduled=true -} - -####################################### - -install_eventscript () -{ - local script_name="$1" - local script_contents="$2" - - if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then - # The quoting here is *very* fragile. However, we do - # experience the joy of installing a short script using - # onnode, and without needing to know the IP addresses of the - # nodes. - onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" - else - f="${CTDB_DIR}/tests/events.d/${script_name}" - echo "$script_contents" >"$f" - chmod 755 "$f" - fi -} - -uninstall_eventscript () -{ - local script_name="$1" - - if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then - onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" - else - rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" - fi -} - -####################################### - -# This section deals with the 99.ctdb_test eventscript. - -# Metafunctions: Handle a ctdb-test file on a node. -# given event. -ctdb_test_eventscript_file_create () -{ - local pnn="$1" - local type="$2" - - try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}" -} - -ctdb_test_eventscript_file_remove () -{ - local pnn="$1" - local type="$2" - - try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}" -} - -ctdb_test_eventscript_file_exists () -{ - local pnn="$1" - local type="$2" - - try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1 -} - - -# Handle a flag file on a node that is removed by 99.ctdb_test on the -# given event. -ctdb_test_eventscript_flag () -{ - local cmd="$1" - local pnn="$2" - local event="$3" - - ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}" -} - - -# Handle a trigger that causes 99.ctdb_test to fail it's monitor -# event. -ctdb_test_eventscript_unhealthy_trigger () -{ - local cmd="$1" - local pnn="$2" - - ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger" -} - -# Handle the file that 99.ctdb_test created to show that it has marked -# a node unhealthy because it detected the above trigger. -ctdb_test_eventscript_unhealthy_detected () -{ - local cmd="$1" - local pnn="$2" - - ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" -} - -# Handle a trigger that causes 99.ctdb_test to timeout it's monitor -# event. This should cause the node to be banned. -ctdb_test_eventscript_timeout_trigger () -{ - local cmd="$1" - local pnn="$2" - local event="$3" - - ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout" -} - -# Note that the eventscript can't use the above functions! -ctdb_test_eventscript_install () -{ - - local script='#!/bin/sh -out=$(ctdb pnn) -pnn="${out#PNN:}" - -rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" - -trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" -detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" -timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}" -case "$1" in - monitor) - if [ -e "$trigger" ] ; then - echo "${0}: Unhealthy because \"$trigger\" detected" - touch "$detected" - exit 1 - elif [ -e "$detected" -a ! -e "$trigger" ] ; then - echo "${0}: Healthy again, \"$trigger\" no longer detected" - rm "$detected" - fi - - ;; - *) - if [ -e "$timeout_trigger" ] ; then - echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected" - sleep 9999 - fi - ;; - *) - -esac - -exit 0 -' - install_eventscript "99.ctdb_test" "$script" -} - -ctdb_test_eventscript_uninstall () -{ - uninstall_eventscript "99.ctdb_test" -} - -# Note that this only works if you know all other monitor events will -# succeed. You also need to install the eventscript before using it. -wait_for_monitor_event () -{ - local pnn="$1" - - echo "Waiting for a monitor event on node ${pnn}..." - ctdb_test_eventscript_flag create $pnn "monitor" - - wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" - -} - -# Make sure that $CTDB is set. -: ${CTDB:=ctdb} diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash new file mode 100644 index 0000000000..70c5b5d10b --- /dev/null +++ b/ctdb/tests/scripts/integration.bash @@ -0,0 +1,1053 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +fail () +{ + echo "$*" + exit 1 +} + +###################################################################### + +ctdb_check_time_logs () +{ + local threshold=20 + + local jump=false + local prev="" + local ds_prev="" + local node="" + + out=$(onnode all tail -n 20 /var/log/ctdb.test.time.log 2>&1) + + if [ $? -eq 0 ] ; then + local line + while read line ; do + case "$line" in + \>\>\ NODE:\ *\ \<\<) + node="${line#>> NODE: }" + node=${node% <<*} + ds_prev="" + ;; + *\ *) + set -- $line + ds_curr="$1${2:0:1}" + if [ -n "$ds_prev" ] && \ + [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then + echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})" + jump=true + fi + prev="$line" + ds_prev="$ds_curr" + ;; + esac + done <<<"$out" + else + echo Error getting time logs + fi + if $jump ; then + echo "Check time sync (test client first):" + date + onnode -p all date + echo "Information from test client:" + hostname + top -b -n 1 + echo "Information from cluster nodes:" + onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo" + fi +} + +ctdb_test_exit () +{ + local status=$? + + trap - 0 + + [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + status=$(($testfailures+0)) + + # Avoid making a test fail from this point onwards. The test is + # now complete. + set +e + + echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." + + if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ + $status -ne 0 ] ; then + ctdb_check_time_logs + fi + + eval "$ctdb_test_exit_hook" || true + unset ctdb_test_exit_hook + + if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then + + restart_ctdb + else + # This could be made unconditional but then we might get + # duplication from the recovery in restart_ctdb. We want to + # leave the recovery in restart_ctdb so that future tests that + # might do a manual restart mid-test will benefit. + echo "Forcing a recovery..." + onnode 0 $CTDB recover + fi + + exit $status +} + +ctdb_test_exit_hook_add () +{ + ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" +} + +ctdb_test_usage() +{ + local status=${1:-2} + + cat <&1) || { + + echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" + echo "$out" + return 1 + } + + if $verbose ; then + echo "Output of \"$cmd\":" + echo "$out" + fi +} + +sanity_check_output () +{ + local min_lines="$1" + local regexp="$2" # Should be anchored as necessary. + local output="$3" + + local ret=0 + + local num_lines=$(echo "$output" | wc -l) + echo "There are $num_lines lines of output" + if [ $num_lines -lt $min_lines ] ; then + echo "BAD: that's less than the required number (${min_lines})" + ret=1 + fi + + local status=0 + local unexpected # local doesn't pass through status of command on RHS. + unexpected=$(echo "$output" | egrep -v "$regexp") || status=$? + + # Note that this is reversed. + if [ $status -eq 0 ] ; then + echo "BAD: unexpected lines in output:" + echo "$unexpected" | cat -A + ret=1 + else + echo "Output lines look OK" + fi + + return $ret +} + +sanity_check_ips () +{ + local ips="$1" # list of "ip node" lines + + echo "Sanity checking IPs..." + + local x ipp prev + prev="" + while read x ipp ; do + [ "$ipp" = "-1" ] && break + if [ -n "$prev" -a "$ipp" != "$prev" ] ; then + echo "OK" + return 0 + fi + prev="$ipp" + done <<<"$ips" + + echo "BAD: a node was -1 or IPs are only assigned to one node" + echo "Are you running an old version of CTDB?" + return 1 +} + +# This returns a list of "ip node" lines in $out +all_ips_on_node() +{ + local node=$@ + try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" +} + +select_test_node_and_ips () +{ + all_ips_on_node 0 + + # When selecting test_node we just want a node that has public + # IPs. This will work and is economically semi-random. :-) + local x + read x test_node <<<"$out" + + test_node_ips="" + local ip pnn + while read ip pnn ; do + if [ "$pnn" = "$test_node" ] ; then + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + echo "Selected node ${test_node} with IPs: ${test_node_ips}." + test_ip="${test_node_ips%% *}" +} + +####################################### + +# Wait until either timeout expires or command succeeds. The command +# will be tried once per second. +wait_until () +{ + local timeout="$1" ; shift # "$@" is the command... + + local negate=false + if [ "$1" = "!" ] ; then + negate=true + shift + fi + + echo -n "<${timeout}|" + local t=$timeout + while [ $t -gt 0 ] ; do + local rc=0 + "$@" || rc=$? + if { ! $negate && [ $rc -eq 0 ] ; } || \ + { $negate && [ $rc -ne 0 ] ; } ; then + echo "|$(($timeout - $t))|" + echo "OK" + return 0 + fi + echo -n . + t=$(($t - 1)) + sleep 1 + done + + echo "*TIMEOUT*" + + return 1 +} + +sleep_for () +{ + echo -n "=${1}|" + for i in $(seq 1 $1) ; do + echo -n '.' + sleep 1 + done + echo '|' +} + +_cluster_is_healthy () +{ + local out x count line + + out=$($CTDB -Y status 2>/dev/null) || return 1 + + { + read x + count=0 + while read line ; do + # We need to see valid lines if we're going to be healthy. + [ "${line#:[0-9]}" != "$line" ] && count=$(($count + 1)) + # A line indicating a node is unhealthy causes failure. + [ "${line##:*:*:*1:}" != "$line" ] && return 1 + done + [ $count -gt 0 ] && return $? + } <<<"$out" # Yay bash! +} + +cluster_is_healthy () +{ + if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster is HEALTHY" + return 0 + else + echo "Cluster is UNHEALTHY" + if ! ${ctdb_test_restart_scheduled:-false} ; then + echo "DEBUG AT $(date '+%F %T'):" + local i + for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do + echo "$i" + $i || true + done + fi + return 1 + fi +} + +wait_until_healthy () +{ + local timeout="${1:-120}" + + echo "Waiting for cluster to become healthy..." + + wait_until 120 _cluster_is_healthy +} + +# This function is becoming nicely overloaded. Soon it will collapse! :-) +node_has_status () +{ + local pnn="$1" + local status="$2" + + local bits fpat mpat + case "$status" in + (unhealthy) bits="?:?:?:1:*" ;; + (healthy) bits="?:?:?:0:*" ;; + (disconnected) bits="1:*" ;; + (connected) bits="0:*" ;; + (banned) bits="?:1:*" ;; + (unbanned) bits="?:0:*" ;; + (disabled) bits="?:?:1:*" ;; + (enabled) bits="?:?:0:*" ;; + (stopped) bits="?:?:?:?:1:*" ;; + (notstopped) bits="?:?:?:?:0:*" ;; + (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; + (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; + (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; + (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; + *) + echo "node_has_status: unknown status \"$status\"" + return 1 + esac + + if [ -n "$bits" ] ; then + local out x line + + out=$($CTDB -Y status 2>&1) || return 1 + + { + read x + while read line ; do + # This needs to be done in 2 steps to avoid false matches. + local line_bits="${line#:${pnn}:*:}" + [ "$line_bits" = "$line" ] && continue + [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 + done + return 1 + } <<<"$out" # Yay bash! + elif [ -n "$fpat" ] ; then + $CTDB statistics -n "$pnn" | egrep -q "$fpat" + elif [ -n "$mpat" ] ; then + $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" + else + echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' + return 1 + fi +} + +wait_until_node_has_status () +{ + local pnn="$1" + local status="$2" + local timeout="${3:-30}" + local proxy_pnn="${4:-any}" + + echo "Waiting until node $pnn has status \"$status\"..." + + if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then + for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do + echo "$i" + $i || true + done + + return 1 + fi + +} + +# Useful for superficially testing IP failover. +# IPs must be on nodes matching nodeglob. +ips_are_on_nodeglob () +{ + local nodeglob="$1" ; shift + local ips="$*" + + local out + + all_ips_on_node 1 + + while read ip pnn ; do + for check in $ips ; do + if [ "$check" = "$ip" ] ; then + case "$pnn" in + ($nodeglob) : ;; + (*) return 1 ;; + esac + ips="${ips/${ip}}" # Remove from list + fi + done + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + ips="${ips// }" # Remove any spaces. + [ -z "$ips" ] +} + +wait_until_ips_are_on_nodeglob () +{ + echo "Waiting for IPs to fail over..." + + wait_until 60 ips_are_on_nodeglob "$@" +} + +node_has_some_ips () +{ + local node="$1" + + local out + + all_ips_on_node 1 + + while read ip pnn ; do + if [ "$node" = "$pnn" ] ; then + return 0 + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + return 1 +} + +wait_until_node_has_some_ips () +{ + echo "Waiting for node to have some IPs..." + + wait_until 60 node_has_some_ips "$@" +} + +get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" + out=$(netstat -tanp | + egrep "$pat" | + awk '{ print $4 }') + + [ -n "$out" ] +} + +wait_until_get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + echo "Waiting for ${prog} to establish connection to ${dst_socket}..." + + wait_until 5 get_src_socket "$@" +} + +####################################### + +# filename will be in $tcpdump_filename, pid in $tcpdump_pid +tcpdump_start () +{ + tcpdump_filter="$1" # global + + echo "Running tcpdump..." + tcpdump_filename=$(mktemp) + ctdb_test_exit_hook_add "rm -f $tcpdump_filename" + + # The only way of being sure that tcpdump is listening is to send + # some packets that it will see. So we use dummy pings - the -U + # option to tcpdump ensures that packets are flushed to the file + # as they are captured. + local dummy_addr="127.3.2.1" + local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" + tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + echo "Waiting for tcpdump output file to be ready..." + ping -q "$dummy_addr" >/dev/null 2>&1 & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + tcpdump_listen_for_dummy () + { + tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 + } + + wait_until 10 tcpdump_listen_for_dummy +} + +# By default, wait for 1 matching packet. +tcpdump_wait () +{ + local count="${1:-1}" + local filter="${2:-${tcpdump_filter}}" + + tcpdump_check () + { + local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) + [ $found -ge $count ] + } + + echo "Waiting for tcpdump to capture some packets..." + if ! wait_until 30 tcpdump_check ; then + echo "DEBUG AT $(date '+%F %T'):" + local i + for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + echo "$i" + $i || true + done + return 1 + fi +} + +tcpdump_show () +{ + local filter="${1:-${tcpdump_filter}}" + + tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null +} + +tcptickle_sniff_start () +{ + local src="$1" + local dst="$2" + + local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" + local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" + local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 + local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" + tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" + local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" + + tcpdump_start "$filter" +} + +tcptickle_sniff_wait_show () +{ + tcpdump_wait 1 "$tcptickle_reset" + + echo "GOOD: here are some TCP tickle packets:" + tcpdump_show +} + +gratarp_sniff_start () +{ + tcpdump_start "arp host ${test_ip}" +} + +gratarp_sniff_wait_show () +{ + tcpdump_wait 2 + + echo "GOOD: this should be the some gratuitous ARPs:" + tcpdump_show +} + + +####################################### + +daemons_stop () +{ + echo "Attempting to politely shutdown daemons..." + onnode 1 $CTDB shutdown -n all || true + + echo "Sleeping for a while..." + sleep_for 1 + + if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then + echo "Killing remaining daemons..." + pkill -f $CTDB_DIR/bin/ctdbd + + if pgrep -f $CTDB_DIR/bin/ctdbd >/dev/null ; then + echo "Once more with feeling.." + pkill -9 $CTDB_DIR/bin/ctdbd + fi + fi + + local var_dir=$CTDB_DIR/tests/var + rm -rf $var_dir/test.db +} + +daemons_setup () +{ + local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes + + local var_dir=$CTDB_DIR/tests/var + + mkdir -p $var_dir/test.db/persistent + + local public_addresses=$var_dir/public_addresses.txt + local no_public_addresses=$var_dir/no_public_addresses.txt + rm -f $CTDB_NODES $public_addresses $no_public_addresses + + # If there are (strictly) greater than 2 nodes then we'll randomly + # choose a node to have no public addresses. + local no_public_ips=-1 + [ $num_nodes -gt 2 ] && no_public_ips=$(($RANDOM % $num_nodes)) + echo "$no_public_ips" >$no_public_addresses + + local i + for i in $(seq 1 $num_nodes) ; do + if [ "${CTDB_USE_IPV6}x" != "x" ]; then + echo ::$i >> $nodes + ip addr add ::$i/128 dev lo + else + echo 127.0.0.$i >> $CTDB_NODES + # 2 public addresses on most nodes, just to make things interesting. + if [ $(($i - 1)) -ne $no_public_ips ] ; then + echo "192.0.2.$i/24 lo" >> $public_addresses + echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses + fi + fi + done +} + +daemons_start_1 () +{ + local pnn="$1" + shift # "$@" gets passed to ctdbd + + local var_dir=$CTDB_DIR/tests/var + + local public_addresses=$var_dir/public_addresses.txt + local no_public_addresses=$var_dir/no_public_addresses.txt + + local no_public_ips=-1 + [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses + + if [ "$no_public_ips" = $pnn ] ; then + echo "Node $no_public_ips will have no public IPs." + fi + + local ctdb_options="--reclock=$var_dir/rec.lock --nlist $CTDB_NODES --nopublicipcheck --event-script-dir=$CTDB_DIR/tests/events.d --logfile=$var_dir/daemons.log -d 3 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent --dbdir-state=$var_dir/test.db/state" + + if [ -z "$CTDB_TEST_REAL_CLUSTER" ]; then + ctdb_options="$ctdb_options --public-interface=lo" + fi + + if [ $pnn -eq $no_public_ips ] ; then + ctdb_options="$ctdb_options --public-addresses=/dev/null" + else + ctdb_options="$ctdb_options --public-addresses=$public_addresses" + fi + + # Need full path so we can use "pkill -f" to kill the daemons. + $VALGRIND $CTDB_DIR/bin/ctdbd --socket=$var_dir/sock.$pnn $ctdb_options "$@" ||return 1 +} + +daemons_start () +{ + # "$@" gets passed to ctdbd + + local num_nodes="${CTDB_TEST_NUM_DAEMONS:-2}" # default is 2 nodes + + echo "Starting $num_nodes ctdb daemons..." + + for i in $(seq 0 $(($num_nodes - 1))) ; do + daemons_start_1 $i "$@" + done + + local var_dir=$CTDB_DIR/tests/var + + if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then + ln -sf $var_dir/sock.0 /tmp/ctdb.socket || return 1 + fi +} + +####################################### + +_ctdb_hack_options () +{ + local ctdb_options="$*" + + # We really just want to pass CTDB_OPTIONS but on RH + # /etc/sysconfig/ctdb can, and frequently does, set that variable. + # So instead, we hack badly. We'll add these as we use them. + # Note that these may still be overridden by the above file... but + # we tend to use the exotic options here... so that is unlikely. + + case "$ctdb_options" in + *--start-as-stopped*) + export CTDB_START_AS_STOPPED="yes" + esac +} + +_restart_ctdb () +{ + _ctdb_hack_options "$@" + + if [ -e /etc/redhat-release ] ; then + service ctdb restart + else + /etc/init.d/ctdb restart + fi +} + +_ctdb_start () +{ + _ctdb_hack_options "$@" + + /etc/init.d/ctdb start +} + +setup_ctdb () +{ + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_setup + fi +} + +# Common things to do after starting one or more nodes. +_ctdb_start_post () +{ + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this blocks + # until the recovery is complete. Hopefully this will help the + # cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + + echo "ctdb is ready" +} + +# This assumes that ctdbd is not running on the given node. +ctdb_start_1 () +{ + local pnn="$1" + shift # "$@" is passed to ctdbd start. + + echo -n "Starting CTDB on node ${pnn}..." + + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_start_1 $pnn "$@" + else + onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@" + fi + + # If we're starting only 1 node then we're doing something weird. + ctdb_restart_when_done +} + +restart_ctdb () +{ + # "$@" is passed to ctdbd start. + + echo -n "Restarting CTDB" + if $ctdb_test_restart_scheduled ; then + echo -n " (scheduled)" + fi + echo "..." + + local i + for i in $(seq 1 5) ; do + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_stop + daemons_start "$@" + else + onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" + fi || { + echo "Restart failed. Trying again in a few seconds..." + sleep_for 5 + continue + } + + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { + echo "Cluster didn't become healthy. Restarting..." + continue + } + + local debug_out=$(onnode -p all ctdb status -Y 2>&1; onnode -p all ctdb scriptstatus 2>&1) + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this + # blocks until the recovery is complete. Hopefully this will + # help the cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + + # Cluster is still healthy. Good, we're done! + if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster become UNHEALTHY again. Restarting..." + continue + fi + + echo "Doing a sync..." + onnode -q 0 $CTDB sync + + echo "ctdb is ready" + return 0 + done + + echo "Cluster UNHEALTHY... too many attempts..." + echo "$debug_out" + # Try to make the calling test fail + status=1 + return 1 +} + +ctdb_restart_when_done () +{ + ctdb_test_restart_scheduled=true +} + +####################################### + +install_eventscript () +{ + local script_name="$1" + local script_contents="$2" + + if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + # The quoting here is *very* fragile. However, we do + # experience the joy of installing a short script using + # onnode, and without needing to know the IP addresses of the + # nodes. + onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" + else + f="${CTDB_DIR}/tests/events.d/${script_name}" + echo "$script_contents" >"$f" + chmod 755 "$f" + fi +} + +uninstall_eventscript () +{ + local script_name="$1" + + if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then + onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" + else + rm -vf "${CTDB_DIR}/tests/events.d/${script_name}" + fi +} + +####################################### + +# This section deals with the 99.ctdb_test eventscript. + +# Metafunctions: Handle a ctdb-test file on a node. +# given event. +ctdb_test_eventscript_file_create () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_remove () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_exists () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1 +} + + +# Handle a flag file on a node that is removed by 99.ctdb_test on the +# given event. +ctdb_test_eventscript_flag () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}" +} + + +# Handle a trigger that causes 99.ctdb_test to fail it's monitor +# event. +ctdb_test_eventscript_unhealthy_trigger () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger" +} + +# Handle the file that 99.ctdb_test created to show that it has marked +# a node unhealthy because it detected the above trigger. +ctdb_test_eventscript_unhealthy_detected () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" +} + +# Handle a trigger that causes 99.ctdb_test to timeout it's monitor +# event. This should cause the node to be banned. +ctdb_test_eventscript_timeout_trigger () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout" +} + +# Note that the eventscript can't use the above functions! +ctdb_test_eventscript_install () +{ + + local script='#!/bin/sh +out=$(ctdb pnn) +pnn="${out#PNN:}" + +rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" + +trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" +detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" +timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}" +case "$1" in + monitor) + if [ -e "$trigger" ] ; then + echo "${0}: Unhealthy because \"$trigger\" detected" + touch "$detected" + exit 1 + elif [ -e "$detected" -a ! -e "$trigger" ] ; then + echo "${0}: Healthy again, \"$trigger\" no longer detected" + rm "$detected" + fi + + ;; + *) + if [ -e "$timeout_trigger" ] ; then + echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected" + sleep 9999 + fi + ;; + *) + +esac + +exit 0 +' + install_eventscript "99.ctdb_test" "$script" +} + +ctdb_test_eventscript_uninstall () +{ + uninstall_eventscript "99.ctdb_test" +} + +# Note that this only works if you know all other monitor events will +# succeed. You also need to install the eventscript before using it. +wait_for_monitor_event () +{ + local pnn="$1" + + echo "Waiting for a monitor event on node ${pnn}..." + ctdb_test_eventscript_flag create $pnn "monitor" + + wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" + +} + +# Make sure that $CTDB is set. +: ${CTDB:=ctdb} diff --git a/ctdb/tests/scripts/test_wrap b/ctdb/tests/scripts/test_wrap index 6c730ef768..df12e4b731 100755 --- a/ctdb/tests/scripts/test_wrap +++ b/ctdb/tests/scripts/test_wrap @@ -1,7 +1,7 @@ #!/bin/bash # Execute the given command. The intention is that it is a function -# from ctdb_test_functions.bash. +# from "${TEST_SCRIPTS_DIR}/integration.bash". PATH="$(dirname $0):${PATH}" @@ -11,6 +11,6 @@ if [ ! $(which $f >/dev/null 2>&1) ] ; then [ -x "$d/$f" ] && PATH="$d:$PATH" fi -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" "$@" diff --git a/ctdb/tests/simple/00_ctdb_init.sh b/ctdb/tests/simple/00_ctdb_init.sh index 7b05b069c0..bd15fd7ed6 100755 --- a/ctdb/tests/simple/00_ctdb_init.sh +++ b/ctdb/tests/simple/00_ctdb_init.sh @@ -22,7 +22,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/00_ctdb_onnode.sh b/ctdb/tests/simple/00_ctdb_onnode.sh index 3d8221a58c..b17dfe0e76 100755 --- a/ctdb/tests/simple/00_ctdb_onnode.sh +++ b/ctdb/tests/simple/00_ctdb_onnode.sh @@ -18,7 +18,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/01_ctdb_version.sh b/ctdb/tests/simple/01_ctdb_version.sh index 50401d0ba3..5a36ea49eb 100755 --- a/ctdb/tests/simple/01_ctdb_version.sh +++ b/ctdb/tests/simple/01_ctdb_version.sh @@ -25,7 +25,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh index 13beaec639..2f709a8cc0 100755 --- a/ctdb/tests/simple/02_ctdb_listvars.sh +++ b/ctdb/tests/simple/02_ctdb_listvars.sh @@ -25,7 +25,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh index 748e329bed..a58aa3bf11 100755 --- a/ctdb/tests/simple/03_ctdb_getvar.sh +++ b/ctdb/tests/simple/03_ctdb_getvar.sh @@ -26,7 +26,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh index e8eda3abc4..5012e318cc 100755 --- a/ctdb/tests/simple/04_ctdb_setvar.sh +++ b/ctdb/tests/simple/04_ctdb_setvar.sh @@ -30,7 +30,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh index 7bcb414241..a84e4af980 100755 --- a/ctdb/tests/simple/05_ctdb_listnodes.sh +++ b/ctdb/tests/simple/05_ctdb_listnodes.sh @@ -22,7 +22,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh index 5df8e84593..7152ad4171 100755 --- a/ctdb/tests/simple/06_ctdb_getpid.sh +++ b/ctdb/tests/simple/06_ctdb_getpid.sh @@ -24,7 +24,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/07_ctdb_process_exists.sh b/ctdb/tests/simple/07_ctdb_process_exists.sh index 6222ee6d24..83205aa07f 100755 --- a/ctdb/tests/simple/07_ctdb_process_exists.sh +++ b/ctdb/tests/simple/07_ctdb_process_exists.sh @@ -28,7 +28,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh index 4f1e09689d..138f59c6e7 100755 --- a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh +++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh @@ -24,7 +24,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh index 50612a2512..ab6ba146df 100755 --- a/ctdb/tests/simple/09_ctdb_ping.sh +++ b/ctdb/tests/simple/09_ctdb_ping.sh @@ -27,7 +27,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh index 0d7dc5a342..2f39061acd 100755 --- a/ctdb/tests/simple/11_ctdb_ip.sh +++ b/ctdb/tests/simple/11_ctdb_ip.sh @@ -23,7 +23,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh index 76c3f1b875..4a4926d0c6 100755 --- a/ctdb/tests/simple/12_ctdb_getdebug.sh +++ b/ctdb/tests/simple/12_ctdb_getdebug.sh @@ -22,7 +22,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/13_ctdb_setdebug.sh b/ctdb/tests/simple/13_ctdb_setdebug.sh index b3a202d8bb..d1d1f22a8d 100755 --- a/ctdb/tests/simple/13_ctdb_setdebug.sh +++ b/ctdb/tests/simple/13_ctdb_setdebug.sh @@ -30,7 +30,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh index 76c520caf4..6a463ba012 100755 --- a/ctdb/tests/simple/14_ctdb_statistics.sh +++ b/ctdb/tests/simple/14_ctdb_statistics.sh @@ -25,7 +25,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh index 146807f294..eaa60d6920 100755 --- a/ctdb/tests/simple/15_ctdb_statisticsreset.sh +++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh @@ -24,7 +24,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh index 3c23ac1d9d..b770bd6a6a 100755 --- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh +++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh @@ -34,7 +34,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh index 0432946993..1ad9f33436 100755 --- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh +++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh @@ -30,7 +30,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh index b025dc7d3d..7c09e58503 100755 --- a/ctdb/tests/simple/23_ctdb_moveip.sh +++ b/ctdb/tests/simple/23_ctdb_moveip.sh @@ -29,7 +29,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh index f8377caac2..9bed590391 100755 --- a/ctdb/tests/simple/24_ctdb_getdbmap.sh +++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh @@ -23,7 +23,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh index 4a6892cb5d..4082da1d13 100755 --- a/ctdb/tests/simple/25_dumpmemory.sh +++ b/ctdb/tests/simple/25_dumpmemory.sh @@ -22,7 +22,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh index 78541d6e69..8b92c0ef50 100755 --- a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh +++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh @@ -32,7 +32,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/31_ctdb_disable.sh b/ctdb/tests/simple/31_ctdb_disable.sh index 8635816393..d0214549a4 100755 --- a/ctdb/tests/simple/31_ctdb_disable.sh +++ b/ctdb/tests/simple/31_ctdb_disable.sh @@ -29,7 +29,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/32_ctdb_enable.sh b/ctdb/tests/simple/32_ctdb_enable.sh index 7bfdc437d4..7cc3da3129 100755 --- a/ctdb/tests/simple/32_ctdb_enable.sh +++ b/ctdb/tests/simple/32_ctdb_enable.sh @@ -33,7 +33,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/41_ctdb_stop.sh b/ctdb/tests/simple/41_ctdb_stop.sh index 9e95eda421..8f4d77ba68 100755 --- a/ctdb/tests/simple/41_ctdb_stop.sh +++ b/ctdb/tests/simple/41_ctdb_stop.sh @@ -29,7 +29,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/42_ctdb_continue.sh b/ctdb/tests/simple/42_ctdb_continue.sh index 82e1534b63..e02a90d08e 100755 --- a/ctdb/tests/simple/42_ctdb_continue.sh +++ b/ctdb/tests/simple/42_ctdb_continue.sh @@ -31,7 +31,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/43_stop_recmaster_yield.sh b/ctdb/tests/simple/43_stop_recmaster_yield.sh index a47991e8be..e7a825002e 100755 --- a/ctdb/tests/simple/43_stop_recmaster_yield.sh +++ b/ctdb/tests/simple/43_stop_recmaster_yield.sh @@ -22,7 +22,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/51_ctdb_bench.sh b/ctdb/tests/simple/51_ctdb_bench.sh index e629c01e93..d4f7c542d1 100755 --- a/ctdb/tests/simple/51_ctdb_bench.sh +++ b/ctdb/tests/simple/51_ctdb_bench.sh @@ -26,7 +26,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/52_ctdb_fetch.sh b/ctdb/tests/simple/52_ctdb_fetch.sh index 5936419066..13b3f1fafd 100755 --- a/ctdb/tests/simple/52_ctdb_fetch.sh +++ b/ctdb/tests/simple/52_ctdb_fetch.sh @@ -26,7 +26,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/53_ctdb_transaction.sh b/ctdb/tests/simple/53_ctdb_transaction.sh index 760f78b510..b99b3a9fe4 100755 --- a/ctdb/tests/simple/53_ctdb_transaction.sh +++ b/ctdb/tests/simple/53_ctdb_transaction.sh @@ -22,7 +22,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/54_ctdb_transaction_recovery.sh b/ctdb/tests/simple/54_ctdb_transaction_recovery.sh index 13a92cccf4..baeea22ab1 100755 --- a/ctdb/tests/simple/54_ctdb_transaction_recovery.sh +++ b/ctdb/tests/simple/54_ctdb_transaction_recovery.sh @@ -41,7 +41,7 @@ recovery_loop_start() ctdb_test_exit_hook_add "kill $RECLOOP_PID >/dev/null 2>&1" } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh index 4b46cbbb3c..612366c310 100755 --- a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh +++ b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh @@ -40,7 +40,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/71_ctdb_wipedb.sh b/ctdb/tests/simple/71_ctdb_wipedb.sh index 3859661908..0cd07ccbca 100755 --- a/ctdb/tests/simple/71_ctdb_wipedb.sh +++ b/ctdb/tests/simple/71_ctdb_wipedb.sh @@ -24,7 +24,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/72_update_record_persistent.sh b/ctdb/tests/simple/72_update_record_persistent.sh index d232b4006f..5039492537 100755 --- a/ctdb/tests/simple/72_update_record_persistent.sh +++ b/ctdb/tests/simple/72_update_record_persistent.sh @@ -26,7 +26,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/73_tunable_NoIPTakeover.sh b/ctdb/tests/simple/73_tunable_NoIPTakeover.sh index 2f44ce4456..eee3da9db2 100755 --- a/ctdb/tests/simple/73_tunable_NoIPTakeover.sh +++ b/ctdb/tests/simple/73_tunable_NoIPTakeover.sh @@ -24,7 +24,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh index a2bb916635..db5b79275a 100755 --- a/ctdb/tests/simple/75_readonly_records_basic.sh +++ b/ctdb/tests/simple/75_readonly_records_basic.sh @@ -39,7 +39,7 @@ Expected results: EOF } -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/integration.bash" ctdb_test_init "$@" -- cgit