diff options
| -rwxr-xr-x | ctdb/tests/complex/11_ctdb_delip_removes_ip.sh | 81 | ||||
| -rw-r--r-- | ctdb/tests/scripts/integration.bash | 5 | ||||
| -rwxr-xr-x | ctdb/tests/simple/16_ctdb_config_add_ip.sh | 111 | ||||
| -rwxr-xr-x | ctdb/tests/simple/17_ctdb_config_delete_ip.sh | 59 | ||||
| -rwxr-xr-x | ctdb/tests/simple/23_ctdb_moveip.sh | 94 | ||||
| -rwxr-xr-x | ctdb/tests/simple/31_ctdb_disable.sh | 35 | ||||
| -rwxr-xr-x | ctdb/tests/simple/32_ctdb_enable.sh | 41 | ||||
| -rwxr-xr-x | ctdb/tests/simple/41_ctdb_stop.sh | 33 | ||||
| -rwxr-xr-x | ctdb/tests/simple/42_ctdb_continue.sh | 37 |
9 files changed, 85 insertions, 411 deletions
diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh index 043c345fac..d67cb070c8 100755 --- a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh +++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh @@ -5,36 +5,7 @@ test_info() cat <<EOF Verify that a node's public IP address can be deleted using 'ctdb deleteip'. -Check that the address is actually deleted from the interface. - -Prerequisites: - -* An active CTDB cluster with at least 2 active nodes. - -* Test must be run on a real or virtual cluster rather than against - local daemons. There is nothing intrinsic to this test that forces - this - it is because tests run against local daemons don't use the - regular eventscripts. Local daemons put public addresses on - loopback, so we can't reliably test when IPs have moved between - nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Use 'ctdb ip' on one of the nodes to list the IP addresses being - served. -3. Select an IP address being served by the node and check that it - actually appears on the interface it is supposed to be on. -4. Delete the IP address using 'ctdb delip'. -5. Verify that the deleted IP address is no longer listed using the - all_ips_on_node helper function. -6. Verify that the deleted IP address no longer appears on the - interface it was on. - -Expected results: - -* 'ctdb delip' removes an IP address from the list of public IP - addresses being served by a node and from the network interface. +This is an extended version of simple/17_ctdb_config_delete_ip.sh EOF } @@ -51,55 +22,23 @@ cluster_is_healthy # Reset configuration ctdb_restart_when_done -echo "Getting list of public IPs..." -all_ips_on_node -v 0 - -# Select an IP/node to remove. -num_ips=$(echo "$out" | wc -l) -num_to_remove=$(($RANDOM % $num_ips)) - -# Find the details in the list. -i=0 -while [ $i -le $num_to_remove ] ; do - read ip_to_remove test_node - i=$(($i + 1)) -done <<<"$out" - -echo "Determining interface for ${ip_to_remove} on ${test_node}." -try_command_on_node $test_node "ctdb ip -Y -v" -iface=$(echo "$out" | awk -F: -v ip=${ip_to_remove} -v pnn=${test_node} '$2 == ip && $3 == pnn { print $4 }') -echo "$iface" -[ -n "$iface" ] - -echo "Checking that node ${test_node} hosts ${ip_to_remove} on interface ${iface}..." -try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet[[:space:]]*${ip_to_remove}/'" - -echo "Attempting to remove ${ip_to_remove} from node ${test_node}." -try_command_on_node $test_node $CTDB delip $ip_to_remove - -echo "Sleeping..." -sleep_for 1 +select_test_node_and_ips +get_test_ip_mask_and_iface -test_node_ips="" -while read ip pnn ; do - [ "$pnn" = "$test_node" ] && \ - test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" -done <<<"$out" # bashism to avoid problem setting variable in pipeline. +echo "Checking that node ${test_node} hosts ${test_ip} on interface ${iface}..." +try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet[[:space:]]*${test_ip}/'" -if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then - echo "GOOD: That worked!" -else - echo "BAD: The remove IP address is still there!" - testfailures=1 -fi +echo "Attempting to remove ${test_ip} from node ${test_node}." +try_command_on_node $test_node $CTDB delip $test_ip +wait_until_ips_are_on_node '!' $test_node $test_ip timeout=60 increment=5 count=0 -echo "Waiting for ${ip_to_remove} to disappear from ${iface}..." +echo "Waiting for ${test_ip} to disappear from ${iface}..." while : ; do try_command_on_node -v $test_node "ip addr show dev $iface" - if echo "$out" | grep -E 'inet[[:space:]]*${ip_to_remove}/'; then + if echo "$out" | grep -E 'inet[[:space:]]*${test_ip}/'; then echo "Still there..." if [ $(($count * $increment)) -ge $timeout ] ; then echo "BAD: Timed out waiting..." diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash index 1ca77cabdb..1835949a4f 100644 --- a/ctdb/tests/scripts/integration.bash +++ b/ctdb/tests/scripts/integration.bash @@ -152,7 +152,8 @@ sanity_check_ips () prev="$ipp" done <<<"$ips" - echo "BAD: a node was -1 or IPs are only assigned to one node" + echo "BAD: a node was -1 or IPs are only assigned to one node:" + echo "$ips" echo "Are you running an old version of CTDB?" return 1 } @@ -483,7 +484,7 @@ node_has_some_ips () wait_until_node_has_some_ips () { - echo "Waiting for node to have some IPs..." + echo "Waiting for some IPs to be assigned to node ${test_node}" wait_until 60 node_has_some_ips "$@" } diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh index d4301c86ee..b5d76ea7ad 100755 --- a/ctdb/tests/simple/16_ctdb_config_add_ip.sh +++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh @@ -5,32 +5,8 @@ test_info() cat <<EOF Verify that an IP address can be added to a node using 'ctdb addip'. -This test goes to some trouble to figure out which IP address to add -but assumes a 24-bit subnet mask. It does not handle IPv6. It does -not do any network level checks that the new IP address is reachable -but simply trusts 'ctdb ip' that the address has been added. There is -also an extra prerequisite that the node being added to already has -public addresses - this is difficult to avoid if the extra address is -to be sensibly chosen. - -Prerequisites: - -* An active CTDB cluster with at least 2 active nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Use 'ctdb ip' on one of the nodes to list the IP addresses being - served. -3. Add an additional public address to be served by the node, using - 'ctdb addip'. -4. Verify that this IP address has been added to the list of IP - addresses being served by the node, using the 'ctdb ip' command. - -Expected results: - -* 'ctdb ip' adds an IP address to the list of public IP addresses - being served by a node. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". EOF } @@ -45,79 +21,16 @@ cluster_is_healthy # Reset configuration ctdb_restart_when_done -echo "Getting list of public IPs..." -all_ips_on_node 0 - -# When selecting test_node we just want a node that has public IPs. -# This will work and is economically semi-randomly. :-) -read x test_node <<<"$out" - -test_node_ips="" -all_ips="" -while read ip pnn ; do - all_ips="${all_ips}${all_ips:+ }${ip}" - [ "$pnn" = "$test_node" ] && \ - test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" -done <<<"$out" - -echo "Selected node ${test_node} with IPs: $test_node_ips" - -# Try to find a free IP adddress. This is inefficient but should -# succeed quickly. -if [ -z "$TEST_LOCAL_DAEMONS" ] ; then - try_command_on_node $test_node "ip addr show" - all_test_node_ips=$(echo "$out" | sed -rn -e 's@^[[:space:]]+inet[[:space:]]+([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+/[[:digit:]]+).*[[:space:]]([^[:space:]]+)+$@\1:\2@p') -else - all_test_node_ips="" -fi -add_ip="" - -# Use an IP already on one of the nodes, remove the last octet and -# loop through the possible IP addreses. -for i in $test_node_ips ; do - prefix="${i%.*}" - for j in $(seq 101 199) ; do - try="${prefix}.${j}" - # Try to make sure it isn't used anywhere! - - # First, make sure it isn't an existing public address on the - # cluster. - for k in $all_ips ; do - [ "$try" = "$k" ] && continue 2 - done - - # Also make sure it isn't some other address in use on the - # node. - for k in $all_test_node_ips ; do - [ "$try" = "${k%/*}" ] && continue 2 - done - - # Get the interface details for $i, which our address is a - # close relative of. This should never fail but it can't hurt - # to be careful... - try_command_on_node $test_node "ctdb ip -v -Y" - while IFS=":" read x ip pnn iface x ; do - if [ "$i" = "$ip" ]; then - add_ip="$try/32:$iface" - break 3 - fi - done <<<"$out" - done -done +select_test_node_and_ips +get_test_ip_mask_and_iface -if [ -z "$add_ip" ] ; then - echo "BAD: Unable to find IP address to add." - exit 1 -fi +echo "Deleting IP $test_ip from all nodes" +try_command_on_node $test_node $CTDB delip -n all $test_ip +wait_until_ips_are_on_node '!' $test_node $test_ip -echo "Adding IP: ${add_ip/:/ on interface }" -try_command_on_node $test_node $CTDB addip ${add_ip/:/ } +# Debugging... +try_command_on_node -v all $CTDB ip -echo "Waiting for IP to be added..." -if wait_until 60 ips_are_on_node $test_node ${add_ip%/*} ; then - echo "That worked!" -else - echo "BAD: IP didn't get added." - try_command_on_node $test_node $CTDB ip -n all - exit 1 -fi +echo "Adding IP ${test_ip}/${mask} on ${iface}, node ${test_node}" +try_command_on_node $test_node $CTDB addip ${test_ip}/${mask} $iface +wait_until_ips_are_on_node $test_node $test_ip diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh index 1ad9f33436..80d2699429 100755 --- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh +++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh @@ -5,28 +5,8 @@ test_info() cat <<EOF Verify that a node's public IP address can be deleted using 'ctdb deleteip'. -This test does not do any network level checks that the IP address is -no longer reachable but simply trusts 'ctdb ip' that the address has -been deleted. - -Prerequisites: - -* An active CTDB cluster with at least 2 active nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Use 'ctdb ip' on one of the nodes to list the IP addresses being - served. -3. Delete one public IP address being be served by the node, using - 'ctdb delip'. -4. Verify that the delete IP address is no longer listed using the - all_ips_on_node helper function. - -Expected results: - -* 'ctdb delip' removes an IP address from the list of public IP - addresses being served by a node. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". EOF } @@ -41,35 +21,8 @@ cluster_is_healthy # Reset configuration ctdb_restart_when_done -echo "Getting list of public IPs..." -all_ips_on_node -v 0 - -# Select an IP/node to remove. -num_ips=$(echo "$out" | wc -l) -num_to_remove=$(($RANDOM % $num_ips)) - -# Find the details in the list. -i=0 -while [ $i -le $num_to_remove ] ; do - read ip_to_remove test_node - i=$(($i + 1)) -done <<<"$out" - -echo "Attempting to remove ${ip_to_remove} from node ${test_node}." -try_command_on_node $test_node $CTDB delip $ip_to_remove - -echo "Sleeping..." -sleep_for 1 - -test_node_ips="" -while read ip pnn ; do - [ "$pnn" = "$test_node" ] && \ - test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" -done <<<"$out" # bashism to avoid problem setting variable in pipeline. +select_test_node_and_ips -if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then - echo "GOOD: That worked!" -else - echo "BAD: The remove IP address is still there!" - testfailures=1 -fi +echo "Deleting IP ${test_ip} from node ${test_node}" +try_command_on_node $test_node $CTDB delip $test_ip +wait_until_ips_are_on_node '!' $test_node $test_ip diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh index 63ededfb74..f6e90272b2 100755 --- a/ctdb/tests/simple/23_ctdb_moveip.sh +++ b/ctdb/tests/simple/23_ctdb_moveip.sh @@ -5,27 +5,10 @@ test_info() cat <<EOF Verify that 'ctdb moveip' allows movement of public IPs between cluster nodes. -To work, this test unsets DeterministicIPs and sets NoIPFailback. - -This test does not do any network level checks that the IP address is -no longer reachable but simply trusts 'ctdb ip' that the address has -been deleted. - -Prerequisites: - -* An active CTDB cluster with at least 2 active nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Use 'ctdb ip' on one of the nodes to list the IP addresses being - served. -3. Use 'ctdb moveip' to move an address from one node to another. -4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". -Expected results: - -* 'ctdb moveip' allows an IP address to be moved between cluster nodes. +To work, this test unsets DeterministicIPs and sets NoIPFailback. EOF } @@ -40,62 +23,39 @@ cluster_is_healthy # Reset configuration ctdb_restart_when_done -try_command_on_node 0 "$CTDB listnodes | wc -l" -num_nodes="$out" -echo "There are $num_nodes nodes..." - -if [ $num_nodes -lt 2 ] ; then - echo "Less than 2 nodes!" - exit 1 -fi - -echo "Getting list of public IPs..." -all_ips_on_node -v 0 +select_test_node_and_ips sanity_check_ips "$out" -# Select an IP/node to move. -num_ips=$(echo "$out" | wc -l) -num_to_move=$(($RANDOM % $num_ips)) - -# Find the details in the list. -i=0 -while [ $i -le $num_to_move ] ; do - read ip_to_move test_node - i=$(($i + 1)) -done <<<"$out" - -# Can only move address to a node that is willing to host $ip_to_move. -# This inefficient but shouldn't take long or get stuck. -to_node=$test_node -while [ $test_node -eq $to_node ] ; do - n=$(($RANDOM % $num_ips)) - i=0 - while [ $i -le $n ] ; do - read x to_node - i=$(($i + 1)) +# Find a target node - it must be willing to host $test_ip +try_command_on_node any "$CTDB listnodes | wc -l" +num_nodes="$out" +to_node="" +for i in $(seq 0 $(($num_nodes - 1)) ) ; do + [ $i -ne $test_node ] || continue + all_ips_on_node $i + while read ip x ; do + if [ "$ip" = "$test_ip" ] ; then + to_node="$i" + break 2 + fi done <<<"$out" done +if [ -z "$to_node" ] ; then + echo "Unable to find target node" + exit 1 +fi + +echo "Target node is ${to_node}" + echo "Turning off DeterministicIPs..." try_command_on_node 0 $CTDB setvar DeterministicIPs 0 -n all echo "Turning on NoIPFailback..." try_command_on_node 0 $CTDB setvar NoIPFailback 1 -n all -echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}." -try_command_on_node $test_node $CTDB moveip $ip_to_move $to_node - -if wait_until_ips_are_on_node '!' $test_node $ip_to_move ; then - echo "IP moved from ${test_node}." -else - echo "BAD: IP didn't move from ${test_node}." - exit 1 -fi - -if wait_until_ips_are_on_node "$to_node" $ip_to_move ; then - echo "IP moved to ${to_node}." -else - echo "BAD: IP didn't move to ${to_node}." - exit 1 -fi +echo "Attempting to move ${test_ip} from node ${test_node} to node ${to_node}" +try_command_on_node $test_node $CTDB moveip $test_ip $to_node +wait_until_ips_are_on_node '!' $test_node $test_ip +wait_until_ips_are_on_node $to_node $test_ip diff --git a/ctdb/tests/simple/31_ctdb_disable.sh b/ctdb/tests/simple/31_ctdb_disable.sh index 4331d26ce8..e3bb5d6104 100755 --- a/ctdb/tests/simple/31_ctdb_disable.sh +++ b/ctdb/tests/simple/31_ctdb_disable.sh @@ -5,27 +5,13 @@ test_info() cat <<EOF Verify the operation of 'ctdb disable'. -This is a superficial test of the 'ctdb disable' command. It trusts -information from CTDB that indicates that the IP failover has happened -correctly. Another test should check that the failover has actually -happened at the networking level. +* Verify that the status of the node changes to 'disabled'. -Prerequisites: +* Verify that the IP addreses served by the disabled node are failed + over to other nodes. -* An active CTDB cluster with at least 2 active nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Disable one of the nodes using 'ctdb disable -n <node>'. -3. Verify that the status of the node changes to 'disabled'. -4. Verify that the IP addreses served by the disabled node are failed - over to other nodes. - -Expected results: - -* The status of the disabled node changes as expected and IP addresses - failover as expected. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". EOF } @@ -43,15 +29,6 @@ ctdb_restart_when_done select_test_node_and_ips echo "Disabling node $test_node" - try_command_on_node 1 $CTDB disable -n $test_node - -# Avoid a potential race condition... wait_until_node_has_status $test_node disabled - -if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then - echo "All IPs moved." -else - echo "Some IPs didn't move." - testfailures=1 -fi +wait_until_ips_are_on_node '!' $test_node $test_node_ips diff --git a/ctdb/tests/simple/32_ctdb_enable.sh b/ctdb/tests/simple/32_ctdb_enable.sh index bf60d81da2..dddf881331 100755 --- a/ctdb/tests/simple/32_ctdb_enable.sh +++ b/ctdb/tests/simple/32_ctdb_enable.sh @@ -5,31 +5,15 @@ test_info() cat <<EOF Verify the operation of 'ctdb enable'. -This is a superficial test of the 'ctdb enable' command. It trusts -information from CTDB that indicates that the IP failover has happened -correctly. Another test should check that the failover has actually -happened at the networking level. +After disabling a node... -Prerequisites: +* Verify that the status of a re-enabled node changes back to 'OK'. -* An active CTDB cluster with at least 2 active nodes. +* Verify that some public IP addreses are rebalanced to a re-enabled + node. -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Disable one of the nodes using 'ctdb disable -n <node>'. -3. Verify that the status of the node changes to 'disabled'. -4. Verify that the public IP addreses served by the disabled node are - failed over to other nodes. -5. Enable the disabled node using 'ctdb enable -n '<node>'. -6. Verify that the status changes back to 'OK'. -7. Verify that some public IP addreses are failed back to the node. - - -Expected results: - -* The status of a re-enabled node changes as expected and IP addresses - fail back as expected. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". EOF } @@ -47,19 +31,12 @@ select_test_node_and_ips echo "Disabling node $test_node" try_command_on_node 1 $CTDB disable -n $test_node - wait_until_node_has_status $test_node disabled -if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then - echo "All IPs moved." -else - echo "Some IPs didn't move." - testfailures=1 -fi +echo "Waiting for IPs to no longer be hosted on node ${test_node}" +wait_until_ips_are_on_node '!' $test_node $test_node_ips -echo "Reenabling node $test_node" +echo "Re-enabling node $test_node" try_command_on_node 1 $CTDB enable -n $test_node - wait_until_node_has_status $test_node enabled - wait_until_node_has_some_ips "$test_node" diff --git a/ctdb/tests/simple/41_ctdb_stop.sh b/ctdb/tests/simple/41_ctdb_stop.sh index dc1a5c3ac9..e11fb0fc7e 100755 --- a/ctdb/tests/simple/41_ctdb_stop.sh +++ b/ctdb/tests/simple/41_ctdb_stop.sh @@ -5,27 +5,13 @@ test_info() cat <<EOF Verify the operation of the 'ctdb stop' command. -This is a superficial test of the 'ctdb stop' command. It trusts -information from CTDB that indicates that the IP failover has -happened correctly. Another test should check that the failover -has actually happened at the networking level. +* Verify that the status of the node changes to 'stopped'. -Prerequisites: +* Verify that the public IP addresses that were being served by + the node are failed over to one of the other nodes. -* An active CTDB cluster with at least 2 active nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Stop one of the nodes using the 'ctdb stop' command. -3. Verify that the status of the node changes to 'stopped'. -4. Verify that the public IP addresses that were being served by - the node are failed over to one of the other nodes. - -Expected results: - -* The status of the stopped nodes changes as expected and IP addresses - failover as expected. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". EOF } @@ -44,12 +30,5 @@ select_test_node_and_ips echo "Stopping node ${test_node}..." try_command_on_node 1 $CTDB stop -n $test_node - wait_until_node_has_status $test_node stopped - -if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then - echo "All IPs moved." -else - echo "Some IPs didn't move." - testfailures=1 -fi +wait_until_ips_are_on_node '!' $test_node $test_node_ips diff --git a/ctdb/tests/simple/42_ctdb_continue.sh b/ctdb/tests/simple/42_ctdb_continue.sh index f8f301d596..b75759fce4 100755 --- a/ctdb/tests/simple/42_ctdb_continue.sh +++ b/ctdb/tests/simple/42_ctdb_continue.sh @@ -5,29 +5,13 @@ test_info() cat <<EOF Verify the operation of the 'ctdb continue' command. -This is a superficial test of the 'ctdb continue' command. It trusts -information from CTDB that indicates that the IP failover and failback -has happened correctly. Another test should check that the failover -and failback has actually happened at the networking level. +After stopping a node... -Prerequisites: +* Verify that the status of the node changes back to 'OK' and that + some public IP addresses move back to the node. -* An active CTDB cluster with at least 2 active nodes. - -Steps: - -1. Verify that the status on all of the ctdb nodes is 'OK'. -2. Stop one of the nodes using the 'ctdb stop' command. -3. Verify that the status of the node changes to 'stopped'. -4. Verify that the public IP addresses that were being served by - the node are failed over to one of the other nodes. -5. Use 'ctdb continue' to bring the node back online. -6. Verify that the status of the node changes back to 'OK' and that - some public IP addresses move back to the node. - -Expected results: - -* The 'ctdb continue' command successfully brings a stopped node online. +This test does not do any network level checks to make sure IP +addresses are actually on interfaces. It just consults "ctdb ip". EOF } @@ -46,19 +30,10 @@ select_test_node_and_ips echo "Stopping node ${test_node}..." try_command_on_node 1 $CTDB stop -n $test_node - wait_until_node_has_status $test_node stopped - -if wait_until_ips_are_on_node '!' $test_node $test_node_ips ; then - echo "All IPs moved." -else - echo "Some IPs didn't move." - testfailures=1 -fi +wait_until_ips_are_on_node '!' $test_node $test_node_ips echo "Continuing node $test_node" try_command_on_node 1 $CTDB continue -n $test_node - wait_until_node_has_status $test_node notstopped - wait_until_node_has_some_ips "$test_node" |