3 new tests. 24_ctdb_getdbmap.sh is only 1/2 implemented but does

something vaguely useful.  ctdb_test_exit unsets $ctdb_test_exit_hook.
Fix bug in 17_ctdb_config_delete_ip.sh.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit f90f6e19952d58b8590fe550ecf2308bd9b065dc)

6 files changed, 273 insertions, 3 deletions

diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash
index 60b8e9de77..b086b7fb5c 100644
--- a/ctdb/tests/scripts/ctdb_test_functions.bash
+++ b/ctdb/tests/scripts/ctdb_test_functions.bash
@@ -64,6 +64,7 @@ ctdb_test_exit ()
     [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status
 
     eval "$ctdb_test_exit_hook"
+    unset ctdb_test_exit_hook
 
     if ! onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy ; then
 	echo "Restarting ctdb on all nodes to get back into known state..."
diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh
index b94aad6ea5..b12a296490 100755
--- a/ctdb/tests/simple/06_ctdb_getpid.sh
+++ b/ctdb/tests/simple/06_ctdb_getpid.sh
@@ -35,9 +35,7 @@ onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
 # This is an attempt at being independent of the number of nodes
 # reported by "ctdb getpid -n all".
 try_command_on_node 0 "ctdb listnodes | wc -l"
-
 num_nodes="$out"
-
 echo "There are $num_nodes nodes..."
 
 # Call getpid a few different ways and make sure the answer is always the same.
diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
index ac47f63189..56586492ac 100755
--- a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
+++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh
@@ -47,7 +47,7 @@ num_to_remove=$(($RANDOM % $num_ips))
 
 # Find the details in the list.
 i=0
-while [ $i -lt $num_to_remove ] ; do
+while [ $i -le $num_to_remove ] ; do
     read ip_to_remove test_node
     i=$(($i + 1))
 done <<<"$out"
diff --git a/ctdb/tests/simple/22_ctdb_enablemonitor.sh b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
new file mode 100755
index 0000000000..995a8ae2dc
--- /dev/null
+++ b/ctdb/tests/simple/22_ctdb_enablemonitor.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that 'ctdb enablemonitor' works correctly.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+* 00_ctdb_install_eventscript.sh successfully installed its event
+  script.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Disable monitoring on a node using 'ctdb disablemonitor -n <node>.
+3. Create a file called /tmp/ctdb-test-unhealthy-trigger.<node> on the
+   node.
+4. Verify that the status of the node does not change to unhealthy
+   within the interval indicated by the MonitorInterval variable,
+   since monitoring is disabled.
+5. Now enable monitoring on the node using 'ctdb enablemonitor -n <node>.
+6. Verify that the status of the node changes to unhealthy within the
+   interval indicated by the MonitorInterval variable.
+7. Check that the file /tmp/ctdb-test-unhealthy-detected.<node> is
+   created, indicating that the event script is the reason the node
+   has been marked as unhealthy.
+
+Expected results:
+
+* When monitoring is enabled on a node, event scripts are executed and
+  status changes are monitored.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+test_node=1
+
+# We need this for later, so we know how long to sleep.
+try_command_on_node -v 0 ctdb getvar MonitorInterval -n $test_node
+monitor_interval="${out#*= }"
+echo "Monitor interval on node $test_node is $monitor_interval seconds."
+
+try_command_on_node -v 0 ctdb disablemonitor -n $test_node
+
+sanity_check_output \
+    1 \
+    '^Monitoring mode:DISABLED$' \
+    "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monoff
+
+trigger="/tmp/ctdb-test-unhealthy-trigger.${test_node}"
+detected="/tmp/ctdb-test-unhealthy-detected.${test_node}"
+
+recovered_flag="/tmp/ctdb-test-flag.recovered.${test_node}"
+try_command_on_node $test_node touch "$recovered_flag"
+
+ctdb_test_exit_hook="onnode $test_node rm -vf $trigger; restart_ctdb"
+
+echo "Creating trigger file on node $test_node to see if it goes unhealthy..."
+try_command_on_node $test_node touch "$trigger"
+
+sleep_for $monitor_interval
+
+try_command_on_node 0 $CTDB_TEST_WRAPPER node_has_status $test_node healthy
+
+try_command_on_node $test_node test ! -e "$detected"
+
+echo "OK: flag file was not created so monitoring must be disabled."
+
+try_command_on_node -v 0 ctdb enablemonitor -n $test_node
+
+sanity_check_output \
+    1 \
+    '^Monitoring mode:ACTIVE$' \
+    "$out"
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node monon
+
+sleep_for $monitor_interval
+
+try_command_on_node $test_node test -e "$detected"
+
+echo "OK: flag file was created so monitoring must be enabled."
+
+onnode 0 $CTDB_TEST_WRAPPER wait_until_node_has_status $test_node unhealthy $monitor_interval
+
+try_command_on_node -v $test_node ls -l "$detected"
+
+echo "OK, that all worked.  Expect a restart..."
+
+ctdb_test_exit
diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh
new file mode 100755
index 0000000000..cb775be364
--- /dev/null
+++ b/ctdb/tests/simple/23_ctdb_moveip.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that  'ctdb moveip' allows movement of public IPs between cluster nodes.
+
+To work, this test unsets DeterministicIPs and sets NoIPFailback.
+
+This test does not do any network level checks that the IP address is
+no longer reachable but simply trusts 'ctdb ip' that the address has
+been deleted.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Use 'ctdb ip' on one of the nodes to list the IP addresses being
+   served.
+3. Use 'ctdb moveip' to move an address from one node to another.
+4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node.
+
+Expected results:
+
+* 'ctdb moveip' allows an IP address to be moved between cluster nodes.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# Restart when done since things are likely to be broken.
+ctdb_test_exit_hook="restart_ctdb"
+
+try_command_on_node 0 "ctdb listnodes | wc -l"
+num_nodes="$out"
+echo "There are $num_nodes nodes..."
+
+if [ $num_nodes -lt 2 ] ; then
+    echo "Less than 2 nodes!"
+    exit 1
+fi
+
+echo "Getting list of public IPs..."
+try_command_on_node -v 0 'ctdb ip -n all | sed -e "1d"'
+
+# Select an IP/node to move.
+num_ips=$(echo "$out" | wc -l)
+num_to_move=$(($RANDOM % $num_ips))
+
+# Find the details in the list.
+i=0
+while [ $i -le $num_to_move ] ; do
+    read ip_to_move test_node
+    i=$(($i + 1))
+done <<<"$out"
+
+# Can only move address to a node that is willing to host $ip_to_move.
+# This inefficient but shouldn't take long or get stuck.
+to_node=$test_node
+while [ $test_node -eq $to_node ] ; do
+    n=$(($RANDOM % $num_ips))
+    i=0
+    while [ $i -le $n ] ; do
+	read x to_node
+	i=$(($i + 1))
+    done <<<"$out"
+done
+
+echo "Turning off DeterministicIPs..."
+try_command_on_node 0 ctdb setvar DeterministicIPs 0 -n all
+
+echo "Turning on NoIPFailback..."
+try_command_on_node 0 ctdb setvar NoIPFailback 1 -n all
+
+echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}."
+try_command_on_node $test_node ctdb moveip $ip_to_move $to_node
+
+if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ip_to_move ; then
+    echo "IP moved from ${test_node}."
+else
+    echo "BAD: IP didn't move from ${test_node}."
+    exit 1
+fi
+
+if wait_until_ips_are_on_nodeglob "$to_node" $ip_to_move ; then
+    echo "IP moved to ${to_node}."
+else
+    echo "BAD: IP didn't move to ${to_node}."
+    exit 1
+fi
+
+echo "OK, that worked... expect a restart..."
+
+ctdb_test_exit
diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh
new file mode 100755
index 0000000000..5734932ec5
--- /dev/null
+++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+test_info()
+{
+    cat <<EOF
+Verify that  'ctdb getdbmap' operates as expected.
+
+This test creates some test databases using 'ctdb attach'.
+
+Prerequisites:
+
+* An active CTDB cluster with at least 2 active nodes.
+
+Steps:
+
+1. Verify that the status on all of the ctdb nodes is 'OK'.
+2. Get the database on using 'ctdb getdbmap'.
+3. Verify that the output is valid.
+
+Expected results:
+
+* 'ctdb getdbmap' shows a valid listing of databases.
+EOF
+}
+
+. ctdb_test_functions.bash
+
+ctdb_test_init "$@"
+
+set -e
+
+onnode 0 $CTDB_TEST_WRAPPER cluster_is_healthy
+
+# Restart when done since things are likely to be broken.
+ctdb_test_exit_hook="restart_ctdb"
+
+echo "OK, that worked... expect a restart..."
+
+make_temp_db_filename ()
+{
+    dd if=/dev/urandom count=1 bs=512 2>/dev/null |
+    md5sum |
+    awk '{printf "%s.tdb\n", $1}'
+}
+
+try_command_on_node -v 0 "ctdb getdbmap"
+num_db_init=$(echo "$out" | sed -n -e '1s/.*://p')
+
+for i in $(seq 1 5) ; do
+    f=$(make_temp_db_filename)
+    echo "Attempting to create test database \"$f\"..."
+    try_command_on_node 0 ctdb attach "$f"
+    try_command_on_node 0 ctdb getdbmap
+    num=$(echo "$out" | sed -n -e '1s/^.*://p')
+    if [ $num = $(($num_db_init + $i)) ] ; then
+	echo "OK: seem to have the right number of databases"
+    else
+	echo "BAD: we didn't get an additional database"
+	exit 1
+    fi
+
+    # Add check to make sure we have the database we actually added!
+    # Probably do a sanity check on the overall output of getdbmap
+    # each time too.
+done
+
+ctdb_test_exit