diff options
| author | Stefan Metzmacher <metze@samba.org> | 2013-11-13 14:17:32 +0100 |
|---|---|---|
| committer | Michael Adam <obnox@samba.org> | 2013-11-13 14:18:52 +0100 |
| commit | f087a8e2b81fae82fa571ef09d2d1cb682cc8ff8 (patch) | |
| tree | d4a6b14a1594958150b733180d995b5c32025017 /ctdb/tests | |
| parent | e80a5aba3db8e81173fa443991e08ef4a300ea5c (diff) | |
| parent | 25f3c8b5269863aadbe72e304da7012782ef5b25 (diff) | |
| download | samba-f087a8e2b81fae82fa571ef09d2d1cb682cc8ff8.tar.gz samba-f087a8e2b81fae82fa571ef09d2d1cb682cc8ff8.tar.xz samba-f087a8e2b81fae82fa571ef09d2d1cb682cc8ff8.zip | |
Merge branch 'master' of ctdb into 'master' of samba
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Signed-off-by: Michael Adam <obnox@samba.org>
Diffstat (limited to 'ctdb/tests')
337 files changed, 21548 insertions, 0 deletions
diff --git a/ctdb/tests/INSTALL b/ctdb/tests/INSTALL new file mode 100755 index 00000000000..5581989619b --- /dev/null +++ b/ctdb/tests/INSTALL @@ -0,0 +1,91 @@ +#!/bin/sh + +# Script to install the CTDB testsuite on a machine. + +usage () +{ + if [ -n "$1" ] ; then + echo "$1" + echo + fi + + cat <<EOF + $0 --destdir=<DIR1> \\ + --datarootdir=<DIR2> \\ + --libdir=<DIR3> \\ + --bindir=<DIR4> \\ + --etcdir=<DIR5> +EOF + exit 1 +} + +parse_options () +{ + temp=$(getopt -n "$prog" -o "h" -l help,destdir:,datarootdir:,libdir:,bindir:,etcdir: -- "$@") + + [ $? != 0 ] && usage + + eval set -- "$temp" + + destdir="" + datarootdir="" + libdir="" + bindir="" + etcdir="" + + while true ; do + case "$1" in + --destdir) destdir="$2" ; shift 2 ;; + --datarootdir) datarootdir="$2" ; shift 2 ;; + --libdir) libdir="$2" ; shift 2 ;; + --bindir) bindir="$2" ; shift 2 ;; + --etcdir) etcdir="$2" ; shift 2 ;; + --) shift ; break ;; + -h|--help|*) usage ;; # Shouldn't happen, so this is reasonable. + esac + done + + [ $# -gt 0 ] && usage + + [ -n "$destdir" ] || usage "No option --destdir specified" + [ -n "$datarootdir" ] || usage "No option --datarootdir specified" + [ -n "$libdir" ] || usage "No option --libdir specified" + [ -n "$bindir" ] || usage "No option --bindir specified" + [ -n "$etcdir" ] || usage "No option --etcdir specified" +} + +parse_options "$@" + +# Make things neater! +if [ "$destdir" = "/" ] ; then + destdir="" +fi + +data_subdirs="complex events.d eventscripts onnode scripts simple takeover tool" + +ctdb_datadir="${destdir}${datarootdir}/ctdb-tests" +echo "Installing test data files into ${ctdb_datadir}..." +for d in $data_subdirs ; do + mkdir -p "${ctdb_datadir}/${d}" + cp -pr "tests/${d}" "${ctdb_datadir}" +done +# Some of the unit tests have relative symlinks back to in-tree bits +# and pieces. These links will be broken! +for i in "events.d" "functions" "nfs-rpc-checks.d" ; do + ln -sf "${etcdir}/ctdb/${i}" "${ctdb_datadir}/eventscripts/etc-ctdb/${i}" +done +# test_wrap needs to set TEST_BIN_DIR +sed -i -e "s@^TEST_SCRIPTS_DIR=.*@&\nexport TEST_BIN_DIR=\"${libdir}/ctdb-tests\"@" "${ctdb_datadir}/scripts/test_wrap" + +ctdb_libdir="${destdir}${libdir}/ctdb-tests" +mkdir -p "${destdir}${libdir}" +echo "Installing test binary files into ${ctdb_libdir}..." +cp -pr "tests/bin/" "${ctdb_libdir}" + +ctdb_bindir="${destdir}${bindir}" +echo "Installing wrapper scripts into ${ctdb_bindir}..." +mkdir -p "${ctdb_bindir}" +out="${ctdb_bindir}/ctdb_run_tests" +sed -e "s@^test_dir=.*@test_dir=${datarootdir}/ctdb-tests\nexport TEST_BIN_DIR=\"${libdir}/ctdb-tests\"@" "tests/run_tests.sh" >"$out" +chmod 755 "$out" +ln -s "ctdb_run_tests" "${ctdb_bindir}/ctdb_run_cluster_tests" diff --git a/ctdb/tests/README b/ctdb/tests/README new file mode 100644 index 00000000000..1c9983b0922 --- /dev/null +++ b/ctdb/tests/README @@ -0,0 +1,104 @@ +Introduction +------------ + +For a developer, the simplest way of running most tests on a local +machine from within the git repository is: + + make test + +This runs all unit tests (onnode, takeover, tool, eventscripts) and +the tests against local daemons (simple) using the script +tests/run_tests.sh. + +When running tests against a real or virtual cluster the script +tests/run_cluster_tests.sh can be used. This runs all integration +tests (simple, complex). + +Both of these scripts can also take a list of tests to run. You can +also pass options, which are then passed to run_tests. However, if +you just try to pass options to run_tests then you lose the default +list of tests that are run. You can't have everything... + +scripts/run_tests +----------------- + +The above scripts invoke tests/scripts/run_tests. This script has a +lot of command-line switches. Some of the more useful options +include: + + -s Print a summary of tests results after running all tests + + -l Use local daemons for integration tests + + This allows the tests in "simple" to be run against local + daemons. + + All integration tests communicate with cluster nodes using + onnode or the ctdb tool, which both have some test hooks to + support local daemons. + + By default 3 daemons are used. If you want to use a different + number of daemons then do not use this option but set + TEST_LOCAL_DAEMONS to the desired number of daemons instead. + The -l option just sets TEST_LOCAL_DAEMONS to 3... :-) + + -e Exit on the first test failure + + -C Clean up - kill daemons and remove $TEST_VAR_DIR when done + + Tests uses a temporary/var directory for test state. By default, + this directory is not removed when tests are complete, so you + can do forensics or, for integration tests, re-run tests that + have failed against the same directory (with the same local + daemons setup). So this option cleans things up. + + Also kills local daemons associated with directory. + + -V Use <dir> as $TEST_VAR_DIR + + Use the specified temporary temporary/var directory. + + -H No headers - for running single test with other wrapper + + This allows tests to be embedded in some other test framework + and executed one-by-one with all the required + environment/infrastructure. + + This replaces the old ctdb_test_env script. + +How do the tests find remote test programs? +------------------------------------------- + +If the all of the cluster nodes have the CTDB git tree in the same +location as on the test client then no special action is necessary. +The simplest way of doing this is to share the tree to cluster nodes +and test clients via NFS. + +If cluster nodes do not have the CTDB git tree then +CTDB_TEST_REMOTE_DIR can be set to a directory that, on each cluster +node, contains the contents of tests/scripts/ and tests/bin/. + +In the future this will hopefully (also) be supported via a ctdb-test +package. + +Running the ctdb tool under valgrind +------------------------------------ + +The easiest way of doing this is something like: + + VALGRIND="valgrind -q" scripts/run_tests ... + +This can be used to cause all invocations of the ctdb client (and, +with local daemons, the ctdbd daemons themselves) to occur under +valgrind. + +NOTE: Some libc calls seem to do weird things and perhaps cause +spurious output from ctdbd at start time. Please read valgrind output +carefully before reporting bugs. :-) + +How is the ctdb tool invoked? +----------------------------- + +$CTDB determines how to invoke the ctdb client. If not already set +and if $VALGRIND is set, this is set to "$VALGRIND ctdb". If this is +not already set but $VALGRIND is not set, this is simply set to "ctdb" diff --git a/ctdb/tests/TODO b/ctdb/tests/TODO new file mode 100644 index 00000000000..be471cc8f84 --- /dev/null +++ b/ctdb/tests/TODO @@ -0,0 +1,4 @@ +* Make tests know about IPv6. +* Tests that write to database. +* Tests that check actual network connectivity on failover. +* Handle interrupting tests better. diff --git a/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh new file mode 100755 index 00000000000..043c345fac7 --- /dev/null +++ b/ctdb/tests/complex/11_ctdb_delip_removes_ip.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a node's public IP address can be deleted using 'ctdb deleteip'. + +Check that the address is actually deleted from the interface. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +* Test must be run on a real or virtual cluster rather than against + local daemons. There is nothing intrinsic to this test that forces + this - it is because tests run against local daemons don't use the + regular eventscripts. Local daemons put public addresses on + loopback, so we can't reliably test when IPs have moved between + nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Use 'ctdb ip' on one of the nodes to list the IP addresses being + served. +3. Select an IP address being served by the node and check that it + actually appears on the interface it is supposed to be on. +4. Delete the IP address using 'ctdb delip'. +5. Verify that the deleted IP address is no longer listed using the + all_ips_on_node helper function. +6. Verify that the deleted IP address no longer appears on the + interface it was on. + +Expected results: + +* 'ctdb delip' removes an IP address from the list of public IP + addresses being served by a node and from the network interface. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +echo "Getting list of public IPs..." +all_ips_on_node -v 0 + +# Select an IP/node to remove. +num_ips=$(echo "$out" | wc -l) +num_to_remove=$(($RANDOM % $num_ips)) + +# Find the details in the list. +i=0 +while [ $i -le $num_to_remove ] ; do + read ip_to_remove test_node + i=$(($i + 1)) +done <<<"$out" + +echo "Determining interface for ${ip_to_remove} on ${test_node}." +try_command_on_node $test_node "ctdb ip -Y -v" +iface=$(echo "$out" | awk -F: -v ip=${ip_to_remove} -v pnn=${test_node} '$2 == ip && $3 == pnn { print $4 }') +echo "$iface" +[ -n "$iface" ] + +echo "Checking that node ${test_node} hosts ${ip_to_remove} on interface ${iface}..." +try_command_on_node $test_node "ip addr show dev $iface | grep -E 'inet[[:space:]]*${ip_to_remove}/'" + +echo "Attempting to remove ${ip_to_remove} from node ${test_node}." +try_command_on_node $test_node $CTDB delip $ip_to_remove + +echo "Sleeping..." +sleep_for 1 + +test_node_ips="" +while read ip pnn ; do + [ "$pnn" = "$test_node" ] && \ + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" +done <<<"$out" # bashism to avoid problem setting variable in pipeline. + +if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then + echo "GOOD: That worked!" +else + echo "BAD: The remove IP address is still there!" + testfailures=1 +fi + +timeout=60 +increment=5 +count=0 +echo "Waiting for ${ip_to_remove} to disappear from ${iface}..." +while : ; do + try_command_on_node -v $test_node "ip addr show dev $iface" + if echo "$out" | grep -E 'inet[[:space:]]*${ip_to_remove}/'; then + echo "Still there..." + if [ $(($count * $increment)) -ge $timeout ] ; then + echo "BAD: Timed out waiting..." + exit 1 + fi + sleep_for $increment + count=$(($count + 1)) + else + break + fi +done + +echo "GOOD: IP was successfully removed!" diff --git a/ctdb/tests/complex/31_nfs_tickle.sh b/ctdb/tests/complex/31_nfs_tickle.sh new file mode 100755 index 00000000000..ce4ae81c9d5 --- /dev/null +++ b/ctdb/tests/complex/31_nfs_tickle.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that NFS connections are monitored and that NFS tickles are sent. + +We create a connection to the NFS server on a node and confirm that +this connection is registered in the nfs-tickles/ subdirectory in +shared storage. Then disable the relevant NFS server node and ensure +that it send an appropriate reset packet. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +* Cluster nodes must be listening on the NFS TCP port (2049). + +Steps: + +1. Verify that the cluster is healthy. +2. Connect from the current host (test client) to TCP port 2049 using + the public address of a cluster node. +3. Determine the source socket used for the connection. +4. Ensure that CTDB records the source socket details in the nfs-tickles + directory on shared storage. +5. Disable the node that the connection has been made to. +6. Verify that a TCP tickle (a reset packet) is sent to the test client. + +Expected results: + +* CTDB should correctly record the socket in the nfs-tickles directory + and should send a reset packet when the node is disabled. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +ctdb_test_exit_hook_add ctdb_test_eventscript_uninstall + +ctdb_test_eventscript_install + +# We need this for later, so we know how long to sleep. +try_command_on_node any $CTDB getvar MonitorInterval +monitor_interval="${out#*= }" +#echo "Monitor interval on node $test_node is $monitor_interval seconds." + +select_test_node_and_ips + +test_port=2049 + +echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with netcat..." + +nc -d -w $(($monitor_interval * 4)) $test_ip $test_port & +nc_pid=$! +ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1" + +wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc" +src_socket="$out" +echo "Source socket is $src_socket" + +wait_for_monitor_event $test_node + +echo "Sleeping until tickles are synchronised across nodes..." +try_command_on_node $test_node $CTDB getvar TickleUpdateInterval +sleep_for "${out#*= }" + +if try_command_on_node any "test -r /etc/ctdb/events.d/61.nfstickle" ; then + echo "Trying to determine NFS_TICKLE_SHARED_DIRECTORY..." + if [ -f /etc/sysconfig/nfs ]; then + f="/etc/sysconfig/nfs" + elif [ -f /etc/default/nfs ]; then + f="/etc/default/nfs" + elif [ -f /etc/ctdb/sysconfig/nfs ]; then + f="/etc/ctdb/sysconfig/nfs" + fi + try_command_on_node -v any "[ -r $f ] && sed -n -e s@^NFS_TICKLE_SHARED_DIRECTORY=@@p $f" || true + + nfs_tickle_shared_directory="${out:-/gpfs/.ctdb/nfs-tickles}" + + try_command_on_node $test_node hostname + test_hostname=$out + + try_command_on_node -v any cat "${nfs_tickle_shared_directory}/$test_hostname/$test_ip" +else + echo "That's OK, we'll use \"ctdb gettickles\", which is newer..." + try_command_on_node -v any "ctdb -Y gettickles $test_ip $test_port" +fi + +if [ "${out/${src_socket}/}" != "$out" ] ; then + echo "GOOD: NFS connection tracked OK." +else + echo "BAD: Socket not tracked in NFS tickles." + testfailures=1 +fi + +tcptickle_sniff_start $src_socket "${test_ip}:${test_port}" + +# We need to be nasty to make that the node being failed out doesn't +# get a chance to send any tickles and confuse our sniff. +echo "Killing ctdbd on ${test_node}..." +try_command_on_node $test_node killall -9 ctdbd + +wait_until_node_has_status $test_node disconnected + +tcptickle_sniff_wait_show diff --git a/ctdb/tests/complex/32_cifs_tickle.sh b/ctdb/tests/complex/32_cifs_tickle.sh new file mode 100755 index 00000000000..93634e72bd6 --- /dev/null +++ b/ctdb/tests/complex/32_cifs_tickle.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that CIFS connections are monitored and that CIFS tickles are sent. + +We create a connection to the CIFS server on a node and confirm that +this connection is registered by CTDB. Then disable the relevant CIFS +server node and ensure that it send an appropriate reset packet. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +* Clustered Samba must be listening on TCP port 445. + +Steps: + +1. Verify that the cluster is healthy. +2. Connect from the current host (test client) to TCP port 445 using + the public address of a cluster node. +3. Determine the source socket used for the connection. +4. Using the "ctdb gettickle" command, ensure that CTDB records the + connection details. +5. Disable the node that the connection has been made to. +6. Verify that a TCP tickle (a reset packet) is sent to the test client. + +Expected results: + +* CTDB should correctly record the connection and should send a reset + packet when the node is disabled. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +# We need this for later, so we know how long to sleep. +try_command_on_node 0 $CTDB getvar MonitorInterval +monitor_interval="${out#*= }" +#echo "Monitor interval on node $test_node is $monitor_interval seconds." + +select_test_node_and_ips + +test_port=445 + +echo "Connecting to node ${test_node} on IP ${test_ip}:${test_port} with netcat..." + +nc -d -w $(($monitor_interval * 4)) $test_ip $test_port & +nc_pid=$! +ctdb_test_exit_hook_add "kill $nc_pid >/dev/null 2>&1" + +wait_until_get_src_socket "tcp" "${test_ip}:${test_port}" $nc_pid "nc" +src_socket="$out" +echo "Source socket is $src_socket" + +# This should happen as soon as connection is up... but unless we wait +# we sometimes beat the registration. +check_tickles () +{ + try_command_on_node 0 ctdb gettickles $test_ip -n $test_node + # SRC: 10.0.2.45:49091 DST: 10.0.2.143:445 + [ "${out/SRC: ${src_socket} /}" != "$out" ] +} + +echo "Checking if CIFS connection is tracked by CTDB..." +wait_until 10 check_tickles +echo "$out" + +if [ "${out/SRC: ${src_socket} /}" != "$out" ] ; then + echo "GOOD: CIFS connection tracked OK by CTDB." +else + echo "BAD: Socket not tracked by CTDB." + testfailures=1 +fi + +tcptickle_sniff_start $src_socket "${test_ip}:${test_port}" + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled + +tcptickle_sniff_wait_show diff --git a/ctdb/tests/complex/33_gratuitous_arp.sh b/ctdb/tests/complex/33_gratuitous_arp.sh new file mode 100755 index 00000000000..721b0f2a348 --- /dev/null +++ b/ctdb/tests/complex/33_gratuitous_arp.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a gratuitous ARP is sent when a node is failed out. + +We ping a public IP and lookup the MAC address in the ARP table. We +then disable the node and check the ARP table again - the MAC address +should have changed. This test does NOT test connectivity after the +failover. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +Steps: + +1. Verify that the cluster is healthy. +2. Select a public address and its corresponding node. +3. Remove any entries for the chosen address from the ARP table. +4. Send a single ping request packet to the selected public address. +5. Determine the MAC address corresponding to the public address by + checking the ARP table. +6. Disable the selected node. +7. Check the ARP table and check the MAC associated with the public + address. + +Expected results: + +* When a node is disabled the MAC address associated with public + addresses on that node should change. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Removing ${test_ip} from the local ARP table..." +arp -d $test_ip >/dev/null 2>&1 || true + +echo "Pinging ${test_ip}..." +ping -q -n -c 1 $test_ip + +echo "Getting MAC address associated with ${test_ip}..." +original_mac=$(arp -n $test_ip | awk '$2 == "ether" {print $3}') +[ $? -eq 0 ] + +echo "MAC address is: ${original_mac}" + +gratarp_sniff_start + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled + +gratarp_sniff_wait_show + +echo "Getting MAC address associated with ${test_ip} again..." +new_mac=$(arp -n $test_ip | awk '$2 == "ether" {print $3}') +[ $? -eq 0 ] + +echo "MAC address is: ${new_mac}" + +if [ "$original_mac" != "$new_mac" ] ; then + echo "GOOD: MAC address changed" +else + echo "BAD: MAC address did not change" + testfailures=1 +fi diff --git a/ctdb/tests/complex/41_failover_ping_discrete.sh b/ctdb/tests/complex/41_failover_ping_discrete.sh new file mode 100755 index 00000000000..88b2013f061 --- /dev/null +++ b/ctdb/tests/complex/41_failover_ping_discrete.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that it is possible to ping a public address after disabling a node. + +We ping a public IP, disable the node hosting it and then ping the +public IP again. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +Steps: + +1. Verify that the cluster is healthy. +2. Select a public address and its corresponding node. +3. Send a single ping request packet to the selected public address. +4. Disable the selected node. +5. Send another single ping request packet to the selected public address. + +Expected results: + +* When a node is disabled the public address fails over and the + address is still pingable. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Removing ${test_ip} from the local ARP table..." +arp -d $test_ip >/dev/null 2>&1 || true + +echo "Pinging ${test_ip}..." +ping -q -n -c 1 $test_ip + +gratarp_sniff_start + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled + +gratarp_sniff_wait_show + +echo "Removing ${test_ip} from the local ARP table again..." +arp -d $test_ip >/dev/null 2>&1 || true + +echo "Pinging ${test_ip} again..." +ping -q -n -c 1 $test_ip diff --git a/ctdb/tests/complex/42_failover_ssh_hostname.sh b/ctdb/tests/complex/42_failover_ssh_hostname.sh new file mode 100755 index 00000000000..defe15ad114 --- /dev/null +++ b/ctdb/tests/complex/42_failover_ssh_hostname.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that it is possible to SSH to a public address after disabling a node. + +We SSH to a public IP and check the hostname, disable the node hosting +it and then SSH again to confirm that the hostname has changed. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +Steps: + +1. Verify that the cluster is healthy. +2. Select a public address and its corresponding node. +3. SSH to the selected public address and run hostname. +4. Disable the selected node. +5. SSH to the selected public address again and run hostname. + +Expected results: + +* When a node is disabled the public address fails over and it is + still possible to SSH to the node. The hostname should change. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Removing ${test_ip} from the local ARP table..." +arp -d $test_ip >/dev/null 2>&1 || true + +echo "SSHing to ${test_ip} and running hostname..." +original_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) +[ $? -eq 0 ] + +echo "Hostname is: ${original_hostname}" + +gratarp_sniff_start + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled + +gratarp_sniff_wait_show + +echo "SSHing to ${test_ip} and running hostname (again)..." +new_hostname=$(ssh -o "StrictHostKeyChecking no" $test_ip hostname) +[ $? -eq 0 ] + +echo "Hostname is: ${new_hostname}" + +if [ "$original_hostname" != "$new_hostname" ] ; then + echo "GOOD: hostname changed" +else + echo "BAD: hostname did not change" + testfailures=1 +fi diff --git a/ctdb/tests/complex/43_failover_nfs_basic.sh b/ctdb/tests/complex/43_failover_nfs_basic.sh new file mode 100755 index 00000000000..a68f7db6811 --- /dev/null +++ b/ctdb/tests/complex/43_failover_nfs_basic.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a mounted NFS share is still operational after failover. + +We mount an NFS share from a node, write a file via NFS and then +confirm that we can correctly read the file after a failover. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +Steps: + +1. Verify that the cluster is healthy. +2. Select a public address and its corresponding node. +3. Select the 1st NFS share exported on the node. +4. Mount the selected NFS share. +5. Create a file in the NFS mount and calculate its checksum. +6. Disable the selected node. +7. Read the file and calculate its checksum. +8. Compare the checksums. + +Expected results: + +* When a node is disabled the public address fails over and it is + possible to correctly read a file over NFS. The checksums should be + the same before and after. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +nfs_test_setup + +echo "Create file containing random data..." +dd if=/dev/urandom of=$nfs_local_file bs=1k count=1 +original_sum=$(sum $nfs_local_file) +[ $? -eq 0 ] + +gratarp_sniff_start + +echo "Disabling node $test_node" +try_command_on_node 0 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled + +gratarp_sniff_wait_show + +new_sum=$(sum $nfs_local_file) +[ $? -eq 0 ] + +if [ "$original_md5" = "$new_md5" ] ; then + echo "GOOD: file contents unchanged after failover" +else + echo "BAD: file contents are different after failover" + testfailures=1 +fi diff --git a/ctdb/tests/complex/44_failover_nfs_oneway.sh b/ctdb/tests/complex/44_failover_nfs_oneway.sh new file mode 100755 index 00000000000..aaec2ed9905 --- /dev/null +++ b/ctdb/tests/complex/44_failover_nfs_oneway.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a file created on a node is readable via NFS after a failover. + +We write a file into an exported directory on a node, mount the NFS +share from a node, verify that we can read the file via NFS and that +we can still read it after a failover. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +Steps: + +1. Verify that the cluster is healthy. +2. Select a public address and its corresponding node. +3. Select the 1st NFS share exported on the node. +4. Write a file into exported directory on the node and calculate its + checksum. +5. Mount the selected NFS share. +6. Read the file via the NFS mount and calculate its checksum. +7. Compare checksums. +8. Disable the selected node. +9. Read the file via NFS and calculate its checksum. +10. Compare the checksums. + +Expected results: + +* Checksums for the file on all 3 occasions should be the same. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +nfs_test_setup + +echo "Create file containing random data..." +local_f=$(mktemp) +ctdb_test_exit_hook_add rm -f "$local_f" +dd if=/dev/urandom of=$local_f bs=1k count=1 +local_sum=$(sum $local_f) + +scp -p "$local_f" "${test_ip}:${nfs_remote_file}" +try_command_on_node $test_node "chmod 644 $nfs_remote_file" + +nfs_sum=$(sum $nfs_local_file) + +if [ "$local_sum" = "$nfs_sum" ] ; then + echo "GOOD: file contents read correctly via NFS" +else + echo "BAD: file contents are different over NFS" + echo " original file: $local_sum" + echo " NFS file: $nfs_sum" + exit 1 +fi + +gratarp_sniff_start + +echo "Disabling node $test_node" +try_command_on_node 0 $CTDB disable -n $test_node +wait_until_node_has_status $test_node disabled + +gratarp_sniff_wait_show + +new_sum=$(sum $nfs_local_file) +[ $? -eq 0 ] + +if [ "$nfs_sum" = "$new_sum" ] ; then + echo "GOOD: file contents unchanged after failover" +else + echo "BAD: file contents are different after failover" + echo " original file: $nfs_sum" + echo " NFS file: $new_sum" + exit 1 +fi diff --git a/ctdb/tests/complex/45_failover_nfs_kill.sh b/ctdb/tests/complex/45_failover_nfs_kill.sh new file mode 100755 index 00000000000..52b423fb12b --- /dev/null +++ b/ctdb/tests/complex/45_failover_nfs_kill.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a mounted NFS share is still operational after failover. + +We mount an NFS share from a node, write a file via NFS and then +confirm that we can correctly read the file after a failover. + +Prerequisites: + +* An active CTDB cluster with at least 2 nodes with public addresses. + +* Test must be run on a real or virtual cluster rather than against + local daemons. + +* Test must not be run from a cluster node. + +Steps: + +1. Verify that the cluster is healthy. +2. Select a public address and its corresponding node. +3. Select the 1st NFS share exported on the node. +4. Mount the selected NFS share. +5. Create a file in the NFS mount and calculate its checksum. +6. Kill CTDB on the selected node. +7. Read the file and calculate its checksum. +8. Compare the checksums. + +Expected results: + +* When a node is disabled the public address fails over and it is + possible to correctly read a file over NFS. The checksums should be + the same before and after. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +set -e + +ctdb_test_init "$@" + +ctdb_test_check_real_cluster + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +nfs_test_setup + +echo "Create file containing random data..." +dd if=/dev/urandom of=$nfs_local_file bs=1k count=1 +original_sum=$(sum $nfs_local_file) +[ $? -eq 0 ] + +gratarp_sniff_start + +echo "Killing node $test_node" +try_command_on_node $test_node $CTDB getpid +pid=${out#*:} +try_command_on_node $test_node kill -9 $pid +wait_until_node_has_status $test_node disconnected + +gratarp_sniff_wait_show + +new_sum=$(sum $nfs_local_file) +[ $? -eq 0 ] + +if [ "$original_md5" = "$new_md5" ] ; then + echo "GOOD: file contents unchanged after failover" +else + echo "BAD: file contents are different after failover" + testfailures=1 +fi diff --git a/ctdb/tests/complex/README b/ctdb/tests/complex/README new file mode 100644 index 00000000000..72de39656af --- /dev/null +++ b/ctdb/tests/complex/README @@ -0,0 +1,2 @@ +Complex integration tests. These need a real or virtual cluster. +That is, they can not be run against local daemons. diff --git a/ctdb/tests/complex/scripts/local.bash b/ctdb/tests/complex/scripts/local.bash new file mode 100644 index 00000000000..eb4c41c8b61 --- /dev/null +++ b/ctdb/tests/complex/scripts/local.bash @@ -0,0 +1,143 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + local pat="^${proto}[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$" + out=$(netstat -tanp | + egrep "$pat" | + awk '{ print $4 }') + + [ -n "$out" ] +} + +wait_until_get_src_socket () +{ + local proto="$1" + local dst_socket="$2" + local pid="$3" + local prog="$4" + + echo "Waiting for ${prog} to establish connection to ${dst_socket}..." + + wait_until 5 get_src_socket "$@" +} + +####################################### + +# filename will be in $tcpdump_filename, pid in $tcpdump_pid +tcpdump_start () +{ + tcpdump_filter="$1" # global + + echo "Running tcpdump..." + tcpdump_filename=$(mktemp) + ctdb_test_exit_hook_add "rm -f $tcpdump_filename" + + # The only way of being sure that tcpdump is listening is to send + # some packets that it will see. So we use dummy pings - the -U + # option to tcpdump ensures that packets are flushed to the file + # as they are captured. + local dummy_addr="127.3.2.1" + local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo" + tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + echo "Waiting for tcpdump output file to be ready..." + ping -q "$dummy_addr" >/dev/null 2>&1 & + ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1" + + tcpdump_listen_for_dummy () + { + tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1 + } + + wait_until 10 tcpdump_listen_for_dummy +} + +# By default, wait for 1 matching packet. +tcpdump_wait () +{ + local count="${1:-1}" + local filter="${2:-${tcpdump_filter}}" + + tcpdump_check () + { + local found=$(tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null | wc -l) + [ $found -ge $count ] + } + + echo "Waiting for tcpdump to capture some packets..." + if ! wait_until 30 tcpdump_check ; then + echo "DEBUG AT $(date '+%F %T'):" + local i + for i in "onnode -q 0 $CTDB status" "netstat -tanp" "tcpdump -n -e -r $tcpdump_filename" ; do + echo "$i" + $i || true + done + return 1 + fi +} + +tcpdump_show () +{ + local filter="${1:-${tcpdump_filter}}" + + tcpdump -n -r $tcpdump_filename "$filter" 2>/dev/null +} + +tcptickle_sniff_start () +{ + local src="$1" + local dst="$2" + + local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}" + local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}" + local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14] == 4) and (tcp[15] == 210)" # win == 1234 + local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)" + tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0" + local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})" + + tcpdump_start "$filter" +} + +tcptickle_sniff_wait_show () +{ + tcpdump_wait 1 "$tcptickle_reset" + + echo "GOOD: here are some TCP tickle packets:" + tcpdump_show +} + +gratarp_sniff_start () +{ + tcpdump_start "arp host ${test_ip}" +} + +gratarp_sniff_wait_show () +{ + tcpdump_wait 2 + + echo "GOOD: this should be the some gratuitous ARPs:" + tcpdump_show +} + + +ctdb_test_check_real_cluster () +{ + [ -z "$TEST_LOCAL_DAEMONS" ] || \ + die "ERROR: This test must be run against a real/virtual cluster, not local daemons." + + local h=$(hostname) + + local i + for i in $(onnode -q all hostname) ; do + [ "$h" != "$i" ] || \ + die "ERROR: This test must not be run from a cluster node." + done +} + diff --git a/ctdb/tests/events.d/00.test b/ctdb/tests/events.d/00.test new file mode 100755 index 00000000000..e3e15eb7301 --- /dev/null +++ b/ctdb/tests/events.d/00.test @@ -0,0 +1,105 @@ +#!/bin/sh +# event script for 'make test' + +cmd="$1" +shift + +case $cmd in + monitor) + echo "monitor event" + echo "monitor event stderr" >&2 + exit 0 + ;; + + startrecovery) + echo "ctdb startrecovery event" + exit 0; + ;; + + init) + echo "ctdb init event" + exit 0; + ;; + setup) + echo "ctdb setup event" + exit 0; + ;; + startup) + echo "ctdb startup event" + IFACES=`ctdb ifaces -Y | grep -v '^:Name:LinkStatus:References:'` + for I in $IFACES; do + IFACE=`echo -n "$I" | cut -d ':' -f2` + ctdb setifacelink $IFACE up + done + exit 0; + ;; + + takeip) + if [ $# != 3 ]; then + echo "must supply interface, IP and maskbits" + exit 1 + fi + iface=$1 + ip=$2 + maskbits=$3 + + [ -n "$TEST_LOCAL_DAEMONS" ] || { + /sbin/ip addr add $ip/$maskbits dev $iface || { + echo "Failed to add $ip/$maskbits on dev $iface" + exit 1 + } + } + echo "ctdb takeip event for $1 $2 $3" + exit 0; + ;; + + + ################################################## + # called when ctdbd wants to release an IP address + releaseip) + if [ $# != 3 ]; then + echo "must supply interface, IP and maskbits" + exit 1 + fi + iface=$1 + ip=$2 + maskbits=$3 + [ -n "$TEST_LOCAL_DAEMONS" ] || { + /sbin/ip addr del $ip/$maskbits dev $iface || { + echo "Failed to del $ip on dev $iface" + exit 1 + } + } + echo "ctdb releaseip event for $1 $2 $3" + exit 0 + ;; + + updateip) + echo "ctdb updateip event for $1" + exit 0 + ;; + + recovered) + echo "ctdb recovered event" + exit 0 + ;; + + ipreallocated) + echo "ctdb ipreallocated event" + exit 0 + ;; + + + shutdown) + echo "ctdb shutdown event" + exit 0 + ;; + + stopped) + echo "ctdb stopped event" + exit 0 + ;; +esac + +echo "Invalid command $cmd" +exit 1 diff --git a/ctdb/tests/eventscripts/00.ctdb.init.001.sh b/ctdb/tests/eventscripts/00.ctdb.init.001.sh new file mode 100755 index 00000000000..320025ac282 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool supports check" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.002.sh b/ctdb/tests/eventscripts/00.ctdb.init.002.sh new file mode 100755 index 00000000000..2777cc5eb4c --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.002.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool does no support check" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="no" + +ok <<EOF +WARNING: The installed 'tdbtool' does not offer the 'check' subcommand. + Using 'tdbdump' for database checks. + Consider updating 'tdbtool' for better checks! +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.003.sh b/ctdb/tests/eventscripts/00.ctdb.init.003.sh new file mode 100755 index 00000000000..27702104601 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.003.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool supports check, good TDB" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + +touch "${CTDB_DBDIR}/foo.tdb.0" +FAKE_TDB_IS_OK="yes" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.004.sh b/ctdb/tests/eventscripts/00.ctdb.init.004.sh new file mode 100755 index 00000000000..b504d08b2c9 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.004.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool supports check, bad TDB" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + +db="${CTDB_DBDIR}/foo.tdb.0" +touch "$db" +FAKE_TDB_IS_OK="no" + +FAKE_DATE_OUTPUT="19690818.103000.000000001" + +ok <<EOF +WARNING: database ${db} is corrupted. + Moving to backup ${db}.${FAKE_DATE_OUTPUT}.corrupt for later analysis. +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.005.sh b/ctdb/tests/eventscripts/00.ctdb.init.005.sh new file mode 100755 index 00000000000..d11ab94885d --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.005.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool does not support check, good TDB" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="no" + +touch "${CTDB_DBDIR}/foo.tdb.0" +FAKE_TDB_IS_OK="yes" + +ok <<EOF +WARNING: The installed 'tdbtool' does not offer the 'check' subcommand. + Using 'tdbdump' for database checks. + Consider updating 'tdbtool' for better checks! +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.006.sh b/ctdb/tests/eventscripts/00.ctdb.init.006.sh new file mode 100755 index 00000000000..745bca0f993 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.006.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool does not support check, bad TDB" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="no" + +db="${CTDB_DBDIR}/foo.tdb.0" +touch "$db" +FAKE_TDB_IS_OK="no" + +FAKE_DATE_OUTPUT="19690818.103000.000000001" + +ok <<EOF +WARNING: The installed 'tdbtool' does not offer the 'check' subcommand. + Using 'tdbdump' for database checks. + Consider updating 'tdbtool' for better checks! +WARNING: database ${db} is corrupted. + Moving to backup ${db}.${FAKE_DATE_OUTPUT}.corrupt for later analysis. +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.007.sh b/ctdb/tests/eventscripts/00.ctdb.init.007.sh new file mode 100755 index 00000000000..1c954d7d8d1 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.007.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool supports check, good persistent TDB" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + +touch "${CTDB_DBDIR}/persistent/foo.tdb.0" +FAKE_TDB_IS_OK="yes" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.008.sh b/ctdb/tests/eventscripts/00.ctdb.init.008.sh new file mode 100755 index 00000000000..a6afdd8671f --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.008.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "TDB check, tdbtool supports check, bad persistent TDB" + +setup_ctdb + +FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + +db="${CTDB_DBDIR}/persistent/foo.tdb.0" +touch "$db" +FAKE_TDB_IS_OK="no" + +required_result 1 <<EOF +Persistent database ${db} is corrupted! CTDB will not start. +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.021.sh b/ctdb/tests/eventscripts/00.ctdb.init.021.sh new file mode 100755 index 00000000000..87dfa4dfdcf --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.021.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Check public IP dropping, none assigned" + +setup_ctdb + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.init.022.sh b/ctdb/tests/eventscripts/00.ctdb.init.022.sh new file mode 100755 index 00000000000..6e594280a3a --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.022.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Check public IP dropping, 1 assigned" + +setup_ctdb + +ctdb_get_1_public_address | +while read dev ip bits ; do + ip addr add "${ip}/${bits}" dev "$dev" + + ok <<EOF +Removing public address ${ip}/${bits} from device ${dev} +EOF + + simple_test +done diff --git a/ctdb/tests/eventscripts/00.ctdb.init.023.sh b/ctdb/tests/eventscripts/00.ctdb.init.023.sh new file mode 100755 index 00000000000..9b97e821313 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.init.023.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Check public IP dropping, all assigned" + +setup_ctdb + +nl=" +" +ctdb_get_my_public_addresses | { + out="" + while read dev ip bits ; do + ip addr add "${ip}/${bits}" dev "$dev" + + msg="Removing public address ${ip}/${bits} from device ${dev}" + out="${out}${out:+${nl}}${msg}" + done + + ok "$out" + + simple_test +} diff --git a/ctdb/tests/eventscripts/00.ctdb.monitor.001.sh b/ctdb/tests/eventscripts/00.ctdb.monitor.001.sh new file mode 100755 index 00000000000..4290d13e15d --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.monitor.001.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Memory check, bad situation, no checks enabled" + +setup_memcheck "bad" + +CTDB_MONITOR_FREE_MEMORY="" +CTDB_MONITOR_FREE_MEMORY_WARN="" +CTDB_CHECK_SWAP_IS_NOT_USED="no" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.monitor.002.sh b/ctdb/tests/eventscripts/00.ctdb.monitor.002.sh new file mode 100755 index 00000000000..6e9401233b2 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.monitor.002.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Memory check, good situation, all enabled" + +setup_memcheck + +CTDB_MONITOR_FREE_MEMORY="500" +CTDB_MONITOR_FREE_MEMORY_WARN="1000" +CTDB_CHECK_SWAP_IS_NOT_USED="yes" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.monitor.003.sh b/ctdb/tests/eventscripts/00.ctdb.monitor.003.sh new file mode 100755 index 00000000000..9e63ab50fd3 --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.monitor.003.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Memory check, bad situation, only swap check" + +setup_memcheck "bad" + +CTDB_MONITOR_FREE_MEMORY="" +CTDB_MONITOR_FREE_MEMORY_WARN="" +CTDB_CHECK_SWAP_IS_NOT_USED="yes" + +ok <<EOF +We are swapping: +$FAKE_PROC_MEMINFO +$(ps foobar) +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.monitor.004.sh b/ctdb/tests/eventscripts/00.ctdb.monitor.004.sh new file mode 100755 index 00000000000..fdf20329e2d --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.monitor.004.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Memory check, bad situation, only memory warning" + +setup_memcheck "bad" + +CTDB_MONITOR_FREE_MEMORY="" +CTDB_MONITOR_FREE_MEMORY_WARN="500" +CTDB_CHECK_SWAP_IS_NOT_USED="no" + +ok <<EOF +WARNING: free memory is low - 468MB free <= ${CTDB_MONITOR_FREE_MEMORY_WARN}MB (CTDB threshold) +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/00.ctdb.monitor.005.sh b/ctdb/tests/eventscripts/00.ctdb.monitor.005.sh new file mode 100755 index 00000000000..a46851a573d --- /dev/null +++ b/ctdb/tests/eventscripts/00.ctdb.monitor.005.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Memory check, bad situation, only memory critical" + +setup_memcheck "bad" + +CTDB_MONITOR_FREE_MEMORY="500" +CTDB_MONITOR_FREE_MEMORY_WARN="" +CTDB_CHECK_SWAP_IS_NOT_USED="no" + +ok <<EOF +CRITICAL: OOM - 468MB free <= ${CTDB_MONITOR_FREE_MEMORY}MB (CTDB threshold) +CRITICAL: Shutting down CTDB!!! +$FAKE_PROC_MEMINFO +$(ps foobar) +CTDB says BYE! +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.init.001.sh b/ctdb/tests/eventscripts/10.interface.init.001.sh new file mode 100755 index 00000000000..fae1a78d506 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.init.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "no public addresses" + +setup_ctdb + +export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" + +ok "No public addresses file found. Nothing to do for 10.interfaces" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.init.002.sh b/ctdb/tests/eventscripts/10.interface.init.002.sh new file mode 100755 index 00000000000..ba33f927c3f --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.init.002.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up" + +setup_ctdb + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.001.sh b/ctdb/tests/eventscripts/10.interface.monitor.001.sh new file mode 100755 index 00000000000..42ef42d81b4 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "no public addresses" + +setup_ctdb + +export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.002.sh b/ctdb/tests/eventscripts/10.interface.monitor.002.sh new file mode 100755 index 00000000000..ba33f927c3f --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.002.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up" + +setup_ctdb + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.003.sh b/ctdb/tests/eventscripts/10.interface.monitor.003.sh new file mode 100755 index 00000000000..1eb7916b932 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.003.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 interface down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +ethtool_interfaces_down $iface + +required_result 1 "ERROR: No link on the public network interface $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.004.sh b/ctdb/tests/eventscripts/10.interface.monitor.004.sh new file mode 100755 index 00000000000..69ffbd00cd5 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.004.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up, 1 is a bond" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.005.sh b/ctdb/tests/eventscripts/10.interface.monitor.005.sh new file mode 100755 index 00000000000..8cf7bbc9c0a --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.005.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 bond, no active slaves" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "None" + +required_result 1 "ERROR: No active slaves for bond device $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.006.sh b/ctdb/tests/eventscripts/10.interface.monitor.006.sh new file mode 100755 index 00000000000..3c483a3516f --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.006.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 bond, active slaves, link down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "" "down" + +required_result 1 "ERROR: public network interface $iface is down" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.007.sh b/ctdb/tests/eventscripts/10.interface.monitor.007.sh new file mode 100755 index 00000000000..c45900e3b2e --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.007.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "unknown interface, up" + +setup_ctdb + +export CTDB_PUBLIC_INTERFACE="dev999" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.008.sh b/ctdb/tests/eventscripts/10.interface.monitor.008.sh new file mode 100755 index 00000000000..f73302b30da --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.008.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "unknown interface, down, up" + +setup_ctdb + +iface="dev999" +export CTDB_PUBLIC_INTERFACE="$iface" + +#EVENTSCRIPTS_TESTS_TRACE="sh -x" +iterate_test 3 "ok_null" \ + 1 'ethtool_interfaces_down "$iface" ; required_result 1 "ERROR: No link on the public network interface $iface"' \ + 2 'ethtool_interfaces_up "$iface"' diff --git a/ctdb/tests/eventscripts/10.interface.monitor.009.sh b/ctdb/tests/eventscripts/10.interface.monitor.009.sh new file mode 100755 index 00000000000..1b785ffdc80 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.009.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down "$iface" + +ok "ERROR: No link on the public network interface $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.010.sh b/ctdb/tests/eventscripts/10.interface.monitor.010.sh new file mode 100755 index 00000000000..4d233193fb6 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.010.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, all down" + +setup_ctdb + +ifaces=$(ctdb_get_interfaces) + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down $ifaces + +msg=$(for i in $ifaces ; do echo "ERROR: No link on the public network interface $i" ; done) + +required_result 1 "$msg" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.011.sh b/ctdb/tests/eventscripts/10.interface.monitor.011.sh new file mode 100755 index 00000000000..21775d41ce5 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.011.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "None" + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down "$iface" + +ok "ERROR: No active slaves for bond device $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.012.sh b/ctdb/tests/eventscripts/10.interface.monitor.012.sh new file mode 100755 index 00000000000..dbe84b77296 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.012.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down" + +setup_ctdb + +ifaces=$(ctdb_get_interfaces) + +for i in $ifaces ; do + setup_bond $i "None" +done + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down $ifaces + +msg=$(for i in $ifaces ; do echo "ERROR: No active slaves for bond device $i" ; done) + +required_result 1 "$msg" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.013.sh b/ctdb/tests/eventscripts/10.interface.monitor.013.sh new file mode 100755 index 00000000000..0fcdcd8d15b --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.013.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 bond, active slaves, link down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "" "up" "down" + +required_result 1 "ERROR: No active slaves for 802.ad bond device $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.014.sh b/ctdb/tests/eventscripts/10.interface.monitor.014.sh new file mode 100755 index 00000000000..ab23d307c29 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.014.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "spurious addresses on interface, no action" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +ip addr add 192.168.253.253/24 dev $iface +ip addr add 192.168.254.254/24 dev $iface + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.015.sh b/ctdb/tests/eventscripts/10.interface.monitor.015.sh new file mode 100755 index 00000000000..1090cb9a554 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.015.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Missing interface, fail" + +setup_ctdb + +iface=$(ctdb_get_1_interface) +ip link delete "$iface" + +required_result 1 <<EOF +ERROR: Interface dev123 does not exist but it is used by public addresses. +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.016.sh b/ctdb/tests/eventscripts/10.interface.monitor.016.sh new file mode 100755 index 00000000000..6fd698a1982 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.016.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Missing interface, CTDB_PARTIALLY_ONLINE_INTERFACES=yes, warn" + +setup_ctdb + +CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +iface=$(ctdb_get_1_interface) +ip link delete "$iface" + +ok <<EOF +ERROR: Interface dev123 does not exist but it is used by public addresses. +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.multi.001.sh b/ctdb/tests/eventscripts/10.interface.multi.001.sh new file mode 100755 index 00000000000..da8dcf188df --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.multi.001.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, removeip" + +setup_ctdb + +public_address=$(ctdb_get_1_public_address) + +ok_null + +simple_test_event "takeip" $public_address +simple_test_event "releaseip" $public_address diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.001.sh b/ctdb/tests/eventscripts/10.interface.releaseip.001.sh new file mode 100755 index 00000000000..934b3dc30ec --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.releaseip.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "error - no args given" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +required_result 1 "ERROR: must supply interface, IP and maskbits" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.002.sh b/ctdb/tests/eventscripts/10.interface.releaseip.002.sh new file mode 100755 index 00000000000..9bcb7f11d67 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.releaseip.002.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "error - remove a non-existent ip" + +setup_ctdb + +public_address=$(ctdb_get_1_public_address) +ip="${public_address% *}" ; ip="${ip#* }" + +required_result 1 <<EOF +RTNETLINK answers: Cannot assign requested address +Failed to del ${ip} on dev ${public_address%% *} +EOF + +simple_test $public_address diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.010.sh b/ctdb/tests/eventscripts/10.interface.releaseip.010.sh new file mode 100755 index 00000000000..b6d9c7a8bd6 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.releaseip.010.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Release 1 IP, 10 connections killed OK" + +setup_ctdb + +ctdb_get_1_public_address | +while read dev ip bits ; do + ip addr add "${ip}/${bits}" dev "$dev" + + # Setup 10 fake connections... + count=10 + out="" + nl=" +" + i=0 + while [ $i -lt $count ] ; do + echo "${ip}:445 10.254.254.1:1230${i}" + # Expected output for killing this connection + out="${out}${out:+${nl}}Killing TCP connection 10.254.254.1:1230${i} ${ip}:445" + i=$(($i + 1)) + done >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" + + ok <<EOF +$out +Killed $count TCP connections to released IP $ip +EOF + + simple_test $dev $ip $bits +done diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.011.sh b/ctdb/tests/eventscripts/10.interface.releaseip.011.sh new file mode 100755 index 00000000000..17b7421b669 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.releaseip.011.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Release 1 IP, 10 connections killed, 1 fails" + +setup_ctdb + +ctdb_get_1_public_address | +while read dev ip bits ; do + ip addr add "${ip}/${bits}" dev "$dev" + + # Setup 10 fake connections... + count=10 + out="" + nl=" +" + i=0 + while [ $i -lt $count ] ; do + echo "${ip}:445 10.254.254.1:1230${i}" + # Expected output for killing this connection + out="${out}${out:+${nl}}Killing TCP connection 10.254.254.1:1230${i} ${ip}:445" + i=$(($i + 1)) + done >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" + + # Note that the fake TCP killing done by the "ctdb killtcp" stub + # can only kill conections in the file, so killing this connection + # will never succeed so it will look like a time out. + FAKE_NETSTAT_TCP_ESTABLISHED="${ip}:445|10.254.254.1:43210" + + ok <<EOF +Killing TCP connection 10.254.254.1:43210 ${ip}:445 +$out +Waiting for 1 connections to be killed for IP ${ip} +Waiting for 1 connections to be killed for IP ${ip} +Waiting for 1 connections to be killed for IP ${ip} +Timed out killing tcp connections for IP $ip +EOF + + simple_test $dev $ip $bits +done diff --git a/ctdb/tests/eventscripts/10.interface.startup.001.sh b/ctdb/tests/eventscripts/10.interface.startup.001.sh new file mode 100755 index 00000000000..42ef42d81b4 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.startup.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "no public addresses" + +setup_ctdb + +export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.startup.002.sh b/ctdb/tests/eventscripts/10.interface.startup.002.sh new file mode 100755 index 00000000000..ba33f927c3f --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.startup.002.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up" + +setup_ctdb + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.takeip.001.sh b/ctdb/tests/eventscripts/10.interface.takeip.001.sh new file mode 100755 index 00000000000..934b3dc30ec --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.takeip.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "error - no args given" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +required_result 1 "ERROR: must supply interface, IP and maskbits" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.takeip.002.sh b/ctdb/tests/eventscripts/10.interface.takeip.002.sh new file mode 100755 index 00000000000..8960b08997d --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.takeip.002.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "add an ip" + +setup_ctdb + +public_address=$(ctdb_get_1_public_address) + +ok_null + +simple_test $public_address diff --git a/ctdb/tests/eventscripts/10.interface.takeip.003.sh b/ctdb/tests/eventscripts/10.interface.takeip.003.sh new file mode 100755 index 00000000000..203cff05be1 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.takeip.003.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "error - add same IP twice" + +setup_ctdb + +public_address=$(ctdb_get_1_public_address) +dev="${public_address%% *}" +t="${public_address#* }" +ip="${t% *}" +bits="${t#* }" + +# This is a bit gross and contrived. The method of quoting the error +# message so it makes it to required_result() is horrible. Hopefully +# improvements will come. + +err2="\ +RTNETLINK answers: File exists +Failed to add $ip/$bits on dev $dev" + +#EVENTSCRIPTS_TESTS_TRACE="sh -x" +iterate_test -- $public_address -- 2 "ok_null" \ + 2 'required_result 1 "$err2"' diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.001.sh b/ctdb/tests/eventscripts/13.per_ip_routing.001.sh new file mode 100755 index 00000000000..8523c1010f1 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.001.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "not configured" + +setup_ctdb + +ok_null +simple_test_event "takeip" + +ok_null +simple_test_event "ipreallocate" + +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.002.sh b/ctdb/tests/eventscripts/13.per_ip_routing.002.sh new file mode 100755 index 00000000000..d6320c65d89 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.002.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "missing config file" + +setup_ctdb +setup_ctdb_policy_routing + +# Error because policy routing is configured but the configuration +# file is missing. +required_result 1 <<EOF +error: CTDB_PER_IP_ROUTING_CONF=${CTDB_BASE}/policy_routing file not found +EOF + +for i in "startup" "ipreallocated" "monitor" ; do + simple_test_event "$i" +done + diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.003.sh b/ctdb/tests/eventscripts/13.per_ip_routing.003.sh new file mode 100755 index 00000000000..bb2c4b70fb2 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.003.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "empty config, ipreallocated" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 0 + +# ipreallocated should silently add any missing routes +ok_null +simple_test_event "ipreallocated" + +# empty configuration file should mean there are no routes +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.004.sh b/ctdb/tests/eventscripts/13.per_ip_routing.004.sh new file mode 100755 index 00000000000..4595313b715 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.004.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "empty config, takeip" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 0 + +public_address=$(ctdb_get_1_public_address) + +ok_null +simple_test_event "takeip" $public_address + +# empty configuration file should mean there are no routes +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.005.sh b/ctdb/tests/eventscripts/13.per_ip_routing.005.sh new file mode 100755 index 00000000000..9495cc54ce7 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.005.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, takeip" + +setup_ctdb +setup_ctdb_policy_routing + +# Configuration for 1 IP +create_policy_routing_config 1 default + +# takeip should add routes for the given address +ctdb_get_1_public_address | +while read dev ip bits ; do + ok_null + simple_test_event "takeip" $dev $ip $bits +done + +# Should have routes for 1 IP +check_routes 1 default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.006.sh b/ctdb/tests/eventscripts/13.per_ip_routing.006.sh new file mode 100755 index 00000000000..b93b6cdea1e --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.006.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, takeip, releaseip" + +setup_ctdb +setup_ctdb_policy_routing + +# create config for 1 IP +create_policy_routing_config 1 default + +ctdb_get_1_public_address | +while read dev ip bits ; do + # takeip adds routes + ok_null + simple_test_event "takeip" $dev $ip $bits + + # releaseip removes routes + ok_null + simple_test_event "releaseip" $dev $ip $bits +done + +# should have no routes +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.007.sh b/ctdb/tests/eventscripts/13.per_ip_routing.007.sh new file mode 100755 index 00000000000..096bc96d996 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.007.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, ipreallocated" + +setup_ctdb +setup_ctdb_policy_routing + +# create config for 1 IP +create_policy_routing_config 1 default + +# no takeip, but ipreallocated should add any missing routes +ok_null +simple_test_event "ipreallocated" + +# should have routes for 1 IP +check_routes 1 default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.008.sh b/ctdb/tests/eventscripts/13.per_ip_routing.008.sh new file mode 100755 index 00000000000..9bb0c195fdd --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.008.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, takeip twice" + +setup_ctdb +setup_ctdb_policy_routing + +# create config for 1 IP +create_policy_routing_config 1 default + +ctdb_get_1_public_address | +while read dev ip bits ; do + ok_null + simple_test_event "takeip" $dev $ip $bits + + # 2nd takeip event for the same IP should be a no-op + ok_null + simple_test_event "takeip" $dev $ip $bits +done + +# should be routes for 1 IP +check_routes 1 default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.009.sh b/ctdb/tests/eventscripts/13.per_ip_routing.009.sh new file mode 100755 index 00000000000..cbea1ade61c --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.009.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "All IPs configured, takeip 1 address" + +setup_ctdb +setup_ctdb_policy_routing + +# configure all addresses +create_policy_routing_config all default + +# add routes for all 1 IP +ctdb_get_1_public_address | +while read dev ip bits ; do + ok_null + simple_test_event "takeip" $dev $ip $bits +done + +# for 1 IP +check_routes 1 default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.010.sh b/ctdb/tests/eventscripts/13.per_ip_routing.010.sh new file mode 100755 index 00000000000..d11585ec510 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.010.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "All IPs configured, takeip on all nodes" + +setup_ctdb +setup_ctdb_policy_routing + +# create config for all IPs +create_policy_routing_config all default + +ctdb_get_my_public_addresses | +while read dev ip bits ; do + ok_null + simple_test_event "takeip" $dev $ip $bits +done + +# should have routes for all IPs +check_routes all default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.011.sh b/ctdb/tests/eventscripts/13.per_ip_routing.011.sh new file mode 100755 index 00000000000..d8ec9ac2106 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.011.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "__auto_link_local__, takeip all on node" + +setup_ctdb +setup_ctdb_policy_routing + +# do link local fu instead of creating configuration +export CTDB_PER_IP_ROUTING_CONF="__auto_link_local__" + +# add routes for all addresses +ctdb_get_my_public_addresses | +while read dev ip bits ; do + ok_null + simple_test_event "takeip" $dev $ip $bits +done + +check_routes all diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.012.sh b/ctdb/tests/eventscripts/13.per_ip_routing.012.sh new file mode 100755 index 00000000000..6c8a6ab4b33 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.012.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, takeip, releaseip, ipreallocated" + +# This partly tests the test infrastructure. If the (stub) "ctdb +# moveip" doesn't do anything then the IP being released will still be +# on the node and the ipreallocated event will add the routes back. + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 1 default + +ctdb_get_1_public_address | +while read dev ip bits ; do + ok_null + simple_test_event "takeip" $dev $ip $bits + + ok_null + ctdb moveip $ip 1 + simple_test_event "releaseip" $dev $ip $bits + + ok_null + simple_test_event "ipreallocated" +done + +# all routes should have been removed and not added back +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.013.sh b/ctdb/tests/eventscripts/13.per_ip_routing.013.sh new file mode 100755 index 00000000000..567622edd93 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.013.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, releaseip of unassigned" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 1 default + +ctdb_get_1_public_address | +while read dev ip bits ; do + ok <<EOF +WARNING: Failed to delete policy routing rule + Command "ip rule del from $ip pref $CTDB_PER_IP_ROUTING_RULE_PREF table ctdb.$ip" failed: + RTNETLINK answers: No such file or directory +EOF + + simple_test_event "releaseip" $dev $ip $bits +done + +# there should be no routes +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.014.sh b/ctdb/tests/eventscripts/13.per_ip_routing.014.sh new file mode 100755 index 00000000000..ee08c36fa32 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.014.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, takeip, moveip, ipreallocated" + +# We move the IP to another node but don't run releaseip. +# ipreallocated should remove the bogus routes. + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 1 default + +ctdb_get_1_public_address | +while read dev ip bits ; do + ok_null + # Set up the routes for an IP that we have + simple_test_event "takeip" $dev $ip $bits + + # Now move that IPs but don't run the associated "releaseip" + ctdb moveip $ip 1 + + # This should handle removal of the routes + ok "Removing ip rule/routes for unhosted public address $ip" + simple_test_event "ipreallocated" +done + +# no routes left +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.015.sh b/ctdb/tests/eventscripts/13.per_ip_routing.015.sh new file mode 100755 index 00000000000..2b9ecba0534 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.015.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, releaseip of unassigned" + +setup_ctdb +setup_ctdb_policy_routing + +export IP_ROUTE_BAD_TABLE_ID=true + +create_policy_routing_config 1 default + +ctdb_get_1_public_address | +{ + read dev ip bits + + ok <<EOF +WARNING: Failed to delete policy routing rule + Command "ip rule del from $ip pref $CTDB_PER_IP_ROUTING_RULE_PREF table ctdb.$ip" failed: + Error: argument ctdb.$ip is wrong: invalid table ID + Error: argument ctdb.$ip is wrong: table id value is invalid +EOF + + simple_test_event "releaseip" $dev $ip $bits +} + + +# there should be no routes +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.016.sh b/ctdb/tests/eventscripts/13.per_ip_routing.016.sh new file mode 100755 index 00000000000..85320b65487 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.016.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "empty config, reconfigure, NOOP" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 0 + +ok "Reconfiguring service \"${service_name}\"..." +simple_test_event "reconfigure" + +check_routes 0 diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.017.sh b/ctdb/tests/eventscripts/13.per_ip_routing.017.sh new file mode 100755 index 00000000000..8870015501f --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.017.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, reconfigure" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 1 default + +# no takeip, but reconfigure should add any missing routes +ok "Reconfiguring service \"${service_name}\"..." +simple_test_event "reconfigure" + +check_routes 1 default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.018.sh b/ctdb/tests/eventscripts/13.per_ip_routing.018.sh new file mode 100755 index 00000000000..ce919890c0b --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.018.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, ipreallocated, more routes, reconfigure" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 1 + +# no takeip, but ipreallocated should add any missing routes +ok_null +simple_test_event "ipreallocated" + +create_policy_routing_config 1 default + +# reconfigure should update routes even though rules are unchanged +ok "Reconfiguring service \"${service_name}\"..." +simple_test_event "reconfigure" + +check_routes 1 default diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.019.sh b/ctdb/tests/eventscripts/13.per_ip_routing.019.sh new file mode 100755 index 00000000000..072c929d8b4 --- /dev/null +++ b/ctdb/tests/eventscripts/13.per_ip_routing.019.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 IP configured, ipreallocated, less routes, reconfigure" + +setup_ctdb +setup_ctdb_policy_routing + +create_policy_routing_config 1 default + +# no takeip, but ipreallocated should add any missing routes +ok_null +simple_test_event "ipreallocated" + +# rewrite the configuration to take out the default routes, as per the +# above change to $args +create_policy_routing_config 1 + +# reconfigure should update routes even though rules are unchanged +ok "Reconfiguring service \""${service_name}\""..." +simple_test_event "reconfigure" + +check_routes 1 diff --git a/ctdb/tests/eventscripts/20.multipathd.monitor.001.sh b/ctdb/tests/eventscripts/20.multipathd.monitor.001.sh new file mode 100755 index 00000000000..4eafefc60bb --- /dev/null +++ b/ctdb/tests/eventscripts/20.multipathd.monitor.001.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "No multipath devices configure to check" + +setup_multipathd + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/20.multipathd.monitor.002.sh b/ctdb/tests/eventscripts/20.multipathd.monitor.002.sh new file mode 100755 index 00000000000..fbfe9528430 --- /dev/null +++ b/ctdb/tests/eventscripts/20.multipathd.monitor.002.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 multipath devices configure to check, all up" + +setup_multipathd "mpatha" "mpathb" "mpathc" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/20.multipathd.monitor.003.sh b/ctdb/tests/eventscripts/20.multipathd.monitor.003.sh new file mode 100755 index 00000000000..d9a212555e6 --- /dev/null +++ b/ctdb/tests/eventscripts/20.multipathd.monitor.003.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 multipath devices configure to check, one down" + +setup_multipathd "mpatha" "!mpathb" "mpathc" + +required_result 1 <<EOF +ERROR: multipath device "mpathb" has no active paths +multipath monitoring failed +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/20.multipathd.monitor.004.sh b/ctdb/tests/eventscripts/20.multipathd.monitor.004.sh new file mode 100755 index 00000000000..5f45c73c5a8 --- /dev/null +++ b/ctdb/tests/eventscripts/20.multipathd.monitor.004.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 multipath devices configure to check, multipath hangs" + +setup_multipathd "mpatha" "!mpathb" "mpathc" +export FAKE_MULTIPATH_HANG="yes" + +required_result 1 <<EOF +ERROR: callout to multipath checks hung +multipath monitoring failed +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/40.vsftpd.monitor.001.sh b/ctdb/tests/eventscripts/40.vsftpd.monitor.001.sh new file mode 100755 index 00000000000..fdad12ae9a6 --- /dev/null +++ b/ctdb/tests/eventscripts/40.vsftpd.monitor.001.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "not managed, check no-op" + +setup_vsftpd "down" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/41.httpd.monitor.001.sh b/ctdb/tests/eventscripts/41.httpd.monitor.001.sh new file mode 100755 index 00000000000..f400eaa89b4 --- /dev/null +++ b/ctdb/tests/eventscripts/41.httpd.monitor.001.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "not managed, check no-op" + +setup_httpd "down" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/49.winbind.monitor.001.sh b/ctdb/tests/eventscripts/49.winbind.monitor.001.sh new file mode 100755 index 00000000000..94253d82f6b --- /dev/null +++ b/ctdb/tests/eventscripts/49.winbind.monitor.001.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "not managed, check no-op" + +setup_winbind "down" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/49.winbind.monitor.050.sh b/ctdb/tests/eventscripts/49.winbind.monitor.050.sh new file mode 100755 index 00000000000..02589b33e89 --- /dev/null +++ b/ctdb/tests/eventscripts/49.winbind.monitor.050.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "auto-start, simple" + +setup_winbind "down" + +export CTDB_SERVICE_AUTOSTARTSTOP="yes" +export CTDB_MANAGED_SERVICES="foo winbind bar" + +ok <<EOF +Starting service "winbind" - now managed +&Starting winbind: OK +EOF +simple_test diff --git a/ctdb/tests/eventscripts/49.winbind.monitor.051.sh b/ctdb/tests/eventscripts/49.winbind.monitor.051.sh new file mode 100755 index 00000000000..fbad928acad --- /dev/null +++ b/ctdb/tests/eventscripts/49.winbind.monitor.051.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "auto-stop, simple" + +setup_winbind + +export CTDB_SERVICE_AUTOSTARTSTOP="yes" +export CTDB_MANAGED_SERVICES="foo" +unset CTDB_MANAGES_WINBIND + +ok <<EOF +Stopping service "winbind" - no longer managed +&Stopping winbind: OK +EOF +simple_test diff --git a/ctdb/tests/eventscripts/49.winbind.monitor.101.sh b/ctdb/tests/eventscripts/49.winbind.monitor.101.sh new file mode 100755 index 00000000000..ec2952b76b2 --- /dev/null +++ b/ctdb/tests/eventscripts/49.winbind.monitor.101.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all OK" + +setup_winbind + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/49.winbind.monitor.102.sh b/ctdb/tests/eventscripts/49.winbind.monitor.102.sh new file mode 100755 index 00000000000..e4a4cacb3f2 --- /dev/null +++ b/ctdb/tests/eventscripts/49.winbind.monitor.102.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "winbind down" + +setup_winbind +wbinfo_down + +required_result 1 "ERROR: wbinfo -p returned error" + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.001.sh b/ctdb/tests/eventscripts/50.samba.monitor.001.sh new file mode 100755 index 00000000000..ac3708f5fd4 --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.001.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "not managed, check no-op" + +setup_samba "down" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.050.sh b/ctdb/tests/eventscripts/50.samba.monitor.050.sh new file mode 100755 index 00000000000..69530f3dac4 --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.050.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "auto-start, simple" + +setup_samba "down" + +export CTDB_SERVICE_AUTOSTARTSTOP="yes" +export CTDB_MANAGED_SERVICES="foo samba winbind bar" + +ok <<EOF +Starting service "samba" - now managed +&Starting smb: OK +EOF +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.051.sh b/ctdb/tests/eventscripts/50.samba.monitor.051.sh new file mode 100755 index 00000000000..04c1fce1141 --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.051.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "auto-stop, simple" + +setup_samba + +export CTDB_SERVICE_AUTOSTARTSTOP="yes" +export CTDB_MANAGED_SERVICES="foo" +unset CTDB_MANAGES_SAMBA + +ok <<EOF +Stopping service "samba" - no longer managed +&Stopping smb: OK +EOF +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.101.sh b/ctdb/tests/eventscripts/50.samba.monitor.101.sh new file mode 100755 index 00000000000..cf3b53a8f17 --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.101.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all OK" + +setup_samba + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.103.sh b/ctdb/tests/eventscripts/50.samba.monitor.103.sh new file mode 100755 index 00000000000..6f71a96760a --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.103.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "port 445 down" + +setup_samba +tcp_port_down 445 + +required_result 1 "ERROR: samba tcp port 445 is not responding" + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.104.sh b/ctdb/tests/eventscripts/50.samba.monitor.104.sh new file mode 100755 index 00000000000..9de022376be --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.104.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "port 139 down" + +setup_samba +tcp_port_down 139 + +required_result 1 "ERROR: samba tcp port 139 is not responding" + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.105.sh b/ctdb/tests/eventscripts/50.samba.monitor.105.sh new file mode 100755 index 00000000000..9936eff8064 --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.105.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "non-existent share path" + +setup_samba +shares_missing "ERROR: samba directory \"%s\" not available" 2 + +required_result 1 "$MISSING_SHARES_TEXT" + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.106.sh b/ctdb/tests/eventscripts/50.samba.monitor.106.sh new file mode 100755 index 00000000000..8fabfb33b6d --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.106.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "non-existent share - not checked" + +setup_samba +shares_missing "ERROR: samba directory \"%s\" not available" 2 + +export CTDB_SAMBA_SKIP_SHARE_CHECK="yes" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/50.samba.monitor.107.sh b/ctdb/tests/eventscripts/50.samba.monitor.107.sh new file mode 100755 index 00000000000..573ff80e4fc --- /dev/null +++ b/ctdb/tests/eventscripts/50.samba.monitor.107.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "port 139 down, default tcp checker, debug" + +export CTDB_SCRIPT_DEBUGLEVEL=4 + +setup_samba +tcp_port_down 139 + +required_result 1 <<EOF +ERROR: samba tcp port 139 is not responding +DEBUG: "ctdb checktcpport 139" was able to bind to port +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/60.ganesha.monitor.101.sh b/ctdb/tests/eventscripts/60.ganesha.monitor.101.sh new file mode 100755 index 00000000000..d68ad6a4a2f --- /dev/null +++ b/ctdb/tests/eventscripts/60.ganesha.monitor.101.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all services available" + +setup_nfs_ganesha + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh b/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh new file mode 100755 index 00000000000..95ce450e79f --- /dev/null +++ b/ctdb/tests/eventscripts/60.ganesha.monitor.131.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "rquotad down" + +setup_nfs_ganesha +rpc_services_down "rquotad" + +ok<<EOF +ERROR: rquotad failed RPC check: +rpcinfo: RPC: Program not registered +program rquotad version 1 is not available +Trying to restart rquotad [rpc.rquotad] +EOF + +simple_test diff --git a/ctdb/tests/eventscripts/60.ganesha.monitor.141.sh b/ctdb/tests/eventscripts/60.ganesha.monitor.141.sh new file mode 100755 index 00000000000..9cd82f84cc2 --- /dev/null +++ b/ctdb/tests/eventscripts/60.ganesha.monitor.141.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "statd down, 6 iterations" + +# statd fails and attempts to restart it fail. + +setup_nfs_ganesha +rpc_services_down "status" + +ok_null +simple_test || exit $? + +ok<<EOF +Trying to restart statd [rpc.statd] +EOF +simple_test || exit $? + +ok_null +simple_test || exit $? + +ok<<EOF +ERROR: status failed RPC check: +rpcinfo: RPC: Program not registered +program status version 1 is not available +Trying to restart statd [rpc.statd] +EOF +simple_test || exit $? + +ok_null +simple_test || exit $? + +required_result 1 <<EOF +ERROR: status failed RPC check: +rpcinfo: RPC: Program not registered +program status version 1 is not available +EOF +simple_test || exit $? diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.001.sh b/ctdb/tests/eventscripts/60.nfs.monitor.001.sh new file mode 100755 index 00000000000..c62e5cf8789 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.001.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "not managed, check no-op" + +setup_nfs "down" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.101.sh b/ctdb/tests/eventscripts/60.nfs.monitor.101.sh new file mode 100755 index 00000000000..1a689278106 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.101.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all services available" + +setup_nfs + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.102.sh b/ctdb/tests/eventscripts/60.nfs.monitor.102.sh new file mode 100755 index 00000000000..bb988aa9a2d --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.102.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all services available, check nfsd thread count, count matches" + +setup_nfs + +CTDB_MONITOR_NFS_THREAD_COUNT="yes" +RPCNFSDCOUNT=8 +FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6 7 8" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.103.sh b/ctdb/tests/eventscripts/60.nfs.monitor.103.sh new file mode 100755 index 00000000000..75d7291fb85 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.103.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all services available, check nfsd thread count, not enough threads" + +setup_nfs + +CTDB_MONITOR_NFS_THREAD_COUNT="yes" +RPCNFSDCOUNT=8 +FAKE_NFSD_THREAD_PIDS="1 2 3 4 5" + +ok "Attempting to correct number of nfsd threads from 5 to 8" + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.104.sh b/ctdb/tests/eventscripts/60.nfs.monitor.104.sh new file mode 100755 index 00000000000..a052be81319 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.104.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +# Add this extra test to catch a design change where we only ever +# increase the number of threads. That is, this test would need to be +# consciously removed. +define_test "all services available, check nfsd thread count, too many threads" + +setup_nfs + +CTDB_MONITOR_NFS_THREAD_COUNT="yes" +RPCNFSDCOUNT=4 +FAKE_NFSD_THREAD_PIDS="1 2 3 4 5 6" + +ok "Attempting to correct number of nfsd threads from 6 to 4" + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.111.sh b/ctdb/tests/eventscripts/60.nfs.monitor.111.sh new file mode 100755 index 00000000000..414fcc8066b --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.111.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "knfsd down, 1 iteration" + +setup_nfs +rpc_services_down "nfs" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.112.sh b/ctdb/tests/eventscripts/60.nfs.monitor.112.sh new file mode 100755 index 00000000000..49ee3357498 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.112.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "knfsd down, 6 iterations" + +# knfsd fails and attempts to restart it fail. + +setup_nfs +rpc_services_down "nfs" + +iterate_test 10 'rpc_set_service_failure_response "nfsd"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.113.sh b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh new file mode 100755 index 00000000000..505df1b5275 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.113.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "knfsd down, 6 iterations, dump 5 threads, none hung" + +# knfsd fails and attempts to restart it fail. +setup_nfs +rpc_services_down "nfs" + +# Additionally, any hung threads should have stack traces dumped. +CTDB_NFS_DUMP_STUCK_THREADS=5 +FAKE_NFSD_THREAD_PIDS="" + +iterate_test 10 'rpc_set_service_failure_response "nfsd"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.114.sh b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh new file mode 100755 index 00000000000..496f5e7dee2 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.114.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "knfsd down, 6 iterations, dump 5 threads, 3 hung" + +# knfsd fails and attempts to restart it fail. +setup_nfs +rpc_services_down "nfs" + +# Additionally, any hung threads should have stack traces dumped. +CTDB_NFS_DUMP_STUCK_THREADS=5 +FAKE_NFSD_THREAD_PIDS="1001 1002 1003" + +iterate_test 10 'rpc_set_service_failure_response "nfsd"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.121.sh b/ctdb/tests/eventscripts/60.nfs.monitor.121.sh new file mode 100755 index 00000000000..6d27f60b27a --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.121.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "lockd down, 15 iterations" + +# This simulates an ongoing failure in the eventscript's automated +# attempts to restart the service. That is, the eventscript is unable +# to restart the service. + +setup_nfs +rpc_services_down "nlockmgr" + +#EVENTSCRIPTS_TESTS_TRACE="sh -x" +iterate_test 15 "ok_null" \ + 10 "rpc_set_service_failure_response 'lockd'" \ + 15 "rpc_set_service_failure_response 'lockd'" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.122.sh b/ctdb/tests/eventscripts/60.nfs.monitor.122.sh new file mode 100755 index 00000000000..fc5cea87c75 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.122.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "lockd down, 15 iterations, back up after 10" + +# This simulates a success the eventscript's automated attempts to +# restart the service. + +setup_nfs +rpc_services_down "nlockmgr" + +# Iteration 10 should try to restart rpc.lockd. However, our test +# stub rpc.lockd does nothing, so we have to explicitly flag it as up. + +iterate_test 15 "ok_null" \ + 10 "rpc_set_service_failure_response 'lockd'" \ + 11 "rpc_services_up nlockmgr" + diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.131.sh b/ctdb/tests/eventscripts/60.nfs.monitor.131.sh new file mode 100755 index 00000000000..1cf72a92844 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.131.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "rquotad down, 5 iterations" + +setup_nfs +rpc_services_down "rquotad" + +iterate_test 5 'rpc_set_service_failure_response "rquotad"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.132.sh b/ctdb/tests/eventscripts/60.nfs.monitor.132.sh new file mode 100755 index 00000000000..b8f3f2b0411 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.132.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "rquotad down, 5 iterations, back up after 1" + +# rquotad fails once but then comes back of its own accord after 1 +# failure. + +setup_nfs +rpc_services_down "rquotad" + +iterate_test 5 'ok_null' \ + 1 'rpc_set_service_failure_response "rquotad"' \ + 2 'rpc_services_up "rquotad"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.141.sh b/ctdb/tests/eventscripts/60.nfs.monitor.141.sh new file mode 100755 index 00000000000..c77b1a7b052 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.141.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "statd down, 6 iterations" + +# statd fails and attempts to restart it fail. + +setup_nfs +rpc_services_down "status" + +iterate_test 6 'ok_null' \ + 2 'rpc_set_service_failure_response "statd"' \ + 4 'rpc_set_service_failure_response "statd"' \ + 6 'rpc_set_service_failure_response "statd"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.142.sh b/ctdb/tests/eventscripts/60.nfs.monitor.142.sh new file mode 100755 index 00000000000..4373d8d6426 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.142.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "statd down, 8 iterations, back up after 2" + +# statd fails and the first attempt to restart it succeeds. + +setup_nfs +rpc_services_down "status" + +iterate_test 8 'ok_null' \ + 2 'rpc_set_service_failure_response "statd"' \ + 3 'rpc_services_up "status"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.151.sh b/ctdb/tests/eventscripts/60.nfs.monitor.151.sh new file mode 100755 index 00000000000..ea9aa7830ec --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.151.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "mountd down, 1 iteration" + +setup_nfs +rpc_services_down "mountd" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.152.sh b/ctdb/tests/eventscripts/60.nfs.monitor.152.sh new file mode 100755 index 00000000000..c4eb4194ae2 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.152.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "mountd down, 10 iterations" + +# This simulates an ongoing failure in the eventscript's automated +# attempts to restart the service. That is, the eventscript is unable +# to restart the service. + +setup_nfs +rpc_services_down "mountd" + +iterate_test 10 "ok_null" \ + 5 "rpc_set_service_failure_response 'mountd'" \ + 10 "rpc_set_service_failure_response 'mountd'" + +#export FAKE_NETSTAT_TCP_ESTABLISHED="10.0.0.1:2049|10.254.254.1:12301 10.0.0.1:2049|10.254.254.1:12302 10.0.0.1:2049|10.254.254.1:12303 10.0.0.1:2049|10.254.254.2:12304 10.0.0.1:2049|10.254.254.2:12305" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.153.sh b/ctdb/tests/eventscripts/60.nfs.monitor.153.sh new file mode 100755 index 00000000000..cf33e39bbc8 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.153.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "mountd down, 10 iterations, back up after 5" + +setup_nfs +rpc_services_down "mountd" + +# Iteration 5 should try to restart rpc.mountd. However, our test +# stub rpc.mountd does nothing, so we have to explicitly flag it as +# up. +iterate_test 10 "ok_null" \ + 5 "rpc_set_service_failure_response 'mountd'" \ + 6 "rpc_services_up mountd" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.161.sh b/ctdb/tests/eventscripts/60.nfs.monitor.161.sh new file mode 100755 index 00000000000..1e07c181c2e --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.161.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "2nd share missing" + +setup_nfs + +shares_missing "ERROR: nfs directory \"%s\" not available" 2 + +required_result 1 "$MISSING_SHARES_TEXT" + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.162.sh b/ctdb/tests/eventscripts/60.nfs.monitor.162.sh new file mode 100755 index 00000000000..ccd4ca84ea1 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.162.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "2nd share missing, skipping share checks" + +setup_nfs +export CTDB_NFS_SKIP_SHARE_CHECK="yes" + +shares_missing "ERROR: nfs directory \"%s\" not available" 2 + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.multi.001.sh b/ctdb/tests/eventscripts/60.nfs.multi.001.sh new file mode 100755 index 00000000000..f983df7615a --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.001.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, ipreallocated -> reconfigure" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +ok_null + +simple_test_event "takeip" $public_address + +ok <<EOF +Reconfiguring service "nfs"... +EOF + +simple_test_event "ipreallocated" diff --git a/ctdb/tests/eventscripts/60.nfs.multi.002.sh b/ctdb/tests/eventscripts/60.nfs.multi.002.sh new file mode 100755 index 00000000000..350c1bc726d --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.002.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, monitor -> reconfigure" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +ok_null + +simple_test_event "takeip" $public_address + +# This currently assumes that ctdb scriptstatus will always return a +# good status (when replaying). That should change and we will need +# to split this into 2 tests. +ok <<EOF +Reconfiguring service "nfs"... +Replaying previous status for this script due to reconfigure... +EOF + +simple_test_event "monitor" diff --git a/ctdb/tests/eventscripts/60.nfs.multi.003.sh b/ctdb/tests/eventscripts/60.nfs.multi.003.sh new file mode 100755 index 00000000000..68f45ab15dd --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.003.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, monitor -> reconfigure, replay error" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="foo: bar error occurred" + +ok_null + +simple_test_event "takeip" $public_address + +ctdb_fake_scriptstatus 1 "ERROR" "$err" + +required_result 1 <<EOF +Reconfiguring service "nfs"... +Replaying previous status for this script due to reconfigure... +$err +EOF + +simple_test_event "monitor" diff --git a/ctdb/tests/eventscripts/60.nfs.multi.004.sh b/ctdb/tests/eventscripts/60.nfs.multi.004.sh new file mode 100755 index 00000000000..b071ec8bd9b --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.004.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, monitor -> reconfigure, replay timedout" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="waiting, waiting..." + +ok_null + +simple_test_event "takeip" $public_address + +ctdb_fake_scriptstatus -62 "TIMEDOUT" "$err" + +required_result 1 <<EOF +Reconfiguring service "nfs"... +Replaying previous status for this script due to reconfigure... +[Replay of TIMEDOUT scriptstatus - note incorrect return code.] $err +EOF + +simple_test_event "monitor" diff --git a/ctdb/tests/eventscripts/60.nfs.multi.005.sh b/ctdb/tests/eventscripts/60.nfs.multi.005.sh new file mode 100755 index 00000000000..82802aa01e8 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.005.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, monitor -> reconfigure, replay disabled" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="" + +ok_null + +simple_test_event "takeip" $public_address + +ctdb_fake_scriptstatus -8 "DISABLED" "$err" + +ok <<EOF +Reconfiguring service "nfs"... +Replaying previous status for this script due to reconfigure... +[Replay of DISABLED scriptstatus - note incorrect return code.] $err +EOF + +simple_test_event "monitor" diff --git a/ctdb/tests/eventscripts/60.nfs.multi.006.sh b/ctdb/tests/eventscripts/60.nfs.multi.006.sh new file mode 100755 index 00000000000..84bb9ef0668 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.006.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "reconfigure (synthetic), twice" +# This checks that the lock is released... + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="" + +ok <<EOF +Reconfiguring service "nfs"... +EOF + +simple_test_event "reconfigure" +simple_test_event "reconfigure" diff --git a/ctdb/tests/eventscripts/README b/ctdb/tests/eventscripts/README new file mode 100644 index 00000000000..266c530bad0 --- /dev/null +++ b/ctdb/tests/eventscripts/README @@ -0,0 +1,47 @@ +eventscript unit tests +====================== + +This directory contains some eventscript unit tests for CTDB. These +tests can be run as a non-privileged user. There are a lot of stub +implementations of commands (located in stubs/) used to make the +eventscripts think they're running against a real system. + +Test case filenames look like: + + <eventscript>.<event>.NNN.sh + +The test helper functions will run <eventscript> with specified +options. If using the simple_test() or iterate_test() helper +functions then the 1st <event> argument is automatically passed. When +simple_test_event() is used the event name must be explicitly passed +as the 1st argument - this is more flexible and supports multiple +events per test. + +Examples: + +* ../run_tests.sh . + + Run all tests, displaying minimal output. + +* ../run_tests.sh -s . + + Run all tests, displaying minimal output and a summary. + +* ../run_tests.sh -s ./10.interface.*.sh + + Run all the tests against the 10.interface eventscript. + +* ../run_tests.sh -v -s . + + Run all tests, displaying extra output and a summary. + +* ../run_tests.sh -sq . + + Run all tests, displaying only a summary. + +* ../run_tests.sh -X ./10.interface.startup.002.sh + + Run a test and have the eventscript itself run with "sh -x". This + will usually make a test fail because the (undesirable) trace output + will be included with the output of the eventscript. However, this + is useful for finding out why a test might be failing. diff --git a/ctdb/tests/eventscripts/etc-ctdb/events.d b/ctdb/tests/eventscripts/etc-ctdb/events.d new file mode 120000 index 00000000000..69d2396b339 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/events.d @@ -0,0 +1 @@ +../../../config/events.d
\ No newline at end of file diff --git a/ctdb/tests/eventscripts/etc-ctdb/functions b/ctdb/tests/eventscripts/etc-ctdb/functions new file mode 120000 index 00000000000..86ba904ac48 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/functions @@ -0,0 +1 @@ +../../../config/functions
\ No newline at end of file diff --git a/ctdb/tests/eventscripts/etc-ctdb/nfs-rpc-checks.d b/ctdb/tests/eventscripts/etc-ctdb/nfs-rpc-checks.d new file mode 120000 index 00000000000..991b966b114 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/nfs-rpc-checks.d @@ -0,0 +1 @@ +../../../config/nfs-rpc-checks.d
\ No newline at end of file diff --git a/ctdb/tests/eventscripts/etc-ctdb/public_addresses b/ctdb/tests/eventscripts/etc-ctdb/public_addresses new file mode 100644 index 00000000000..cd2f6be4e1e --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/public_addresses @@ -0,0 +1,9 @@ +10.0.0.1/24 dev123 +10.0.0.2/24 dev123 +10.0.0.3/24 dev123 +10.0.0.4/24 dev123 +10.0.0.5/24 dev123 +10.0.0.6/24 dev123 +10.0.1.1/24 dev456 +10.0.1.2/24 dev456 +10.0.1.3/24 dev456 diff --git a/ctdb/tests/eventscripts/etc-ctdb/rc.local b/ctdb/tests/eventscripts/etc-ctdb/rc.local new file mode 100755 index 00000000000..6052d87f130 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/rc.local @@ -0,0 +1,61 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Use a "service" command in $PATH if one exists. +service () +{ + if _t=$(which "service" 2>/dev/null) ; then + "$_t" "$@" + else + _nice="" + _service "$@" + fi +} + +nice_service () +{ + if _t=$(which "service" 2>/dev/null) ; then + nice "$_t" "$@" + else + _nice="nice" + _service "$@" + fi +} + +# Always succeeds +set_proc () { : ; } + +get_proc () +{ + case "$1" in + net/bonding/*) + cat "$FAKE_PROC_NET_BONDING/${1##*/}" + ;; + sys/net/ipv4/conf/all/arp_filter) + echo 1 + ;; + fs/nfsd/threads) + echo "$FAKE_NFSD_THREAD_PIDS" | wc -w + ;; + */stack) + echo "[<ffffffff87654321>] fake_stack_trace_for_pid_${1}+0x0/0xff" + ;; + meminfo) + echo "$FAKE_PROC_MEMINFO" + ;; + *) + echo "get_proc: \"$1\" not implemented" + exit 1 + esac +} + +# Always succeeds +iptables () { : ; } + +# Do not actually background - we want to see the output +background_with_logging () +{ + "$@" 2>&1 </dev/null | sed -e 's@^@\&@' +} + +CTDB_INIT_STYLE="redhat" +PATH="${EVENTSCRIPTS_PATH}:$PATH" diff --git a/ctdb/tests/eventscripts/etc-ctdb/statd-callout b/ctdb/tests/eventscripts/etc-ctdb/statd-callout new file mode 100755 index 00000000000..51779bd7ba6 --- /dev/null +++ b/ctdb/tests/eventscripts/etc-ctdb/statd-callout @@ -0,0 +1,5 @@ +#!/bin/sh + +# For now, always succeed. + +exit 0 diff --git a/ctdb/tests/eventscripts/etc/init.d/nfs b/ctdb/tests/eventscripts/etc/init.d/nfs new file mode 100755 index 00000000000..43eb308e340 --- /dev/null +++ b/ctdb/tests/eventscripts/etc/init.d/nfs @@ -0,0 +1,7 @@ +#!/bin/sh + +# This is not used. The fake "service" script is used instead. This +# is only needed to shut up functions like startstop_nfs(), which look +# for this script. + +exit 0 diff --git a/ctdb/tests/eventscripts/etc/init.d/nfslock b/ctdb/tests/eventscripts/etc/init.d/nfslock new file mode 100755 index 00000000000..43eb308e340 --- /dev/null +++ b/ctdb/tests/eventscripts/etc/init.d/nfslock @@ -0,0 +1,7 @@ +#!/bin/sh + +# This is not used. The fake "service" script is used instead. This +# is only needed to shut up functions like startstop_nfs(), which look +# for this script. + +exit 0 diff --git a/ctdb/tests/eventscripts/etc/samba/smb.conf b/ctdb/tests/eventscripts/etc/samba/smb.conf new file mode 100644 index 00000000000..da89db2b81f --- /dev/null +++ b/ctdb/tests/eventscripts/etc/samba/smb.conf @@ -0,0 +1,42 @@ +[global] + # enable clustering + clustering=yes + ctdb:registry.tdb=yes + + security = ADS + auth methods = guest sam winbind + + netbios name = cluster1 + workgroup = CLUSTER1 + realm = CLUSTER1.COM + server string = "Clustered Samba" + disable netbios = yes + disable spoolss = yes + fileid:mapping = fsname + use mmap = yes + gpfs:sharemodes = yes + gpfs:leases = yes + passdb backend = tdbsam + preferred master = no + kernel oplocks = yes + syslog = 1 + host msdfs = no + notify:inotify = no + vfs objects = shadow_copy2 syncops gpfs fileid + shadow:snapdir = .snapshots + shadow:fixinodes = yes + wide links = no + smbd:backgroundqueue = False + read only = no + use sendfile = yes + strict locking = yes + posix locking = yes + large readwrite = yes + force unknown acl user = yes + nfs4:mode = special + nfs4:chown = yes + nfs4:acedup = merge + nfs4:sidmap = /etc/samba/sidmap.tdb + map readonly = no + ea support = yes + dmapi support = no diff --git a/ctdb/tests/eventscripts/etc/sysconfig/ctdb b/ctdb/tests/eventscripts/etc/sysconfig/ctdb new file mode 100644 index 00000000000..4584c112fd1 --- /dev/null +++ b/ctdb/tests/eventscripts/etc/sysconfig/ctdb @@ -0,0 +1,2 @@ +CTDB_RECOVERY_LOCK="/some/place/on/shared/storage" +CTDB_DEBUGLEVEL=ERR diff --git a/ctdb/tests/eventscripts/etc/sysconfig/nfs b/ctdb/tests/eventscripts/etc/sysconfig/nfs new file mode 100644 index 00000000000..090d786ea68 --- /dev/null +++ b/ctdb/tests/eventscripts/etc/sysconfig/nfs @@ -0,0 +1,2 @@ +NFS_HOSTNAME="cluster1" +STATD_HOSTNAME="$NFS_HOSTNAME -H /etc/ctdb/statd-callout " diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh new file mode 100644 index 00000000000..e6186a07230 --- /dev/null +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -0,0 +1,1037 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Augment PATH with relevant stubs/ directories. We do this by actually +# setting PATH, and also by setting $EVENTSCRIPTS_PATH and then +# prepending that to $PATH in rc.local to avoid the PATH reset in +# functions. + +EVENTSCRIPTS_PATH="" + +if [ -d "${TEST_SUBDIR}/stubs" ] ; then + EVENTSCRIPTS_PATH="${TEST_SUBDIR}/stubs" +fi + +export EVENTSCRIPTS_PATH + +PATH="${EVENTSCRIPTS_PATH}:${PATH}" + +export EVENTSCRIPTS_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_eventscripts" +if [ -d "$EVENTSCRIPTS_TESTS_VAR_DIR" -a \ + "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/unit_eventscripts" ] ; then + rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR" +fi +mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR" +export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb" + +export CTDB_LOGFILE="${EVENTSCRIPTS_TESTS_VAR_DIR}/log.ctdb" +touch "$CTDB_LOGFILE" || die "Unable to create CTDB_LOGFILE=$CTDB_LOGFILE" + +if [ -d "${TEST_SUBDIR}/etc" ] ; then + cp -a "${TEST_SUBDIR}/etc" "$EVENTSCRIPTS_TESTS_VAR_DIR" + export CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc" +else + die "Unable to setup \$CTDB_ETCDIR" +fi + +if [ -d "${TEST_SUBDIR}/etc-ctdb" ] ; then + cp -prL "${TEST_SUBDIR}/etc-ctdb" "$EVENTSCRIPTS_TESTS_VAR_DIR" + export CTDB_BASE="${EVENTSCRIPTS_TESTS_VAR_DIR}/etc-ctdb" +else + die "Unable to set \$CTDB_BASE" +fi +export CTDB_BASE + +if [ ! -d "${CTDB_BASE}/events.d" ] ; then + cat <<EOF +ERROR: Directory ${CTDB_BASE}/events.d does not exist. + +That means that no eventscripts can be tested. + +One possible explanation: + + You have CTDB installed via RPMs (or similar), so the regular + CTDB_BASE directory is in /etc/ctdb/ + + BUT + + You have done a regular "configure" and "make install" so the tests + are installed under /usr/local/. + +If so, one possible hack to fix this is to create a symlink: + + ln -s /etc/ctdb /usr/local/etc/ctdb + +This is nasty but it works... :-) +EOF + exit 1 +fi + +###################################################################### + +if "$TEST_VERBOSE" ; then + debug () { echo "$@" ; } +else + debug () { : ; } +fi + +eventscripts_tests_cleanup_hooks="" + +# This loses quoting! +eventscripts_test_add_cleanup () +{ + eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*" +} + +trap 'eval $eventscripts_tests_cleanup_hooks' 0 + + +###################################################################### + +# General setup fakery + +setup_generic () +{ + debug "Setting up shares (3 existing shares)" + # Create 3 fake shares/exports. + export FAKE_SHARES="" + for i in $(seq 1 3) ; do + _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing" + mkdir -p "$_s" + FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}" + done + + export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding" + mkdir -p "$FAKE_PROC_NET_BONDING" + rm -f "$FAKE_PROC_NET_BONDING"/* + + export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down" + mkdir -p "$FAKE_ETHTOOL_LINK_DOWN" + rm -f "$FAKE_ETHTOOL_LINK_DOWN"/* + + # This can only have 2 levels. We don't want to resort to usings + # something dangerous like "rm -r" setup time. + export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state" + mkdir -p "$FAKE_IP_STATE" + rm -f "$FAKE_IP_STATE"/*/* + rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true + rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true + + + export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db" + mkdir -p "${CTDB_DBDIR}/persistent" + + export FAKE_TDBTOOL_SUPPORTS_CHECK="yes" + export FAKE_TDB_IS_OK + export FAKE_DATE_OUTPUT + + export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN + export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR") +} + +tcp_port_down () +{ + for _i ; do + debug "Marking TCP port \"${_i}\" as not listening" + FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g") + done +} + +shares_missing () +{ + _fmt="$1" ; shift + + # Replace some shares with non-existent ones. + _t="" + _n=1 + _nl=" +" + export MISSING_SHARES_TEXT="" + for _i in $FAKE_SHARES ; do + if [ $_n = "$1" ] ; then + shift + _i="${_i%_existing}_missing" + debug "Replacing share $_n with missing share \"$_i\"" + rmdir "$_i" 2>/dev/null || true + MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}") + fi + _t="${_t}${_t:+ }${_i}" + _n=$(($_n + 1)) + done + FAKE_SHARES="$_t" +} + +# Setup some fake /proc/net/bonding files with just enough info for +# the eventscripts. + +# arg1 is interface name, arg2 is currently active slave (use "None" +# if none), arg3 is MII status ("up" or "down"). +setup_bond () +{ + _iface="$1" + _slave="${2:-${_iface}_sl_0}" + _mii_s="${3:-up}" + _mii_subs="${4:-${_mii_s:-up}}" + echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s" + cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Currently Active Slave: $_slave +# Status of the bond +MII Status: $_mii_s +# Status of 1st pretend adapter +MII Status: $_mii_subs +# Status of 2nd pretend adapter +MII Status: $_mii_subs +EOF +} + +ethtool_interfaces_down () +{ + for _i ; do + echo "Marking interface $_i DOWN for ethtool" + touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}" + done +} + +ethtool_interfaces_up () +{ + for _i ; do + echo "Marking interface $_i UP for ethtool" + rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}" + done +} + +setup_nmap_output_filter () +{ + OUT_FILTER="-e 's@^(DEBUG: # Nmap 5.21 scan initiated) .+ (as:)@\1 DATE \2@' -e 's@^(DEBUG: # Nmap done at) .+ (--)@\1 DATE \2@'" +} + +dump_routes () +{ + echo "# ip rule show" + ip rule show + + ip rule show | + while read _p _x _i _x _t ; do + # Remove trailing colon after priority/preference. + _p="${_p%:}" + # Only remove rules that match our priority/preference. + [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue + + echo "# ip route show table $_t" + ip route show table "$_t" + done +} + +# Copied from 13.per_ip_routing for now... so this is lazy testing :-( +ipv4_host_addr_to_net () +{ + _host="$1" + _maskbits="$2" + + # Convert the host address to an unsigned long by splitting out + # the octets and doing the math. + _host_ul=0 + for _o in $(export IFS="." ; echo $_host) ; do + _host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug + done + + # Calculate the mask and apply it. + _mask_ul=$(( 0xffffffff << (32 - $_maskbits) )) + _net_ul=$(( $_host_ul & $_mask_ul )) + + # Now convert to a network address one byte at a time. + _net="" + for _o in $(seq 1 4) ; do + _net="$(($_net_ul & 255))${_net:+.}${_net}" + _net_ul=$(($_net_ul >> 8)) + done + + echo "${_net}/${_maskbits}" +} + +###################################################################### + +# CTDB fakery + +# Evaluate an expression that probably calls functions or uses +# variables from the CTDB functions file. This is used for test +# initialisation. +eventscript_call () +{ + ( + . "$CTDB_BASE/functions" + "$@" + ) +} + +# Set output for ctdb command. Option 1st argument is return code. +ctdb_set_output () +{ + _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out" + cat >"$_out" + + _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc" + echo "${1:-0}" >"$_rc" + + eventscripts_test_add_cleanup "rm -f $_out $_rc" +} + +setup_ctdb () +{ + setup_generic + + export FAKE_CTDB_NUMNODES="${1:-3}" + echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes" + + export FAKE_CTDB_PNN="${2:-0}" + echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}" + + export CTDB_PUBLIC_ADDRESSES="${CTDB_BASE}/public_addresses" + if [ -n "$3" ] ; then + echo "Setting up CTDB_PUBLIC_ADDRESSES: $3" + CTDB_PUBLIC_ADDRESSES=$(mktemp) + for _i in $3 ; do + _ip="${_i%@*}" + _ifaces="${_i#*@}" + echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES" + done + eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES" + fi + + export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb" + + export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down" + mkdir -p "$FAKE_CTDB_IFACES_DOWN" + rm -f "$FAKE_CTDB_IFACES_DOWN"/* + + export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus" + mkdir -p "$FAKE_CTDB_SCRIPTSTATUS" + rm -f "$FAKE_CTDB_SCRIPTSTATUS"/* + + export CTDB_PARTIALLY_ONLINE_INTERFACES +} + +setup_memcheck () +{ + setup_ctdb + + _swap_total="5857276" + + if [ "$1" = "bad" ] ; then + _swap_free=" 4352" + _mem_cached=" 112" + _mem_free=" 468" + else + _swap_free="$_swap_total" + _mem_cached="1112" + _mem_free="1468" + fi + + export FAKE_PROC_MEMINFO="\ +MemTotal: 3940712 kB +MemFree: 225268 kB +Buffers: 146120 kB +Cached: 1139348 kB +SwapCached: 56016 kB +Active: 2422104 kB +Inactive: 1019928 kB +Active(anon): 1917580 kB +Inactive(anon): 523080 kB +Active(file): 504524 kB +Inactive(file): 496848 kB +Unevictable: 4844 kB +Mlocked: 4844 kB +SwapTotal: ${_swap_total} kB +SwapFree: ${_swap_free} kB +..." + + export FAKE_FREE_M="\ + total used free shared buffers cached +Mem: 3848 3634 213 0 142 ${_mem_cached} +-/+ buffers/cache: 2379 ${_mem_free} +Swap: 5719 246 5473" + + export CTDB_MONITOR_FREE_MEMORY + export CTDB_MONITOR_FREE_MEMORY_WARN + export CTDB_CHECK_SWAP_IS_NOT_USED +} + +ctdb_get_interfaces () +{ + # The echo/subshell forces all the output onto 1 line. + echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}') +} + +ctdb_get_1_interface () +{ + _t=$(ctdb_get_interfaces) + echo ${_t%% *} +} + +# Print all public addresses as: interface IP maskbits +# Each line is suitable for passing to takeip/releaseip +ctdb_get_all_public_addresses () +{ + _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + while IFS="/$IFS" read _ip _maskbits _ifaces ; do + echo "$_ifaces $_ip $_maskbits" + done <"$_f" +} + +# Print public addresses on this node as: interface IP maskbits +# Each line is suitable for passing to takeip/releaseip +ctdb_get_my_public_addresses () +{ + ctdb ip -v -Y | { + read _x # skip header line + + while IFS=":" read _x _ip _x _iface _x ; do + [ -n "$_iface" ] || continue + while IFS="/$IFS" read _i _maskbits _x ; do + if [ "$_ip" = "$_i" ] ; then + echo $_iface $_ip $_maskbits + break + fi + done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + done + } +} + +# Prints the 1st public address as: interface IP maskbits +# This is suitable for passing to takeip/releaseip +ctdb_get_1_public_address () +{ + ctdb_get_my_public_addresses | head -n 1 +} + +ctdb_not_implemented () +{ + export CTDB_NOT_IMPLEMENTED="$1" + ctdb_not_implemented="\ +DEBUG: ctdb: command \"$1\" not implemented in stub" +} + +ctdb_fake_scriptstatus () +{ + _code="$1" + _status="$2" + _err_out="$3" + + _d1=$(date '+%s.%N') + _d2=$(date '+%s.%N') + + echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script" +} + +###################################################################### + +setup_ctdb_policy_routing () +{ + service_name="per_ip_routing" + + export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing" + export CTDB_PER_IP_ROUTING_RULE_PREF=100 + export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000 + export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000 + + # Tests need to create and populate this file + rm -f "$CTDB_PER_IP_ROUTING_CONF" +} + +# Create policy routing configuration in $CTDB_PER_IP_ROUTING_CONF. +# $1 is the number of assigned IPs to use (<num>, all), defaulting to +# 1. If $2 is "default" then a default route is also added. +create_policy_routing_config () +{ + _num_ips="${1:-1}" + _should_add_default="$2" + + ctdb_get_my_public_addresses | + if [ "$_num_ips" = "all" ] ; then + cat + else + head -n "$_num_ips" + fi | + while read _dev _ip _bits ; do + _net=$(ipv4_host_addr_to_net "$_ip" "$_bits") + _gw="${_net%.*}.1" # a dumb, calculated default + + echo "$_ip $_net" + + if [ "$_should_add_default" = "default" ] ; then + echo "$_ip 0.0.0.0/0 $_gw" + fi + done >"$CTDB_PER_IP_ROUTING_CONF" +} + +# Check the routes against those that are expected. $1 is the number +# of assigned IPs to use (<num>, all), defaulting to 1. If $2 is +# "default" then expect default routes to have been added. +check_routes () +{ + _num_ips="${1:-1}" + _should_add_default="$2" + + _policy_rules="" + _policy_routes="" + + ctdb_get_my_public_addresses | + if [ "$_num_ips" = "all" ] ; then + cat + else + head -n "$_num_ips" + fi | { + while read _dev _ip _bits ; do + _net=$(ipv4_host_addr_to_net "$_ip" "$_bits") + _gw="${_net%.*}.1" # a dumb, calculated default + + _policy_rules="${_policy_rules} +${CTDB_PER_IP_ROUTING_RULE_PREF}: from $_ip lookup ctdb.$_ip " + _policy_routes="${_policy_routes} +# ip route show table ctdb.$_ip +$_net dev $_dev scope link " + + if [ "$_should_add_default" = "default" ] ; then + _policy_routes="${_policy_routes} +default via $_gw dev $_dev " + fi + done + + ok <<EOF +# ip rule show +0: from all lookup local ${_policy_rules} +32766: from all lookup main +32767: from all lookup default ${_policy_routes} +EOF + + simple_test_command dump_routes + } +} + +###################################################################### + +# Samba/winbind fakery + +setup_samba () +{ + setup_ctdb + + service_name="samba" + + if [ "$1" != "down" ] ; then + + debug "Marking Samba services as up, listening and managed by CTDB" + # Get into known state. + eventscript_call ctdb_service_managed + + # All possible service names for all known distros. + for i in "smb" "nmb" "samba" ; do + service "$i" force-started + done + + export CTDB_SAMBA_SKIP_SHARE_CHECK="no" + export CTDB_MANAGED_SERVICES="foo samba bar" + + export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139" + export FAKE_WBINFO_FAIL="no" + + # Some things in 50.samba are backgrounded and waited for. If + # we don't sleep at all then timeouts can happen. This avoids + # that... :-) + export FAKE_SLEEP_FORCE=0.1 + else + debug "Marking Samba services as down, not listening and not managed by CTDB" + # Get into known state. + eventscript_call ctdb_service_unmanaged + + # All possible service names for all known distros. + for i in "smb" "nmb" "samba" ; do + service "$i" force-stopped + done + + export CTDB_SAMBA_SKIP_SHARE_CHECK="no" + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_SAMBA + + export FAKE_TCP_LISTEN="" + export FAKE_WBINFO_FAIL="yes" + fi + + # This is ugly but if this file isn't removed before each test + # then configuration changes between tests don't stick. + rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache" +} + +setup_winbind () +{ + setup_ctdb + + service_name="winbind" + + if [ "$1" != "down" ] ; then + + debug "Marking Winbind service as up and managed by CTDB" + # Get into known state. + eventscript_call ctdb_service_managed + + service "winbind" force-started + + export CTDB_MANAGED_SERVICES="foo winbind bar" + + export FAKE_WBINFO_FAIL="no" + + else + debug "Marking Winbind service as down and not managed by CTDB" + # Get into known state. + eventscript_call ctdb_service_unmanaged + + service "winbind" force-stopped + + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_WINBIND + + export FAKE_WBINFO_FAIL="yes" + fi +} + +wbinfo_down () +{ + debug "Making wbinfo commands fail" + FAKE_WBINFO_FAIL="yes" +} + +###################################################################### + +# NFS fakery + +setup_nfs () +{ + setup_ctdb + + service_name="nfs" + + export FAKE_RPCINFO_SERVICES="" + + export CTDB_NFS_SKIP_SHARE_CHECK="no" + + export CTDB_MONITOR_NFS_THREAD_COUNT RPCNFSDCOUNT FAKE_NFSD_THREAD_PIDS + export CTDB_NFS_DUMP_STUCK_THREADS + + # Reset the failcounts for nfs services. + eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' + + if [ "$1" != "down" ] ; then + debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB" + + eventscript_call ctdb_service_managed + service "nfs" force-started # might not be enough + + export CTDB_MANAGED_SERVICES="foo nfs bar" + + rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status" + else + debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB" + + eventscript_call ctdb_service_unmanaged + service "nfs" force-stopped # might not be enough + eventscript_call startstop_nfs stop + + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_NFS + fi +} + +setup_nfs_ganesha () +{ + setup_nfs "$@" + export CTDB_NFS_SERVER_MODE="ganesha" + if [ "$1" != "down" ] ; then + export CTDB_MANAGES_NFS="yes" + fi + + # We do not support testing the Ganesha-nfsd-specific part of the + # eventscript. + export CTDB_SKIP_GANESHA_NFSD_CHECK="yes" + export CTDB_NFS_SKIP_SHARE_CHECK="yes" +} + +rpc_services_down () +{ + for _i ; do + debug "Marking RPC service \"${_i}\" as unavailable" + FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g") + done +} + +rpc_services_up () +{ + for _i ; do + debug "Marking RPC service \"${_i}\" as available" + case "$_i" in + nfs) _t="2:3" ;; + mountd) _t="1:3" ;; + rquotad) _t="1:2" ;; + nlockmgr) _t="3:4" ;; + status) _t="1:1" ;; + *) die "Internal error - unsupported RPC service \"${_i}\"" ;; + esac + + FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}" + done +} + +# Set the required result for a particular RPC program having failed +# for a certain number of iterations. This is probably still a work +# in progress. Note that we could hook aggressively +# nfs_check_rpc_service() to try to implement this but we're better +# off testing nfs_check_rpc_service() using independent code... even +# if it is incomplete and hacky. So, if the 60.nfs eventscript +# changes and the tests start to fail then it may be due to this +# function being incomplete. +rpc_set_service_failure_response () +{ + _progname="$1" + # The number of failures defaults to the iteration number. This + # will be true when we fail from the 1st iteration... but we need + # the flexibility to set the number of failures. + _numfails="${2:-${iteration}}" + + _etc="$CTDB_ETCDIR" # shortcut for readability + for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do + if [ -r "$_c" ] ; then + . "$_c" + break + fi + done + + # A handy newline. :-) + _nl=" +" + + # Default + ok_null + + _file=$(ls "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9]."${_progname}.check") + [ -r "$_file" ] || die "RPC check file \"$_file\" does not exist or is not unique" + + while read _op _li _actions ; do + # Skip comments + case "$_op" in + \#*) continue ;; + esac + + _hit=false + if [ "$_op" != "%" ] ; then + if [ $_numfails $_op $_li ] ; then + _hit=true + fi + else + if [ $(($_numfails $_op $_li)) -eq 0 ] ; then + _hit=true + fi + fi + if $_hit ; then + _out="" + _rc=0 + for _action in $_actions ; do + case "$_action" in + verbose) + _ver=1 + _pn="$_progname" + case "$_progname" in + nfsd) _ver=3 ; _pn="nfs" ;; + lockd) _ver=4 ; _pn="nlockmgr" ;; + statd) _pn="status" ;; + esac + _out="\ +ERROR: $_pn failed RPC check: +rpcinfo: RPC: Program not registered +program $_pn version $_ver is not available" + ;; + restart*) + _p="rpc.${_progname}" + case "$_action" in + *:b) _bg="&" ;; + *) _bg="" ;; + esac + case "$_progname" in + nfsd) + _t="\ +Trying to restart NFS service" + + if [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] ; then + for _pid in $FAKE_NFSD_THREAD_PIDS ; do + _t="\ +$_t +${_bg}Stack trace for stuck nfsd thread [${_pid}]: +${_bg}[<ffffffff87654321>] fake_stack_trace_for_pid_${_pid}/stack+0x0/0xff" + done + fi + + _t="\ +${_t} +${_bg}Starting nfslock: OK +${_bg}Starting nfs: OK" + ;; + lockd) + _t="\ +Trying to restart lock manager service +${_bg}Starting nfslock: OK" + ;; + *) + _t="Trying to restart $_progname [${_p}]" + esac + _out="${_out}${_out:+${_nl}}${_t}" + ;; + unhealthy) + _rc=1 + esac + done + required_result $_rc "$_out" + return + fi + done <"$_file" +} + +###################################################################### + +# VSFTPD fakery + +setup_vsftpd () +{ + service_name="vsftpd" + + if [ "$1" != "down" ] ; then + die "setup_vsftpd up not implemented!!!" + else + debug "Setting up VSFTPD environment: service down, not managed by CTDB" + + eventscript_call ctdb_service_unmanaged + service vsftpd force-stopped + + export CTDB_MANAGED_SERVICES="foo" + unset CTDB_MANAGES_VSFTPD + fi +} + +###################################################################### + +# HTTPD fakery + +setup_httpd () +{ + if [ "$1" != "down" ] ; then + die "setup_httpd up not implemented!!!" + else + debug "Setting up HTTPD environment: service down, not managed by CTDB" + + for service_name in "apache2" "httpd" ; do + eventscript_call ctdb_service_unmanaged + service "$service_name" force-stopped + done + + export CTDB_MANAGED_SERVICES="foo" + unset CTDB_MANAGES_HTTPD + fi +} + +###################################################################### + +# multipathd fakery + +setup_multipathd () +{ + for i ; do + case "$i" in + \!*) + _t="${i#!}" + echo "Marking ${_t} as having no active paths" + FAKE_MULTIPATH_FAILURES="${FAKE_MULTIPATH_FAILURES}${FAKE_MULTIPATH+FAILURES:+ }${_t}" + ;; + *) + _t="$i" + esac + CTDB_MONITOR_MPDEVICES="${CTDB_MONITOR_MPDEVICES}${CTDB_MONITOR_MPDEVICES:+ }${_t}" + done + + export CTDB_MONITOR_MPDEVICES FAKE_MULTIPATH_FAILURES + export FAKE_SLEEP_FORCE=0.1 +} + +###################################################################### + +# Result and test functions + +# Set some globals and print the summary. +define_test () +{ + desc="$1" + + _f=$(basename "$0" ".sh") + + # Remaining format should be NN.service.event.NNN or NN.service.NNN: + _num="${_f##*.}" + _f="${_f%.*}" + case "$_f" in + *.*.*) + script="${_f%.*}" + event="${_f##*.}" + ;; + *.*) + script="$_f" + unset event + ;; + *) + die "Internal error - unknown testcase filename format" + esac + + printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" +} + +_extra_header () +{ + cat <<EOF +CTDB_BASE="$CTDB_BASE" +CTDB_ETCDIR="$CTDB_ETCDIR" +ctdb client is "$(which ctdb)" +EOF +} + +# Run an eventscript once. The test passes if the return code and +# output match those required. + +# Any args are passed to the eventscript. + +simple_test () +{ + [ -n "$event" ] || die 'simple_test: $event not set' + + _extra_header=$(_extra_header) + + echo "Running eventscript \"$script $event${1:+ }$*\"" + _shell="" + if $TEST_COMMAND_TRACE ; then + _shell="sh -x" + else + _shell="sh" + fi + _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1) + + result_check "$_extra_header" +} + +simple_test_event () +{ + # If something has previously failed then don't continue. + : ${_passed:=true} + $_passed || return 1 + + event="$1" ; shift + echo "==================================================" + simple_test "$@" +} + +simple_test_command () +{ + # If something has previously failed then don't continue. + : ${_passed:=true} + $_passed || return 1 + + echo "==================================================" + echo "Running command \"$*\"" + _out=$("$@" 2>&1) + + result_check +} + +check_ctdb_logfile () +{ + # If something has previously failed then don't continue. + : ${_passed:=true} + $_passed || return 1 + + echo "==================================================" + echo "Checking CTDB_LOGFILE=\"${CTDB_LOGFILE}\"" + _out=$(cat "$CTDB_LOGFILE" 2>&1) + + result_check +} + +# Run an eventscript iteratively. +# - 1st argument is the number of iterations. +# - 2nd argument is something to eval to do setup for every iteration. +# The easiest thing to do here is to define a function and pass it +# here. +# - Subsequent arguments come in pairs: an iteration number and +# something to eval for that iteration. Each time an iteration +# number is matched the associated argument is given to eval after +# the default setup is done. The iteration numbers need to be given +# in ascending order. +# +# Some optional args can be given *before* these, surrounded by extra +# "--" args. These args are passed to the eventscript. Quoting is +# lost. +# +# One use of the 2nd and further arguments is to call +# required_result() to change what is expected of a particular +# iteration. +iterate_test () +{ + [ -n "$event" ] || die 'simple_test: $event not set' + + args="" + if [ "$1" = "--" ] ; then + shift + while [ "$1" != "--" ] ; do + args="${args}${args:+ }$1" + shift + done + shift + fi + + _repeats="$1" + _setup_default="$2" + shift 2 + + echo "Running $_repeats iterations of \"$script $event\" $args" + + _result=true + + for iteration in $(seq 1 $_repeats) ; do + # This is inefficient because the iteration-specific setup + # might completely replace the default one. However, running + # the default is good because it allows you to revert to a + # particular result without needing to specify it explicitly. + eval $_setup_default + if [ $iteration = "$1" ] ; then + eval $2 + shift 2 + fi + + _shell="" + if $TEST_COMMAND_TRACE ; then + _shell="sh -x" + else + _shell="sh" + fi + _out=$($_shell "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1) + _rc=$? + + if [ -n "$OUT_FILTER" ] ; then + _fout=$(echo "$_out" | eval sed -r $OUT_FILTER) + else + _fout="$_out" + fi + + if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then + _passed=true + else + _passed=false + _result=false + fi + + result_print "$_passed" "$_out" "$_rc" "Iteration $iteration" + done + + result_footer "$_result" "$(_extra_header)" +} diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb new file mode 100755 index 00000000000..da84ed7cdfd --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/ctdb @@ -0,0 +1,334 @@ +#!/bin/sh + +prog="ctdb" + +not_implemented_exit_code=1 + +usage () +{ + cat >&2 <<EOF +Usage: $prog [-Y] cmd + +A fake CTDB stub that prints items depending on the variables +FAKE_CTDB_PNN (default 0) depending on command-line options. +EOF + exit 1 +} + +not_implemented () +{ + echo "${prog}: command \"$1\" not implemented in stub" >&2 + exit $not_implemented_exit_code +} + +# Don't set $POSIXLY_CORRECT here. +_temp=$(getopt -n "$prog" -o "Yvhn:" -l help -- "$@") || \ + usage + +eval set -- "$_temp" + +verbose=false +machine_readable=false +nodespec="" + +args="$*" + +while true ; do + case "$1" in + -Y) machine_readable=true ; shift ;; + -v) verbose=true ; shift ;; + -n) nodespec="$2" ; shift 2 ;; + --) shift ; break ;; + -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable. + esac +done + +[ $# -ge 1 ] || usage + +setup_tickles () +{ + # Make sure tickles file exists. + tickles_file="$CTDB_VARDIR/fake-ctdb/tickles" + mkdir -p $(dirname "$tickles_file") + touch "$tickles_file" +} + +ctdb_killtcp () +{ + while read _src _dst ; do + sed -i -e "/^$_dst $_src\$/d" "$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" + done +} + +setup_pstore () +{ + pstore_dir="$CTDB_VARDIR/fake-ctdb/pstore/$1" + mkdir -p "$pstore_dir" +} + +parse_nodespec () +{ + if [ "$nodespec" = "all" ] ; then + nodes="$(seq 0 $((FAKE_CTDB_NUMNODES - 1)) )" + elif [ -n "$nodespec" ] ; then + nodes="$(echo $nodespec | sed -e 's@,@ @g')" + else + _t=$(ctdb_pnn) + nodes="${_t#PNN:}" + fi +} + +# For testing backward compatibility... +for i in $CTDB_NOT_IMPLEMENTED ; do + if [ "$i" = "$1" ] ; then + not_implemented "$i" + fi +done + +ctdb_pnn () +{ + # Defaults to 0 + echo "PNN:${FAKE_CTDB_PNN:-0}" +} + +###################################################################### + +FAKE_CTDB_NODE_STATE="$FAKE_CTDB_STATE/node-state" +FAKE_CTDB_NODES_DISABLED="$FAKE_CTDB_NODE_STATE/0x4" + +###################################################################### + +# NOTE: all nodes share $CTDB_PUBLIC_ADDRESSES + +FAKE_CTDB_IP_LAYOUT="$FAKE_CTDB_STATE/ip-layout" + +ip_reallocate () +{ + touch "$FAKE_CTDB_IP_LAYOUT" + + ( + flock 0 + + _pa="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + + if [ ! -s "$FAKE_CTDB_IP_LAYOUT" ] ; then + sed -n -e 's@^\([^#][^/]*\)/.*@\1 -1@p' \ + "$_pa" >"$FAKE_CTDB_IP_LAYOUT" + fi + + _t="${FAKE_CTDB_IP_LAYOUT}.new" + + _flags="" + for _i in $(seq 0 $((FAKE_CTDB_NUMNODES - 1)) ) ; do + if ls "$FAKE_CTDB_STATE/node-state/"*"/$_i" >/dev/null 2>&1 ; then + # Have non-zero flags + _this=0 + for _j in "$FAKE_CTDB_STATE/node-state/"*"/$_i" ; do + _tf="${_j%/*}" # dirname + _f="${_tf##*/}" # basename + _this=$(( $_this | $_f )) + done + else + _this="0" + fi + _flags="${_flags}${_flags:+,}${_this}" + done + CTDB_TEST_LOGLEVEL=2 \ + "ctdb_takeover_tests" \ + "ctdb_takeover_run_core" "$_flags" <"$FAKE_CTDB_IP_LAYOUT" | + sort >"$_t" + mv "$_t" "$FAKE_CTDB_IP_LAYOUT" + ) <"$FAKE_CTDB_IP_LAYOUT" +} + +ctdb_ip () +{ + # If nobody has done any IP-fu then generate a layout. + [ -f "$FAKE_CTDB_IP_LAYOUT" ] || ip_reallocate + + if $verbose ; then + echo ":Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:" + else + echo ":Public IP:Node:" + fi + + _mypnn=$(ctdb_pnn | sed -e 's@PNN:@@') + + # Join public addresses file with $FAKE_CTDB_IP_LAYOUT, and + # process output line by line... + _pa="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + sed -e 's@/@ @' "$_pa" | sort | join - "$FAKE_CTDB_IP_LAYOUT" | + while read _ip _bit _ifaces _pnn ; do + if $verbose ; then + # If more than 1 interface, assume all addresses are on the 1st. + _first_iface="${_ifaces%%,*}" + # Only show interface if address is on this node. + _my_iface="" + if [ "$_pnn" = "$_mypnn" ]; then + _my_iface="$_first_iface" + fi + echo ":${_ip}:${_pnn}:${_my_iface}:${_first_iface}:${_ifaces}:" + else + echo ":${_ip}:${_pnn}:" + fi + done +} + +ctdb_moveip () +{ + _ip="$1" + _target="$2" + + ip_reallocate # should be harmless and ensures we have good state + + ( + flock 0 + + _t="${FAKE_CTDB_IP_LAYOUT}.new" + + while read _i _pnn ; do + if [ "$_ip" = "$_i" ] ; then + echo "$_ip $_target" + else + echo "$_ip $_pnn" + fi + done | sort >"$_t" + mv "$_t" "$FAKE_CTDB_IP_LAYOUT" + ) <"$FAKE_CTDB_IP_LAYOUT" +} + +###################################################################### + +ctdb_enable () +{ + parse_nodespec + + for _i in $nodes ; do + rm -f "${FAKE_CTDB_NODES_DISABLED}/${_i}" + done + + ip_reallocate +} + +ctdb_disable () +{ + parse_nodespec + + for _i in $nodes ; do + mkdir -p "$FAKE_CTDB_NODES_DISABLED" + touch "${FAKE_CTDB_NODES_DISABLED}/${_i}" + done + + ip_reallocate +} + +###################################################################### + +ctdb_shutdown () +{ + echo "CTDB says BYE!" +} + +###################################################################### + +case "$1" in + gettickles) + setup_tickles + echo ":source ip:port:destination ip:port:" + while read src dst ; do + echo ":${src}:${dst}:" + done <"$tickles_file" + ;; + addtickle) + setup_tickles + echo "$2 $3" >>"$tickles_file" + ;; + deltickle) + setup_tickles + _t=$(grep -F -v "$2 $3" "$tickles_file") + echo "$_t" >"$tickles_file" + ;; + pstore) + setup_pstore "$2" + cat "$4" >"${pstore_dir}/$3" + ;; + pfetch) + setup_pstore "$2" + cat "${pstore_dir}/$3" >"$4" 2>/dev/null + ;; + ifaces) + # Assume -Y. + echo ":Name:LinkStatus:References:" + _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + if [ -r "$_f" ] ; then + while read _ip _iface ; do + case "_$ip" in + \#*) : ;; + *) + _status=1 + # For now assume _iface contains only 1. + if [ -f "{FAKE_CTDB_IFACES_DOWN}/${_iface}" ] ; then + _status=0 + fi + # Nobody looks at references + echo ":${_iface}:${_status}:0" + esac + done <"$_f" | + sort -u + fi + ;; + setifacelink) + # Existence of file means CTDB thinks interface is down. + _f="${FAKE_CTDB_IFACES_DOWN}/$2" + case "$3" in + up) rm -f "$_f" ;; + down) touch "$_f" ;; + *) + echo "ctdb setifacelink: unsupported interface status $3" + exit 1 + esac + ;; + checktcpport) + for _i in $FAKE_TCP_LISTEN ; do + if [ "$2" = "${_i##*:}" ] ; then + exit 98 + fi + done + + exit 0 + ;; + scriptstatus) + $machine_readable || not_implemented "$1, without -Y" + [ "$2" != "all" ] || not_implemented "scriptstatus all" + # For now just assume everything is good. + echo ":Type:Name:Code:Status:Start:End:Error Output...:" + for _i in "$CTDB_BASE/events.d/"*.* ; do + _d1=$(date '+%s.%N') + _b="${_i##*/}" # basename + + _f="$FAKE_CTDB_SCRIPTSTATUS/$_b" + if [ -r "$_f" ] ; then + read _code _status _err_out <"$_f" + else + _code="0" + _status="OK" + if [ ! -x "$_i" ] ; then + _status="DISABLED" + _code="-8" + fi + _err_out="" + fi + _d2=$(date '+%s.%N') + echo ":${2:-monitor}:${_b}:${_code}:${_status}:${_d1}:${_d2}:${_err_out}:" + done + ;; + gratiousarp) : ;; # Do nothing for now + killtcp) ctdb_killtcp "$@" ;; + ip) ctdb_ip "$@" ;; + pnn|xpnn) ctdb_pnn ;; + enable) ctdb_enable "$@";; + disable) ctdb_disable "$@";; + moveip) ctdb_moveip "$@";; + shutdown) ctdb_shutdown "$@";; + *) not_implemented "$1" ;; +esac diff --git a/ctdb/tests/eventscripts/stubs/date b/ctdb/tests/eventscripts/stubs/date new file mode 100755 index 00000000000..2f470a8156f --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/date @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ "$FAKE_DATE_OUTPUT" ] ; then + echo "$FAKE_DATE_OUTPUT" +else + /bin/date "$@" +fi diff --git a/ctdb/tests/eventscripts/stubs/ethtool b/ctdb/tests/eventscripts/stubs/ethtool new file mode 100755 index 00000000000..bd173f438f7 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/ethtool @@ -0,0 +1,12 @@ +#!/bin/sh + +link="yes" + +if [ -f "${FAKE_ETHTOOL_LINK_DOWN}/${1}" ] ; then + link="no" +fi + +# Expect to add more fields later. +cat <<EOF + Link detected: ${link} +EOF diff --git a/ctdb/tests/eventscripts/stubs/exportfs b/ctdb/tests/eventscripts/stubs/exportfs new file mode 100755 index 00000000000..46c65225be1 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/exportfs @@ -0,0 +1,13 @@ +#!/bin/sh + +opts="10.0.0.0/16(rw,async,insecure,no_root_squash,no_subtree_check)" + +for i in $FAKE_SHARES ; do + # Directories longer than 15 characters are printed on their own + # line. + if [ ${#i} -ge 15 ] ; then + printf '%s\n\t\t%s\n' "$i" "$opts" + else + printf '%s\t%s\n' "$i" "$opts" + fi +done diff --git a/ctdb/tests/eventscripts/stubs/free b/ctdb/tests/eventscripts/stubs/free new file mode 100755 index 00000000000..64535099249 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/free @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "$1" = "-m" ] ; then + echo "$FAKE_FREE_M" + exit 0 +else + echo "free: not implemented - $*" + exit 1 +fi diff --git a/ctdb/tests/eventscripts/stubs/ip b/ctdb/tests/eventscripts/stubs/ip new file mode 100755 index 00000000000..053da750b9c --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/ip @@ -0,0 +1,505 @@ +#!/bin/sh + +not_implemented () +{ + echo "ip stub command: \"$1\" not implemented" + exit 127 +} + +###################################################################### + +ip_link () +{ + case "$1" in + set) + shift + # iface="$1" + case "$2" in + up) ip_link_set_up "$1" ;; + down) ip_link_down_up "$1" ;; + *) not_implemented "\"$2\" in \"$orig_args\"" ;; + esac + ;; + show) shift ; ip_link_show "$@" ;; + del*) shift ; ip_link_delete "$@" ;; + *) not_implemented "$*" ;; + esac +} + +ip_link_delete () +{ + mkdir -p "${FAKE_IP_STATE}/interfaces-deleted" + touch "${FAKE_IP_STATE}/interfaces-deleted/$1" +} + +ip_link_set_up () +{ + rm -f "${FAKE_IP_STATE}/interfaces-down/$1" + rm -f "${FAKE_IP_STATE}/interfaces-deleted/$1" +} + +ip_link_set_down () +{ + rm -f "${FAKE_IP_STATE}/interfaces-deleted/$1" + mkdir -p "${FAKE_IP_STATE}/interfaces-down" + touch "${FAKE_IP_STATE}/interfaces-down/$1" +} + +ip_link_show () +{ + dev="$1" + if [ "$dev" = "dev" -a -n "$2" ] ; then + dev="$2" + fi + + if [ -e "${FAKE_IP_STATE}/interfaces-deleted/$dev" ] ; then + echo "Device \"${dev}\" does not exist." >&2 + exit 255 + fi + + mac=$(echo $dev | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@') + _state="UP" + _flags=",UP,LOWER_UP" + if [ -e "${FAKE_IP_STATE}/interfaces-down/$dev" ] ; then + _state="DOWN" + _flags="" + fi + cat <<EOF +${n}: ${dev}: <BROADCAST,MULTICAST${_flags}> mtu 1500 qdisc pfifo_fast state ${_state} qlen 1000 + link/ether ${mac} brd ff:ff:ff:ff:ff:ff +EOF +} + +# This is incomplete because it doesn't actually look up table ids in +# /etc/iproute2/rt_tables. The rules/routes are actually associated +# with the name instead of the number. However, we include a variable +# to fake a bad table id. +[ -n "$IP_ROUTE_BAD_TABLE_ID" ] || IP_ROUTE_BAD_TABLE_ID=false + +ip_check_table () +{ + _cmd="$1" + + [ -n "$_table" ] || not_implemented "ip rule/route without \"table\"" + + # Only allow tables names from 13.per_ip_routing. This is a cheap + # way of avoiding implementing the default/main/local tables. + case "$_table" in + ctdb.*) + if $IP_ROUTE_BAD_TABLE_ID ; then + # Ouch. Simulate inconsistent errors from ip. :-( + case "$_cmd" in + route) + echo "Error: argument "${_table}" is wrong: table id value is invalid" >&2 + + ;; + *) + echo "Error: argument "${_table}" is wrong: invalid table ID" >&2 + esac + exit 255 + fi + ;; + *) not_implemented "table=${_table} ${orig_args}" ;; + esac +} + +###################################################################### + +ip_addr () +{ + case "$1" in + show|list|"") shift ; ip_addr_show "$@" ;; + add*) shift ; ip_addr_add "$@" ;; + del*) shift ; ip_addr_del "$@" ;; + *) not_implemented "\"$1\" in \"$orig_args\"" ;; + esac +} + +ip_addr_show () +{ + dev="" + primary=true + secondary=true + _to="" + while [ -n "$1" ] ; do + case "$1" in + dev) + dev="$2" ; shift 2 + ;; + # Do stupid things and stupid things will happen! + primary) + primary=true ; secondary=false ; shift + ;; + secondary) + secondary=true ; primary=false ; shift + ;; + to) + _to="$2" ; shift 2 + ;; + *) + # Assume an interface name + dev="$1" ; shift 1 + esac + done + devices="$dev" + if [ -z "$devices" ] ; then + # No device specified? Get all the primaries... + devices=$(ls "${FAKE_IP_STATE}/addresses/"*-primary 2>/dev/null | \ + sed -e 's@.*/@@' -e 's@-primary$@@') + fi + calc_brd () + { + case "${local#*/}" in + 24) + brd="${local%.*}.255" + ;; + *) + not_implemented "list ... fake bits other than 24: ${local#*/}" + esac + } + show_iface() + { + pf="${FAKE_IP_STATE}/addresses/${dev}-primary" + sf="${FAKE_IP_STATE}/addresses/${dev}-secondary" + ip_link_show "$dev" + if $primary && [ -r "$pf" ] ; then + read local <"$pf" + if [ -z "$_to" -o "${_to%/*}" = "${local%/*}" ] ; then + calc_brd + cat <<EOF + inet ${local} brd ${brd} scope global ${dev} +EOF + fi + fi + if $secondary && [ -r "$sf" ] ; then + while read local ; do + if [ -z "$_to" -o "${_to%/*}" = "${local%/*}" ] ; then + calc_brd + cat <<EOF + inet ${local} brd ${brd} scope global secondary ${dev} +EOF + fi + done <"$sf" + fi + if [ -z "$_to" ] ; then + cat <<EOF + valid_lft forever preferred_lft forever +EOF + fi + } + n=1 + for dev in $devices ; do + if [ -z "$_to" ] || \ + grep -F "${_to%/*}/" "${FAKE_IP_STATE}/addresses/${dev}-"* >/dev/null ; then + show_iface + fi + n=$(($n + 1)) + done +} + +ip_addr_add () +{ + local="" + dev="" + brd="" + while [ -n "$1" ] ; do + case "$1" in + *.*.*.*/*) + local="$1" ; shift + ;; + local) + local="$2" ; shift 2 + ;; + broadcast|brd) + # For now assume this is always '+'. + if [ "$2" != "+" ] ; then + not_implemented "addr add ... brd $2 ..." + fi + shift 2 + ;; + dev) + dev="$2" ; shift 2 + ;; + *) + not_implemented "$@" + esac + done + if [ -z "$dev" ] ; then + not_implemented "addr add (without dev)" + fi + mkdir -p "${FAKE_IP_STATE}/addresses" + pf="${FAKE_IP_STATE}/addresses/${dev}-primary" + sf="${FAKE_IP_STATE}/addresses/${dev}-secondary" + # We could lock here... but we should be the only ones playing + # around here with these stubs. + if [ ! -f "$pf" ] ; then + echo "$local" >"$pf" + elif grep -Fq "$local" "$pf" ; then + echo "RTNETLINK answers: File exists" >&2 + exit 254 + elif [ -f "$sf" ] && grep -Fq "$local" "$sf" ; then + echo "RTNETLINK answers: File exists" >&2 + exit 254 + else + echo "$local" >>"$sf" + fi +} + +ip_addr_del () +{ + local="" + dev="" + while [ -n "$1" ] ; do + case "$1" in + *.*.*.*/*) + local="$1" ; shift + ;; + local) + local="$2" ; shift 2 + ;; + dev) + dev="$2" ; shift 2 + ;; + *) + not_implemented "addr del ... $1 ..." + esac + done + if [ -z "$dev" ] ; then + not_implemented "addr del (without dev)" + fi + mkdir -p "${FAKE_IP_STATE}/addresses" + pf="${FAKE_IP_STATE}/addresses/${dev}-primary" + sf="${FAKE_IP_STATE}/addresses/${dev}-secondary" + # We could lock here... but we should be the only ones playing + # around here with these stubs. + if [ ! -f "$pf" ] ; then + echo "RTNETLINK answers: Cannot assign requested address" >&2 + exit 254 + elif grep -Fq "$local" "$pf" ; then + # Remove primaries AND SECONDARIES. + rm -f "$pf" "$sf" + elif [ -f "$sf" ] && grep -Fq "$local" "$sf" ; then + grep -Fv "$local" "$sf" >"${sf}.new" + mv "${sf}.new" "$sf" + else + echo "RTNETLINK answers: Cannot assign requested address" >&2 + exit 254 + fi +} + +###################################################################### + +ip_rule () +{ + case "$1" in + show|list|"") shift ; ip_rule_show "$@" ;; + add) shift ; ip_rule_add "$@" ;; + del*) shift ; ip_rule_del "$@" ;; + *) not_implemented "$1 in \"$orig_args\"" ;; + esac + +} + +# All non-default rules are in $FAKE_IP_STATE_RULES/rules. As with +# the real version, rules can be repeated. Deleting just deletes the +# 1st match. + +ip_rule_show () +{ + ip_rule_show_1 () + { + _pre="$1" + _table="$2" + _selectors="$3" + # potentially more options + + printf "%d:\t%s lookup %s \n" $_pre "$_selectors" "$_table" + } + + ip_rule_show_some () + { + _min="$1" + _max="$2" + + [ -f "${FAKE_IP_STATE}/rules" ] || return + + while read _pre _table _selectors ; do + # Only print those in range + [ $_min -le $_pre -a $_pre -le $_max ] || continue + + ip_rule_show_1 $_pre "$_table" "$_selectors" + done <"${FAKE_IP_STATE}/rules" + } + + ip_rule_show_1 0 "local" "from all" + + ip_rule_show_some 1 32765 + + ip_rule_show_1 32766 "main" "from all" + ip_rule_show_1 32767 "default" "from all" + + ip_rule_show_some 32768 2147483648 +} + +ip_rule_common () +{ + _from="" + _pre="" + _table="" + while [ -n "$1" ] ; do + case "$1" in + from) _from="$2" ; shift 2 ;; + pref) _pre="$2" ; shift 2 ;; + table) _table="$2" ; shift 2 ;; + *) not_implemented "$1 in \"$orig_args\"" ;; + esac + done + + [ -n "$_pre" ] || not_implemented "ip rule without \"pref\"" + ip_check_table "rule" + # Relax this if more selectors added later... + [ -n "$_from" ] || not_implemented "ip rule without \"from\"" +} + +ip_rule_add () +{ + ip_rule_common "$@" + + _f="${FAKE_IP_STATE}/rules" + touch "$_f" + ( + flock 0 + # Filter order must be consistent with the comparison in ip_rule_del() + echo "$_pre $_table${_from:+ from }$_from" >>"$_f" + ) <"$_f" +} + +ip_rule_del () +{ + ip_rule_common "$@" + + _f="${FAKE_IP_STATE}/rules" + touch "$_f" + ( + flock 0 + _tmp="$(mktemp)" + _found=false + while read _p _t _s ; do + if ! $_found && \ + [ "$_p" = "$_pre" -a "$_t" = "$_table" -a \ + "$_s" = "${_from:+from }$_from" ] ; then + # Found. Skip this one but not future ones. + _found=true + else + echo "$_p $_t $_s" >>"$_tmp" + fi + done + if cmp -s "$_tmp" "$_f" ; then + # No changes, must not have found what we wanted to delete + echo "RTNETLINK answers: No such file or directory" >&2 + rm -f "$_tmp" + exit 2 + else + mv "$_tmp" "$_f" + fi + ) <"$_f" || exit $? +} + +###################################################################### + +ip_route () +{ + case "$1" in + show|list) shift ; ip_route_show "$@" ;; + flush) shift ; ip_route_flush "$@" ;; + add) shift ; ip_route_add "$@" ;; + *) not_implemented "$1 in \"ip route\"" ;; + esac +} + +ip_route_common () +{ + [ "$1" = table ] || not_implemented "$1 in \"$orig_args\"" + _table="$2" + + ip_check_table "route" +} + +# Routes are in a file per table in the directory +# $FAKE_IP_STATE/routes. These routes just use the table ID +# that is passed and don't do any lookup. This could be "improved" if +# necessary. + +ip_route_show () +{ + ip_route_common "$@" + + # Missing file is just an empty table + cat "$FAKE_IP_STATE/routes/${_table}" 2>/dev/null || true +} + +ip_route_flush () +{ + ip_route_common "$@" + + rm -f "$FAKE_IP_STATE/routes/${_table}" +} + +ip_route_add () +{ + _prefix="" + _dev="" + _gw="" + _table="" + + while [ -n "$1" ] ; do + case "$1" in + *.*.*.*/*|*.*.*.*) _prefix="$1" ; shift 1 ;; + local) _prefix="$2" ; shift 2 ;; + dev) _dev="$2" ; shift 2 ;; + via) _gw="$2" ; shift 2 ;; + table) _table="$2" ; shift 2 ;; + *) not_implemented "$1 in \"$orig_args\"" ;; + esac + done + + ip_check_table "route" + [ -n "$_prefix" ] || not_implemented "ip route without inet prefix in \"$orig_args\"" + [ -n "$_dev" ] || not_implemented "ip route without \"dev\" in \"$orig_args\"" + + # Alias or add missing bits + case "$_prefix" in + 0.0.0.0/0) _prefix="default" ;; + */*) : ;; + *) _prefix="${_prefix}/32" ;; + esac + + _f="$FAKE_IP_STATE/routes/${_table}" + mkdir -p "$FAKE_IP_STATE/routes" + touch "$_f" + + ( + flock 0 + + if [ -n "$_gw" ] ; then + echo "${_prefix} via ${_gw} dev ${_dev} " + else + echo "${_prefix} dev ${_dev} scope link " + fi >>"$_f" + ) <"$_f" +} + + +###################################################################### + +orig_args="$*" + +case "$1" in + link) shift ; ip_link "$@" ;; + addr*) shift ; ip_addr "$@" ;; + rule) shift ; ip_rule "$@" ;; + route) shift ; ip_route "$@" ;; + *) not_implemented "$1" ;; +esac + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/iptables b/ctdb/tests/eventscripts/stubs/iptables new file mode 100755 index 00000000000..2c65f7ba112 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/iptables @@ -0,0 +1,5 @@ +#!/bin/sh + +# Always succeed. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/kill b/ctdb/tests/eventscripts/stubs/kill new file mode 100755 index 00000000000..b69e3e62a39 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/kill @@ -0,0 +1,7 @@ +#!/bin/sh + +# Always succeed. This means that kill -0 will always find a +# process and anything else will successfully kill. This should +# exercise a good avriety of code paths. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/killall b/ctdb/tests/eventscripts/stubs/killall new file mode 100755 index 00000000000..1e182e1e0d7 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/killall @@ -0,0 +1,7 @@ +#!/bin/sh + +# Always succeed. This means that killall -0 will always find a +# process and anything else will successfully kill. This should +# exercise a good avriety of code paths. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/multipath b/ctdb/tests/eventscripts/stubs/multipath new file mode 100755 index 00000000000..64f95e7efb5 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/multipath @@ -0,0 +1,36 @@ +#!/bin/sh + +usage () +{ + die "usage: ${0} -ll device" +} + +[ "$1" = "-ll" ] || usage +shift +[ $# -eq 1 ] || usage + +device="$1" + +if [ -n "$FAKE_MULTIPATH_HANG" ] ; then + FAKE_SLEEP_REALLY="yes" sleep 999 +fi + +path1_state="active" +path2_state="enabled" + +for i in $FAKE_MULTIPATH_FAILURES ; do + if [ "$device" = "$i" ] ; then + path1_state="inactive" + path2_state="inactive" + break + fi +done + + cat <<EOF +${device} (AUTO-01234567) dm-0 , +size=10G features='0' hwhandler='0' wp=rw +|-+- policy='round-robin 0' prio=1 status=${path1_state} +| \`- #:#:#:# vda 252:0 active ready running +\`-+- policy='round-robin 0' prio=1 status=${path2_state} + \`- #:#:#:# vdb 252:16 active ready running +EOF diff --git a/ctdb/tests/eventscripts/stubs/net b/ctdb/tests/eventscripts/stubs/net new file mode 100755 index 00000000000..3f964133134 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/net @@ -0,0 +1,5 @@ +#!/bin/sh + +# Always succeed for now... + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/netstat b/ctdb/tests/eventscripts/stubs/netstat new file mode 100755 index 00000000000..bd542bb09f7 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/netstat @@ -0,0 +1,109 @@ +#!/bin/bash + +prog="netstat" + +# Pretty that we're the shell and that this command could not be +# found. +if [ "$FAKE_NETSTAT_NOT_FOUND" = "yes" ] ; then + echo "sh: ${prog}: command not found" >&2 + exit 127 +fi + +usage () +{ + cat >&2 <<EOF +Usage: $prog [ -t | --unix ] [ -n ] [ -a ] [ -l ] + +A fake netstat stub that prints items depending on the variables +FAKE_NETSTAT_TCP_ESTABLISHED, FAKE_TCP_LISTEN, +FAKE_NETSTAT_UNIX_LISTEN, depending on command-line options. + +Note that -n is ignored. + +EOF + exit 1 +} + +# Defaults. +tcp=false +unix=false +all=false +listen=false + +parse_options () +{ + # $POSIXLY_CORRECT means that the command passed to onnode can + # take options and getopt won't reorder things to make them + # options to this script. + _temp=$(POSIXLY_CORRECT=1 getopt -n "$prog" -o "tnalh" -l unix -l help -- "$@") + + [ $? != 0 ] && usage + + eval set -- "$_temp" + + while true ; do + case "$1" in + -n) shift ;; + -a) all=true ; shift ;; + -t) tcp=true ; shift ;; + -l) listen=true ; shift ;; + --unix) unix=true ; shift ;; + --) shift ; break ;; + -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable. + esac + done + + [ $# -gt 0 ] && usage + + # If neither -t or --unix specified then print all. + $tcp || $unix || { tcp=true ; unix=true ; } +} + +parse_options "$@" + +if $tcp ; then + if $listen ; then + echo "Active Internet connections (servers only)" + elif $all ; then + echo "Active Internet connections (servers and established)" + else + echo "Active Internet connections (w/o servers)" + fi + + echo "Proto Recv-Q Send-Q Local Address Foreign Address State" + + tcp_fmt="tcp 0 0 %-23s %-23s %s\n" + for i in $FAKE_NETSTAT_TCP_ESTABLISHED ; do + src="${i%|*}" + dst="${i#*|}" + printf "$tcp_fmt" $src $dst "ESTABLISHED" + done + while read src dst ; do + printf "$tcp_fmt" $src $dst "ESTABLISHED" + done <"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" + + if $all || $listen ; then + for i in $FAKE_TCP_LISTEN ; do + printf "$tcp_fmt" $i "0.0.0.0:*" "LISTEN" + done + fi +fi + +if $unix ; then + if $listen ; then + echo "Active UNIX domain sockets (servers only)" + elif $all ; then + echo "Active UNIX domain sockets (servers and established)" + else + echo "Active UNIX domain sockets (w/o servers)" + fi + + echo "Proto RefCnt Flags Type State I-Node Path" + + unix_fmt="unix 2 [ ACC ] STREAM LISTENING %-8d %s\n" + if $all || $listen ; then + for i in $FAKE_NETSTAT_UNIX_LISTEN ; do + printf "$unix_fmt" 12345 "$i" + done + fi +fi diff --git a/ctdb/tests/eventscripts/stubs/nmap b/ctdb/tests/eventscripts/stubs/nmap new file mode 100755 index 00000000000..f01fe32d9c4 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/nmap @@ -0,0 +1,75 @@ +#!/bin/bash + +prog="nmap" + +# Pretty that we're the shell and that this command could not be +# found. +if [ "$FAKE_NMAP_NOT_FOUND" = "yes" ] ; then + echo "sh: ${prog}: command not found" >&2 + exit 127 +fi + +usage () +{ + cat >&2 <<EOF +Usage: $prog -n -oG - -PS 127.0.0.1 -p <port>[,<port> ...] + +A fake nmap stub that prints items depending on the variable +FAKE_TCP_LISTEN and the ports specified. + +Note that all options apart from -p are ignored. + +EOF + exit 1 +} + +ports="" + +parse_options () +{ + _temp=$(getopt -n "$prog" -a -o "np:" -l help -l PS: -l oG: -- "$@") + + [ $? != 0 ] && usage + + eval set -- "$_temp" + + while true ; do + case "$1" in + -n) shift ;; + --oG|--PS) shift 2 ;; + -p) ports="${ports}${ports:+ }${2//,/ }" ; shift 2 ;; + --) shift ; break ;; + -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable. + esac + done + + [ $# -gt 0 ] && usage + + [ -n "$ports" ] || usage +} + +# For printing out... +args="$*" + +parse_options "$@" + +port_states="" + +for p in $ports ; do + pn=$(getent services "$p" | sed -e 's@[[:space:]].*@@') + for i in $FAKE_TCP_LISTEN ; do + lp="${i##*:}" + if [ "$p" = "$lp" ] ; then + port_states="${port_states}${port_states:+, }${p}/open/tcp//${pn}///" + continue 2 + fi + done + port_states="${port_states}${port_states:+, }${p}/closed/tcp//${pn}///" +done + +cat <<EOF +# Nmap 5.21 scan initiated $(date) as: nmap $args +Host: 127.0.0.1 () Status: Up +Host: 127.0.0.1 () Ports: $port_states +# Nmap done at $(date) -- 1 IP address (1 host up) scanned in 0.04 seconds +EOF diff --git a/ctdb/tests/eventscripts/stubs/pidof b/ctdb/tests/eventscripts/stubs/pidof new file mode 100755 index 00000000000..b6ad6d83a8d --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/pidof @@ -0,0 +1,10 @@ +#!/bin/sh + +case "$1" in + nfsd) + echo "$FAKE_NFSD_THREAD_PIDS" + ;; + *) + echo "pidof: \"$1\" not implemented" + exit 1 +esac diff --git a/ctdb/tests/eventscripts/stubs/pkill b/ctdb/tests/eventscripts/stubs/pkill new file mode 100755 index 00000000000..b3f1de5a570 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/pkill @@ -0,0 +1,7 @@ +#!/bin/sh + +# Always succeed. This means that pkill -0 will always find a +# process and anything else will successfully kill. This should +# exercise a good avriety of code paths. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/ps b/ctdb/tests/eventscripts/stubs/ps new file mode 100755 index 00000000000..5abeaf94e18 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/ps @@ -0,0 +1,12 @@ +#!/bin/sh + +cat <<EOF +USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND +root 2 0.0 0.0 0 0 ? S Aug28 0:00 [kthreadd] +root 3 0.0 0.0 0 0 ? S Aug28 0:43 \_ [ksoftirqd/0] +... +root 1 0.0 0.0 2976 624 ? Ss Aug28 0:07 init [2] +root 495 0.0 0.0 3888 1640 ? Ss Aug28 0:00 udevd --daemon +... +[MORE FAKE ps OUTPUT] +EOF diff --git a/ctdb/tests/eventscripts/stubs/rpc.lockd b/ctdb/tests/eventscripts/stubs/rpc.lockd new file mode 100755 index 00000000000..e71f6cd8ff0 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/rpc.lockd @@ -0,0 +1,6 @@ +#!/bin/sh + +# Restart always "works". However, the test infrastructure may +# continue to mark the service as down, so that's what matters. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/rpc.mountd b/ctdb/tests/eventscripts/stubs/rpc.mountd new file mode 100755 index 00000000000..e71f6cd8ff0 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/rpc.mountd @@ -0,0 +1,6 @@ +#!/bin/sh + +# Restart always "works". However, the test infrastructure may +# continue to mark the service as down, so that's what matters. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/rpc.rquotad b/ctdb/tests/eventscripts/stubs/rpc.rquotad new file mode 100755 index 00000000000..e71f6cd8ff0 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/rpc.rquotad @@ -0,0 +1,6 @@ +#!/bin/sh + +# Restart always "works". However, the test infrastructure may +# continue to mark the service as down, so that's what matters. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/rpc.statd b/ctdb/tests/eventscripts/stubs/rpc.statd new file mode 100755 index 00000000000..e71f6cd8ff0 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/rpc.statd @@ -0,0 +1,6 @@ +#!/bin/sh + +# Restart always "works". However, the test infrastructure may +# continue to mark the service as down, so that's what matters. + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/rpcinfo b/ctdb/tests/eventscripts/stubs/rpcinfo new file mode 100755 index 00000000000..dd175f3d9cb --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/rpcinfo @@ -0,0 +1,79 @@ +#!/bin/bash + +prog="rpcinfo" + +usage () +{ + cat >&2 <<EOF +Usage: $prog -u host program [version] + +A fake rpcinfo stub that succeeds for items in FAKE_RPCINFO_SERVICES, +depending on command-line options. + +Note that "-u host" is ignored. + +EOF + exit 1 +} + +parse_options () +{ + # $POSIXLY_CORRECT means that the command passed to onnode can + # take options and getopt won't reorder things to make them + # options to this script. + _temp=$(POSIXLY_CORRECT=1 getopt -n "$prog" -o "u:h" -l unix -l help -- "$@") + + [ $? != 0 ] && usage + + eval set -- "$_temp" + + while true ; do + case "$1" in + -u) shift 2 ;; # ignore + --) shift ; break ;; + -h|--help|*) usage ;; # * shouldn't happen, so this is reasonable. + esac + done + + [ 1 -le $# -a $# -le 2 ] || usage + + p="$1" + v="$2" +} + +parse_options "$@" + +for i in ${FAKE_RPCINFO_SERVICES} ; do + # This is stupidly cummulative, but needs to happen after the + # initial split of the list above. + IFS="${IFS}:" + set -- $i + # $1 = program, $2 = low version, $3 = high version + + if [ "$1" = "$p" ] ; then + if [ -n "$v" ] ; then + if [ "$2" -le "$v" -a "$v" -le "$3" ] ; then + echo "program ${p} version ${v} ready and waiting" + exit 0 + else + echo "rpcinfo: RPC: Program/version mismatch; low version = ${2}, high version = ${3}" >&2 + echo "program ${p} version ${v} is not available" + exit 1 + fi + else + for j in $(seq $2 $3) ; do + echo "program ${p} version ${j} ready and waiting" + done + exit 0 + fi + fi +done + +echo "rpcinfo: RPC: Program not registered" >&2 +if [ -n "$v" ] ; then + echo "program ${p} version ${v} is not available" +else + echo "program ${p} is not available" +fi + +exit 1 diff --git a/ctdb/tests/eventscripts/stubs/service b/ctdb/tests/eventscripts/stubs/service new file mode 100755 index 00000000000..5f47b55e003 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/service @@ -0,0 +1,64 @@ +#!/bin/sh + +service_status_dir="${EVENTSCRIPTS_TESTS_VAR_DIR}/service_fake_status" +mkdir -p "$service_status_dir" + +service="$1" +flag="${service_status_dir}/${service}" + +start() +{ + if [ -f "$flag" ] ; then + echo "service: can't start ${service} - already running" + exit 1 + else + touch "$flag" + echo "Starting ${service}: OK" + fi +} + +stop () +{ + if [ -f "$flag" ] ; then + echo "Stopping ${service}: OK" + rm -f "$flag" + else + echo "service: can't stop ${service} - not running" + exit 1 + fi +} + +case "$2" in + start) + start + ;; + stop) + stop + ;; + restart|reload) + stop + start + ;; + status) + if [ -f "$flag" ] ; then + echo "$service running" + exit 0 + else + echo "$service not running" + exit 3 + fi + ;; + force-started) + # For test setup... + touch "$flag" + ;; + force-stopped) + # For test setup... + rm -f "$flag" + ;; + *) + echo "service $service $2 not supported" + exit 1 +esac + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/sleep b/ctdb/tests/eventscripts/stubs/sleep new file mode 100755 index 00000000000..e4542444de7 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/sleep @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "$FAKE_SLEEP_REALLY" = "yes" ] ; then + /bin/sleep "$@" +elif [ -n "$FAKE_SLEEP_FORCE" ] ; then + /bin/sleep "$FAKE_SLEEP_FORCE" +else + : +fi diff --git a/ctdb/tests/eventscripts/stubs/tdbdump b/ctdb/tests/eventscripts/stubs/tdbdump new file mode 100755 index 00000000000..986c5c55499 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/tdbdump @@ -0,0 +1,9 @@ +#!/bin/sh + +if [ "$FAKE_TDB_IS_OK" = "yes" ] ; then + echo "TDB good" + exit 0 +else + echo "TDB busted" + exit 1 +fi diff --git a/ctdb/tests/eventscripts/stubs/tdbtool b/ctdb/tests/eventscripts/stubs/tdbtool new file mode 100755 index 00000000000..c6c0a166044 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/tdbtool @@ -0,0 +1,15 @@ +#!/bin/sh + +if [ -z "$1" ] ; then + if [ "$FAKE_TDBTOOL_SUPPORTS_CHECK" = "yes" ] ; then + echo "check" + fi +fi + +if [ "$FAKE_TDB_IS_OK" = "yes" ] ; then + echo "Database integrity is OK" +else + echo "Database is busted" +fi + +exit 0 diff --git a/ctdb/tests/eventscripts/stubs/testparm b/ctdb/tests/eventscripts/stubs/testparm new file mode 100755 index 00000000000..aac5b181105 --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/testparm @@ -0,0 +1,51 @@ +#!/bin/sh + +not_implemented () +{ + echo "testparm: option \"$1\" not implemented in stub" >&2 + exit 2 +} + +# Ensure that testparm always uses our canned configuration instead of +# the global one, unless some other file is specified. + +file="" +parameter="" +for i ; do + case "$i" in + --parameter-name=*) parameter="${i#--parameter-name=}" ;; + -*) : ;; + *) file="$i" ;; + esac +done + +# Just hard-code parameter requests for now. Later on they could be +# parsed out of the file. +case "$parameter" in + security) echo "ADS" ; exit 0 ;; + smb*ports) echo "445, 139" ; exit 0 ;; + ?*) not_implemented "--parameter-name=$parameter" ;; + # Fall through if $parameter not set +esac + +if [ -n "$file" ] ; then + # This should include the shares, since this is used when the + # samba eventscript caches the output. + cat "$file" +else + # We force our own smb.conf and add the shares. + cat "${CTDB_ETCDIR}/samba/smb.conf" + + for i in $FAKE_SHARES ; do + bi=$(basename "$i") +cat <<EOF + +[${bi}] + path = $i + comment = fake share $bi + guest ok = no + read only = no + browseable = yes +EOF + done +fi diff --git a/ctdb/tests/eventscripts/stubs/wbinfo b/ctdb/tests/eventscripts/stubs/wbinfo new file mode 100755 index 00000000000..4fc6b98331f --- /dev/null +++ b/ctdb/tests/eventscripts/stubs/wbinfo @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ "$FAKE_WBINFO_FAIL" = "yes" ] ; then + exit 1 +fi + +exit 0 diff --git a/ctdb/tests/onnode/0001.sh b/ctdb/tests/onnode/0001.sh new file mode 100755 index 00000000000..28533748f2c --- /dev/null +++ b/ctdb/tests/onnode/0001.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE all hostname" + +define_test "$cmd" "all nodes OK" + +required_result <<EOF + +>> NODE: 192.168.1.101 << +-n 192.168.1.101 hostname + +>> NODE: 192.168.1.102 << +-n 192.168.1.102 hostname + +>> NODE: 192.168.1.103 << +-n 192.168.1.103 hostname + +>> NODE: 192.168.1.104 << +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0002.sh b/ctdb/tests/onnode/0002.sh new file mode 100755 index 00000000000..c3c8c77a0ed --- /dev/null +++ b/ctdb/tests/onnode/0002.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE -q all hostname" + +define_test "$cmd" "all nodes OK" + +required_result <<EOF +-n 192.168.1.101 hostname +-n 192.168.1.102 hostname +-n 192.168.1.103 hostname +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0003.sh b/ctdb/tests/onnode/0003.sh new file mode 100755 index 00000000000..d79bca28061 --- /dev/null +++ b/ctdb/tests/onnode/0003.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE -p all hostname" + +define_test "$cmd" "all nodes OK" + +required_result <<EOF +[192.168.1.101] -n 192.168.1.101 hostname +[192.168.1.102] -n 192.168.1.102 hostname +[192.168.1.103] -n 192.168.1.103 hostname +[192.168.1.104] -n 192.168.1.104 hostname +EOF + +simple_test -s $cmd diff --git a/ctdb/tests/onnode/0004.sh b/ctdb/tests/onnode/0004.sh new file mode 100755 index 00000000000..d0986b2ffda --- /dev/null +++ b/ctdb/tests/onnode/0004.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE -pq all hostname" + +define_test "$cmd" "all nodes OK" + +required_result <<EOF +-n 192.168.1.101 hostname +-n 192.168.1.102 hostname +-n 192.168.1.103 hostname +-n 192.168.1.104 hostname +EOF + +simple_test -s $cmd diff --git a/ctdb/tests/onnode/0005.sh b/ctdb/tests/onnode/0005.sh new file mode 100755 index 00000000000..0eccbb04a39 --- /dev/null +++ b/ctdb/tests/onnode/0005.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE 3 hostname" + +define_test "$cmd" "all nodes OK" + +required_result <<EOF +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0006.sh b/ctdb/tests/onnode/0006.sh new file mode 100755 index 00000000000..b027850240e --- /dev/null +++ b/ctdb/tests/onnode/0006.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE -v 3 hostname" + +define_test "$cmd" "all nodes OK" + +required_result <<EOF + +>> NODE: 192.168.1.104 << +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0070.sh b/ctdb/tests/onnode/0070.sh new file mode 100755 index 00000000000..b071e80a800 --- /dev/null +++ b/ctdb/tests/onnode/0070.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE ok hostname" + +define_test "$cmd" "all nodes OK" + +ctdb_set_output <<EOF +:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:ThisNode: +:0:192.168.1.101:0:0:0:0:0:0:0:Y: +:1:192.168.1.102:0:0:0:0:0:0:0:N: +:2:192.168.1.103:0:0:0:0:0:0:0:N: +:3:192.168.1.104:0:0:0:0:0:0:0:N: +EOF + +required_result <<EOF + +>> NODE: 192.168.1.101 << +-n 192.168.1.101 hostname + +>> NODE: 192.168.1.102 << +-n 192.168.1.102 hostname + +>> NODE: 192.168.1.103 << +-n 192.168.1.103 hostname + +>> NODE: 192.168.1.104 << +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0071.sh b/ctdb/tests/onnode/0071.sh new file mode 100755 index 00000000000..d594323e3fd --- /dev/null +++ b/ctdb/tests/onnode/0071.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE ok hostname" + +define_test "$cmd" "2nd node disconnected" + +ctdb_set_output <<EOF +:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:ThisNode: +:0:192.168.1.101:0:0:0:0:0:0:0:Y: +:1:192.168.1.102:1:0:0:0:0:0:0:N: +:2:192.168.1.103:0:0:0:0:0:0:0:N: +:3:192.168.1.104:0:0:0:0:0:0:0:N: +EOF + +required_result <<EOF + +>> NODE: 192.168.1.101 << +-n 192.168.1.101 hostname + +>> NODE: 192.168.1.103 << +-n 192.168.1.103 hostname + +>> NODE: 192.168.1.104 << +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0072.sh b/ctdb/tests/onnode/0072.sh new file mode 100755 index 00000000000..cb29e3b9a2c --- /dev/null +++ b/ctdb/tests/onnode/0072.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE ok hostname" + +define_test "$cmd" "2nd node disconnected, extra status columns" + +ctdb_set_output <<EOF +:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:X1:X2:X3:X4: +:0:192.168.1.101:0:0:0:0:0:0:0:0:0:0: +:1:192.168.1.102:1:0:0:0:0:0:0:0:0:0: +:2:192.168.1.103:0:0:0:0:0:0:0:0:0:0: +:3:192.168.1.104:0:0:0:0:0:0:0:0:0:0: +EOF + +required_result <<EOF + +>> NODE: 192.168.1.101 << +-n 192.168.1.101 hostname + +>> NODE: 192.168.1.103 << +-n 192.168.1.103 hostname + +>> NODE: 192.168.1.104 << +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0075.sh b/ctdb/tests/onnode/0075.sh new file mode 100755 index 00000000000..4276e9c12c8 --- /dev/null +++ b/ctdb/tests/onnode/0075.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE con hostname" + +define_test "$cmd" "1st node disconnected" + +ctdb_set_output <<EOF +:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:ThisNode: +:0:192.168.1.101:1:0:0:0:0:0:0:N: +:1:192.168.1.102:0:0:0:0:0:0:0:Y: +:2:192.168.1.103:0:0:0:0:0:0:0:N: +:3:192.168.1.104:0:0:0:0:0:0:0:N: +EOF + +required_result <<EOF + +>> NODE: 192.168.1.102 << +-n 192.168.1.102 hostname + +>> NODE: 192.168.1.103 << +-n 192.168.1.103 hostname + +>> NODE: 192.168.1.104 << +-n 192.168.1.104 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0080.sh b/ctdb/tests/onnode/0080.sh new file mode 100755 index 00000000000..bca478ada5f --- /dev/null +++ b/ctdb/tests/onnode/0080.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE recmaster hostname" + +define_test "$cmd" "node 1 (192.168.1.102) is recmaster" + +ctdb_set_output <<EOF +1 +EOF + +required_result <<EOF +-n 192.168.1.102 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0081.sh b/ctdb/tests/onnode/0081.sh new file mode 100755 index 00000000000..412db87e4c6 --- /dev/null +++ b/ctdb/tests/onnode/0081.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE lvsmaster hostname" + +define_test "$cmd" "no lvsmaster" + +ctdb_set_output 255 <<EOF +There is no LVS master +EOF + +required_result 1 <<EOF +onnode: No lvsmaster available +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0090.sh b/ctdb/tests/onnode/0090.sh new file mode 100755 index 00000000000..dd50c51b70d --- /dev/null +++ b/ctdb/tests/onnode/0090.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE natgw hostname" + +define_test "$cmd" "no natgw" + +ctdb_set_output <<EOF +-1 0.0.0.0 +:0:192.168.1.101:0:0:0:0:0: +:1:192.168.1.102:0:0:0:0:0: +:2:192.168.1.103:0:0:0:0:0: +:3:192.168.1.104:0:0:0:0:0: +EOF + +required_result 1 <<EOF +onnode: No natgwlist available +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/0091.sh b/ctdb/tests/onnode/0091.sh new file mode 100755 index 00000000000..528eec16df1 --- /dev/null +++ b/ctdb/tests/onnode/0091.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +cmd="$ONNODE natgw hostname" + +define_test "$cmd" "node 2 (192.168.1.103) is natgw" + +ctdb_set_output <<EOF +2 192.168.1.103 +:0:192.168.1.101:0:0:0:0:0: +:1:192.168.1.102:0:0:0:0:0: +:2:192.168.1.103:0:0:0:0:0: +:3:192.168.1.104:0:0:0:0:0: +EOF + +required_result <<EOF +-n 192.168.1.103 hostname +EOF + +simple_test $cmd diff --git a/ctdb/tests/onnode/README b/ctdb/tests/onnode/README new file mode 100644 index 00000000000..5bb69524df2 --- /dev/null +++ b/ctdb/tests/onnode/README @@ -0,0 +1,36 @@ +onnode unit tests +================= + +Examples: + +* ../run_tests.sh . + + Run all tests, displaying output. + +* ../run_tests.sh -s . + + Run all tests, displaying output and a summary. + +* ../run_tests.sh -sq . + + Run all tests, displaying only a summary. + +* ONNODE=onnode-buggy-001 ../run_tests.sh -s . + + Run against stubs/onnode-buggy-001 instead of default onnode version. + + Add more buggy versions of onnode to this directory as bugs are + fixed to enable test validation using this feature. + +* ../run_tests.sh ./009*.sh + + Run only the specified tests. + +* ONNODE="stubs/onnode-buggy-001" ../run_tests.sh -X ./0090.sh + ../run_tests.sh -X ./0090.sh + + Debug the specified test or test failure by tracing onnode with + "bash -x". The test will fail because the bash trace output will be + included in the test output. + + To see if the test pases, the -X can be dropped... diff --git a/ctdb/tests/onnode/nodes b/ctdb/tests/onnode/nodes new file mode 100644 index 00000000000..e2fe268e8d4 --- /dev/null +++ b/ctdb/tests/onnode/nodes @@ -0,0 +1,4 @@ +192.168.1.101 +192.168.1.102 +192.168.1.103 +192.168.1.104 diff --git a/ctdb/tests/onnode/scripts/local.sh b/ctdb/tests/onnode/scripts/local.sh new file mode 100644 index 00000000000..9973a555633 --- /dev/null +++ b/ctdb/tests/onnode/scripts/local.sh @@ -0,0 +1,86 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Set indirectly by run_tests at top level. +unset CTDB_NODES_SOCKETS + +# Default to just "onnode". +: ${ONNODE:=onnode} + +# Augment PATH with relevant stubs/ directories. + +if [ -d "${TEST_SUBDIR}/stubs" ] ; then + PATH="${TEST_SUBDIR}/stubs:$PATH" +fi + +# Find CTDB nodes file. +if [ -z "$CTDB_NODES_FILE" ] ; then + if [ -r "${TEST_SUBDIR}/nodes" ] ; then + CTDB_NODES_FILE="${TEST_SUBDIR}/nodes" + else + CTDB_NODES_FILE="${CTDB_BASE:-/etc/ctdb}/nodes" + fi +fi + +export CTDB_NODES_FILE + +export ONNODE_TESTS_VAR_DIR="${TEST_VAR_DIR}/unit_onnode" +mkdir -p "$ONNODE_TESTS_VAR_DIR" + +if [ -z "$CTDB_BASE" ] ; then + export CTDB_BASE=$(dirname "$CTDB_NODES_FILE") +fi + +define_test () +{ + _f=$(basename "$0") + + echo "$_f $1 - $2" +} + +# Set output for ctdb command. Option 1st argument is return code. +ctdb_set_output () +{ + _out="$ONNODE_TESTS_VAR_DIR/ctdb.out" + cat >"$_out" + + _rc="$ONNODE_TESTS_VAR_DIR/ctdb.rc" + echo "${1:-0}" >"$_rc" + + trap "rm -f $_out $_rc" 0 +} + +_extra_header () +{ + cat <<EOF +CTDB_NODES_FILE="${CTDB_NODES_FILE}" +CTDB_BASE="$CTDB_BASE" +$(which ctdb) + +EOF +} + +simple_test () +{ + _sort="cat" + if [ "$1" = "-s" ] ; then + shift + _sort="sort" + fi + + if $TEST_COMMAND_TRACE ; then + _onnode=$(which "$1") ; shift + _out=$(bash -x "$_onnode" "$@" 2>&1) + else + _out=$("$@" 2>&1) + fi + _rc=$? + _out=$(echo "$_out" | $_sort ) + + # Can't do this inline or it affects return code + _extra_header="$(_extra_header)" + + # Get the return code back into $? + (exit $_rc) + + result_check "$_extra_header" +} diff --git a/ctdb/tests/onnode/stubs/ctdb b/ctdb/tests/onnode/stubs/ctdb new file mode 100755 index 00000000000..e420d25e029 --- /dev/null +++ b/ctdb/tests/onnode/stubs/ctdb @@ -0,0 +1,33 @@ +#!/bin/sh + +# Fake ctdb client for onnode tests. + +cmd=$(echo "$*" | sed -r -e 's@[[:space:]]+@_@g') + +out="${ONNODE_TESTS_VAR_DIR}/ctdb.out" +if [ -r "$out" ] ; then + cat "$out" + + rc="${ONNODE_TESTS_VAR_DIR}/ctdb.rc" + if [ -r "$rc" ] ; then + exit $(cat "$rc") + fi + + exit 0 +fi + +f="${ONNODE_TESTCASE_DIR}/ctdb.d/${cmd}.sh" +if [ -x "$f" ] ; then + "$f" + exit $? +fi + +f="${ONNODE_TESTCASE_DIR}/ctdb.d/${cmd}.out" +if [ -r "$f" ] ; then + cat "$f" + exit 0 +fi + +echo "fake ctdb: no implementation for \"$*\"" + +exit 1 diff --git a/ctdb/tests/onnode/stubs/onnode-buggy-001 b/ctdb/tests/onnode/stubs/onnode-buggy-001 new file mode 100755 index 00000000000..77a1207d6c2 --- /dev/null +++ b/ctdb/tests/onnode/stubs/onnode-buggy-001 @@ -0,0 +1,376 @@ +#!/bin/bash + +# Run commands on CTDB nodes. + +# See http://ctdb.samba.org/ for more information about CTDB. + +# Copyright (C) Martin Schwenke 2008 + +# Based on an earlier script by Andrew Tridgell and Ronnie Sahlberg. + +# Copyright (C) Andrew Tridgell 2007 + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <http://www.gnu.org/licenses/>. + +prog=$(basename $0) + +usage () +{ + cat >&2 <<EOF +Usage: onnode [OPTION] ... <NODES> <COMMAND> ... + options: + -c Run in current working directory on specified nodes. + -o <prefix> Save standard output from each node to file <prefix>.<ip> + -p Run command in parallel on specified nodes. + -q Do not print node addresses (overrides -v). + -n Allow nodes to be specified by name. + -f Specify nodes file, overrides CTDB_NODES_FILE. + -v Print node address even for a single node. + <NODES> "all", "any", "ok" (or "healthy"), "con" (or "connected"), + "rm" (or "recmaster"), "lvs" (or "lvsmaster"), + "natgw" (or "natgwlist"); or + a node number (0 base); or + a hostname (if -n is specified); or + list (comma separated) of <NODES>; or + range (hyphen separated) of node numbers. +EOF + exit 1 + +} + +invalid_nodespec () +{ + echo "Invalid <nodespec>" >&2 ; echo >&2 + usage +} + +# Defaults. +current=false +parallel=false +verbose=false +quiet=false +prefix="" +names_ok=false + +ctdb_base="${CTDB_BASE:-/etc/ctdb}" + +parse_options () +{ + # $POSIXLY_CORRECT means that the command passed to onnode can + # take options and getopt won't reorder things to make them + # options ot onnode. + local temp + # Not on the previous line - local returns 0! + temp=$(POSIXLY_CORRECT=1 getopt -n "$prog" -o "cf:hno:pqv" -l help -- "$@") + + [ $? != 0 ] && usage + + eval set -- "$temp" + + while true ; do + case "$1" in + -c) current=true ; shift ;; + -f) CTDB_NODES_FILE="$2" ; shift 2 ;; + -n) names_ok=true ; shift ;; + -o) prefix="$2" ; shift 2 ;; + -p) parallel=true ; shift ;; + -q) quiet=true ; shift ;; + -v) verbose=true ; shift ;; + --) shift ; break ;; + -h|--help|*) usage ;; # Shouldn't happen, so this is reasonable. + esac + done + + [ $# -lt 2 ] && usage + + nodespec="$1" ; shift + command="$@" +} + +echo_nth () +{ + local n="$1" ; shift + + shift $n + local node="$1" + + if [ -n "$node" -a "$node" != "#DEAD" ] ; then + echo $node + else + echo "${prog}: \"node ${n}\" does not exist" >&2 + exit 1 + fi +} + +parse_nodespec () +{ + # Subshell avoids hacks to restore $IFS. + ( + IFS="," + for i in $1 ; do + case "$i" in + *-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;; + # Separate lines for readability. + all|any|ok|healthy|con|connected) echo "$i" ;; + rm|recmaster|lvs|lvsmaster|natgw|natgwlist) echo "$i" ;; + *) + [ $i -gt -1 ] 2>/dev/null || $names_ok || invalid_nodespec + echo $i + esac + done + ) +} + +ctdb_status_output="" # cache +get_nodes_with_status () +{ + local all_nodes="$1" + local status="$2" + + local bits + case "$status" in + healthy) + bits="0:0:0:0:0:0" + ;; + connected) + bits="0:[0-1]:[0-1]:[0-1]:[0-1]:[0-1]" + ;; + *) + invalid_nodespec + esac + + if [ -z "$ctdb_status_output" ] ; then + # FIXME: need to do something if $CTDB_NODES_SOCKETS is set. + ctdb_status_output=$(ctdb -Y status 2>/dev/null) + if [ $? -ne 0 ] ; then + echo "${prog}: unable to get status of CTDB nodes" >&2 + exit 1 + fi + ctdb_status_output="${ctdb_status_output#* }" + fi + + local nodes="" + local i + for i in $ctdb_status_output ; do + # Try removing bits from end. + local t="${i%:${bits}:}" + if [ "$t" != "$i" ] ; then + # Succeeded. Get address. NOTE: this is an optimisation. + # It might be better to get the node number and then get + # the nth node to get the address. This would make things + # more consistent if $ctdb_base/nodes actually contained + # hostnames. + nodes="${nodes} ${t#:*:}" + fi + done + + echo $nodes +} + +ctdb_props="" # cache +get_node_with_property () +{ + local all_nodes="$1" + local prop="$2" + + local prop_node="" + if [ "${ctdb_props##:${prop}:}" = "$ctdb_props" ] ; then + prop_node=$(ctdb "$prop" -Y 2>/dev/null) + # We only want the first line. + local nl=" +" + prop_node="${prop_node%%${nl}*}" + if [ $? -eq 0 ] ; then + ctdb_props="${ctdb_props}${ctdb_props:+ }:${prop}:${prop_node}" + else + prop_node="" + fi + else + prop_node="${ctdb_props##:${prop}:}" + prop_node="${prop_node%% *}" + fi + if [ -n "$prop_node" ] ; then + echo_nth "$prop_node" $all_nodes + else + echo "${prog}: No ${prop} available" >&2 + exit 1 + fi +} + +get_any_available_node () +{ + local all_nodes="$1" + + # We do a recursive onnode to find which nodes are up and running. + local out=$($0 -pq all ctdb pnn 2>&1) + local line + while read line ; do + local pnn="${line#PNN:}" + if [ "$pnn" != "$line" ] ; then + echo_nth "$pnn" $all_nodes + return 0 + fi + # Else must be an error message from a down node. + done <<<"$out" + return 1 +} + +get_nodes () +{ + local all_nodes + + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + all_nodes="$CTDB_NODES_SOCKETS" + else + local f="${ctdb_base}/nodes" + if [ -n "$CTDB_NODES_FILE" ] ; then + f="$CTDB_NODES_FILE" + if [ ! -e "$f" -a "${f#/}" = "$f" ] ; then + # $f is relative, try in $ctdb_base + f="${ctdb_base}/${f}" + fi + fi + + if [ ! -r "$f" ] ; then + echo "${prog}: unable to open nodes file \"${f}\"" >&2 + exit 1 + fi + + all_nodes=$(sed -e 's@#.*@@g' -e 's@ *@@g' -e 's@^$@#DEAD@' "$f") + fi + + local nodes="" + local n + for n in $(parse_nodespec "$1") ; do + [ $? != 0 ] && exit 1 # Required to catch exit in above subshell. + case "$n" in + all) + echo "${all_nodes//#DEAD/}" + ;; + any) + get_any_available_node "$all_nodes" || exit 1 + ;; + ok|healthy) + get_nodes_with_status "$all_nodes" "healthy" || exit 1 + ;; + con|connected) + get_nodes_with_status "$all_nodes" "connected" || exit 1 + ;; + rm|recmaster) + get_node_with_property "$all_nodes" "recmaster" || exit 1 + ;; + lvs|lvsmaster) + get_node_with_property "$all_nodes" "lvsmaster" || exit 1 + ;; + natgw|natgwlist) + get_node_with_property "$all_nodes" "natgwlist" || exit 1 + ;; + [0-9]|[0-9][0-9]|[0-9][0-9][0-9]) + echo_nth $n $all_nodes + ;; + *) + $names_ok || invalid_nodespec + echo $n + esac + done +} + +fakessh () +{ + CTDB_SOCKET="$1" sh -c "$2" 3>/dev/null +} + +stdout_filter () +{ + if [ -n "$prefix" ] ; then + cat >"${prefix}.${n//\//_}" + elif $verbose && $parallel ; then + sed -e "s@^@[$n] @" + else + cat + fi +} + +stderr_filter () +{ + if $verbose && $parallel ; then + sed -e "s@^@[$n] @" + else + cat + fi +} + +###################################################################### + +parse_options "$@" + +$current && command="cd $PWD && $command" + +ssh_opts= +if [ -n "$CTDB_NODES_SOCKETS" ] ; then + SSH=fakessh +else + # Could "2>/dev/null || true" but want to see errors from typos in file. + [ -r "${ctdb_base}/onnode.conf" ] && . "${ctdb_base}/onnode.conf" + [ -n "$SSH" ] || SSH=ssh + if [ "$SSH" = "ssh" ] ; then + ssh_opts="-n" + else + : # rsh? All bets are off! + fi +fi + +###################################################################### + +nodes=$(get_nodes "$nodespec") +[ $? != 0 ] && exit 1 # Required to catch exit in above subshell. + +if $quiet ; then + verbose=false +else + # If $nodes contains a space or a newline then assume multiple nodes. + nl=" +" + [ "$nodes" != "${nodes%[ ${nl}]*}" ] && verbose=true +fi + +pids="" +trap 'kill -TERM $pids 2>/dev/null' INT TERM +# There's a small race here where the kill can fail if no processes +# have been added to $pids and the script is interrupted. However, +# the part of the window where it matter is very small. +retcode=0 +for n in $nodes ; do + set -o pipefail 2>/dev/null + if $parallel ; then + { exec 3>&1 ; { $SSH $ssh_opts $EXTRA_SSH_OPTS $n "$command" | stdout_filter >&3 ; } 2>&1 | stderr_filter ; } & + pids="${pids} $!" + else + if $verbose ; then + echo >&2 ; echo ">> NODE: $n <<" >&2 + fi + + { exec 3>&1 ; { $SSH $ssh_opts $EXTRA_SSH_OPTS $n "$command" | stdout_filter >&3 ; } 2>&1 | stderr_filter ; } + [ $? = 0 ] || retcode=$? + fi +done + +$parallel && { + for p in $pids; do + wait $p + [ $? = 0 ] || retcode=$? + done +} + +exit $retcode diff --git a/ctdb/tests/onnode/stubs/ssh b/ctdb/tests/onnode/stubs/ssh new file mode 100755 index 00000000000..7be778f1b92 --- /dev/null +++ b/ctdb/tests/onnode/stubs/ssh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "$*" diff --git a/ctdb/tests/recover.sh b/ctdb/tests/recover.sh new file mode 100755 index 00000000000..c626441786b --- /dev/null +++ b/ctdb/tests/recover.sh @@ -0,0 +1,107 @@ +#!/bin/sh + +killall -q ctdbd + +echo "Starting 4 ctdb daemons" +bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt +bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.2 --socket=/tmp/ctdb.socket.127.0.0.2 +bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.3 --socket=/tmp/ctdb.socket.127.0.0.3 +bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.4 --socket=/tmp/ctdb.socket.127.0.0.4 + +echo +echo "Attaching to some databases" +bin/ctdb_control attach test1.tdb || exit 1 +bin/ctdb_control attach test2.tdb || exit 1 +bin/ctdb_control attach test3.tdb || exit 1 +bin/ctdb_control attach test4.tdb || exit 1 + +echo "Clearing all databases to make sure they are all empty" +bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^dbid://" -e "s/ .*$//" | while read DB; do + seq 0 3 | while read NODE; do + bin/ctdb_control cleardb $NODE $DB + done +done + + +echo +echo +echo "Printing all databases on all nodes. they should all be empty" +echo "=============================================================" +bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^.*name://" -e "s/ .*$//" | while read DBNAME; do + seq 0 3 | while read NODE; do + echo "Content of DBNAME:$DBNAME NODE:$NODE :" + bin/ctdb_control catdb $DBNAME $NODE + done +done + +echo +echo +echo "Populating the databases" +./bin/ctdb_control writerecord 0 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control setdmaster 0 0x220c2a7b 1 + +./bin/ctdb_control writerecord 1 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control writerecord 1 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control setdmaster 1 0x220c2a7b 2 + +./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control setdmaster 2 0x220c2a7b 3 + +./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1 +./bin/ctdb_control setdmaster 3 0x220c2a7b 3 + + +echo +echo +echo "Printing all databases on all nodes. there should be a record there" +echo "=============================================================" +bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^.*name://" -e "s/ .*$//" | while read DBNAME; do + seq 0 3 | while read NODE; do + echo "Content of DBNAME:$DBNAME NODE:$NODE :" + bin/ctdb_control catdb $DBNAME $NODE + done +done + +echo +echo +echo "killing off node #2" +echo "===================" +CTDBPID=`./bin/ctdb_control getpid 2 | sed -e "s/Pid://"` +kill $CTDBPID +sleep 1 + + +echo +echo +echo "wait 3 seconds to let the recovery daemon do its job" +echo "====================================================" +sleep 3 + +echo +echo +echo "Printing all databases on all nodes." +echo "The databases should be the same now on all nodes" +echo "and the record will have been migrated to node 0" +echo "=================================================" +echo "Node 0:" +bin/ctdb_control catdb test4.tdb 0 +echo "Node 1:" +bin/ctdb_control catdb test4.tdb 1 +echo "Node 3:" +bin/ctdb_control catdb test4.tdb 3 +echo "nodemap:" +bin/ctdb_control getnodemap 0 + +echo +echo +echo "Traverse the cluster and dump the database" +bin/ctdb_control catdb test4.tdb + + +#leave the ctdb daemons running so one can look at the box in more detail +#killall -q ctdbd diff --git a/ctdb/tests/run_cluster_tests.sh b/ctdb/tests/run_cluster_tests.sh new file mode 120000 index 00000000000..5236e32d82f --- /dev/null +++ b/ctdb/tests/run_cluster_tests.sh @@ -0,0 +1 @@ +run_tests.sh
\ No newline at end of file diff --git a/ctdb/tests/run_tests.sh b/ctdb/tests/run_tests.sh new file mode 100755 index 00000000000..5fcd89d5dee --- /dev/null +++ b/ctdb/tests/run_tests.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +test_dir=$(dirname "$0") + +case $(basename "$0") in + *run_cluster_tests*) + # Running on a cluster: + # * print summary, run any integration tests against cluster + # * default to running: all integration tests, no unit tests + opts="-s" + tests="simple complex" + ;; + *) + # Running on local machine: + # * print summary, run any integration tests against local daemons + # * default to running: all unit tests, simple integration tests + opts="-s -l" + tests="onnode takeover tool eventscripts simple" + # If running in the source tree then use a fixed TEST_VAR_DIR. + # If this script is installed using the INSTALL script then + # TEST_BIN_DIR will be set, so use this as the test. + if [ -z "$TEST_BIN_DIR" ] ; then + opts="${opts} -V ${test_dir}/var" + fi +esac + +# Allow options to be passed to this script. However, if any options +# are passed there must be a "--" between the options and the tests. +# This makes it easy to handle options that take arguments. +case "$1" in + -*) + while [ -n "$1" ] ; do + case "$1" in + --) shift ; break ;; + *) opts="$opts $1" ; shift ;; + esac + done +esac + +# If no tests are specified, then run the defaults. +[ -n "$1" ] || set -- $tests + +"${test_dir}/scripts/run_tests" $opts "$@" || exit 1 + +echo "All OK" +exit 0 diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh new file mode 100644 index 00000000000..64a176b9174 --- /dev/null +++ b/ctdb/tests/scripts/common.sh @@ -0,0 +1,41 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Common variables and functions for all CTDB tests. + +# This expands the most probable problem cases like "." and "..". +TEST_SUBDIR=$(dirname "$0") +if [ $(dirname "$TEST_SUBDIR") = "." ] ; then + TEST_SUBDIR=$(cd "$TEST_SUBDIR" ; pwd) +fi + +_test_dir=$(dirname "$TEST_SUBDIR") + +# If we are running from within the source tree then, depending on the +# tests that we're running, we may need to add the top level bin/ and +# tools/ subdirectories to $PATH. This means we need a way of +# determining if we're running from within the source tree. There is +# no use looking outside the tests/ subdirectory because anything +# above that level may be meaningless and outside our control. +# Therefore, we'll use existence of $_test_dir/run_tests.sh to +# indicate that we're running in-tree - on a system where the tests +# have been installed, this file will be absent (renamed and placed in +# some bin/ directory). +if [ -f "${_test_dir}/run_tests.sh" ] ; then + ctdb_dir=$(dirname "$_test_dir") + + _tools_dir="${ctdb_dir}/tools" + if [ -d "$_tools_dir" ] ; then + PATH="${_tools_dir}:$PATH" + fi +fi + +_test_bin_dir="${TEST_BIN_DIR:-${_test_dir}/bin}" +if [ -d "$_test_bin_dir" ] ; then + PATH="${_test_bin_dir}:$PATH" +fi + +# Print a message and exit. +die () +{ + echo "$1" >&2 ; exit ${2:-1} +} diff --git a/ctdb/tests/scripts/integration.bash b/ctdb/tests/scripts/integration.bash new file mode 100644 index 00000000000..040a36048de --- /dev/null +++ b/ctdb/tests/scripts/integration.bash @@ -0,0 +1,980 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +. "${TEST_SCRIPTS_DIR}/common.sh" + +# If we're not running on a real cluster then we need a local copy of +# ctdb (and other stuff) in $PATH and we will use local daemons. +if [ -n "$TEST_LOCAL_DAEMONS" ] ; then + export CTDB_NODES_SOCKETS="" + for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do + CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${TEST_VAR_DIR}/sock.${i}" + done + + # Use in-tree binaries if running against local daemons. + # Otherwise CTDB need to be installed on all nodes. + if [ -n "$ctdb_dir" -a -d "${ctdb_dir}/bin" ] ; then + PATH="${ctdb_dir}/bin:${PATH}" + export CTDB_LOCK_HELPER="${ctdb_dir}/bin/ctdb_lock_helper" + fi + + export CTDB_NODES="${TEST_VAR_DIR}/nodes.txt" +fi + +###################################################################### + +export CTDB_TIMEOUT=60 + +if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then + CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" +else + _d=$(cd ${TEST_SCRIPTS_DIR}; echo $PWD) + CTDB_TEST_WRAPPER="$_d/test_wrap" +fi +export CTDB_TEST_WRAPPER + +# If $VALGRIND is set then use it whenever ctdb is called, but only if +# $CTDB is not already set. +[ -n "$CTDB" ] || export CTDB="${VALGRIND}${VALGRIND:+ }ctdb" + +# why??? +PATH="${TEST_SCRIPTS_DIR}:${PATH}" + +###################################################################### + +ctdb_check_time_logs () +{ + local threshold=20 + + local jump=false + local prev="" + local ds_prev="" + local node="" + + out=$(onnode all tail -n 20 "${TEST_VAR_DIR}/ctdb.test.time.log" 2>&1) + + if [ $? -eq 0 ] ; then + local line + while read line ; do + case "$line" in + \>\>\ NODE:\ *\ \<\<) + node="${line#>> NODE: }" + node=${node% <<*} + ds_prev="" + ;; + *\ *) + set -- $line + ds_curr="$1${2:0:1}" + if [ -n "$ds_prev" ] && \ + [ $(($ds_curr - $ds_prev)) -ge $threshold ] ; then + echo "Node $node had time jump of $(($ds_curr - $ds_prev))ds between $(date +'%T' -d @${ds_prev%?}) and $(date +'%T' -d @${ds_curr%?})" + jump=true + fi + prev="$line" + ds_prev="$ds_curr" + ;; + esac + done <<<"$out" + else + echo Error getting time logs + fi + if $jump ; then + echo "Check time sync (test client first):" + date + onnode -p all date + echo "Information from test client:" + hostname + top -b -n 1 + echo "Information from cluster nodes:" + onnode all "top -b -n 1 ; echo '/proc/slabinfo' ; cat /proc/slabinfo" + fi +} + +ctdb_test_exit () +{ + local status=$? + + trap - 0 + + [ $(($testfailures+0)) -eq 0 -a $status -ne 0 ] && testfailures=$status + status=$(($testfailures+0)) + + # Avoid making a test fail from this point onwards. The test is + # now complete. + set +e + + echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." + + if [ -z "$TEST_LOCAL_DAEMONS" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ + $status -ne 0 ] ; then + ctdb_check_time_logs + fi + + eval "$ctdb_test_exit_hook" || true + unset ctdb_test_exit_hook + + if $ctdb_test_restart_scheduled || ! cluster_is_healthy ; then + + restart_ctdb + else + # This could be made unconditional but then we might get + # duplication from the recovery in restart_ctdb. We want to + # leave the recovery in restart_ctdb so that future tests that + # might do a manual restart mid-test will benefit. + echo "Forcing a recovery..." + onnode 0 $CTDB recover + fi + + exit $status +} + +ctdb_test_exit_hook_add () +{ + ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" +} + +ctdb_test_init () +{ + scriptname=$(basename "$0") + testfailures=0 + ctdb_test_restart_scheduled=false + + trap "ctdb_test_exit" 0 +} + +######################################## + +# Sets: $out +try_command_on_node () +{ + local nodespec="$1" ; shift + + local verbose=false + local onnode_opts="" + + while [ "${nodespec#-}" != "$nodespec" ] ; do + if [ "$nodespec" = "-v" ] ; then + verbose=true + else + onnode_opts="$nodespec" + fi + nodespec="$1" ; shift + done + + local cmd="$*" + + out=$(onnode -q $onnode_opts "$nodespec" "$cmd" 2>&1) || { + + echo "Failed to execute \"$cmd\" on node(s) \"$nodespec\"" + echo "$out" + return 1 + } + + if $verbose ; then + echo "Output of \"$cmd\":" + echo "$out" + fi +} + +sanity_check_output () +{ + local min_lines="$1" + local regexp="$2" # Should be anchored as necessary. + local output="$3" + + local ret=0 + + local num_lines=$(echo "$output" | wc -l) + echo "There are $num_lines lines of output" + if [ $num_lines -lt $min_lines ] ; then + echo "BAD: that's less than the required number (${min_lines})" + ret=1 + fi + + local status=0 + local unexpected # local doesn't pass through status of command on RHS. + unexpected=$(echo "$output" | egrep -v "$regexp") || status=$? + + # Note that this is reversed. + if [ $status -eq 0 ] ; then + echo "BAD: unexpected lines in output:" + echo "$unexpected" | cat -A + ret=1 + else + echo "Output lines look OK" + fi + + return $ret +} + +sanity_check_ips () +{ + local ips="$1" # list of "ip node" lines + + echo "Sanity checking IPs..." + + local x ipp prev + prev="" + while read x ipp ; do + [ "$ipp" = "-1" ] && break + if [ -n "$prev" -a "$ipp" != "$prev" ] ; then + echo "OK" + return 0 + fi + prev="$ipp" + done <<<"$ips" + + echo "BAD: a node was -1 or IPs are only assigned to one node" + echo "Are you running an old version of CTDB?" + return 1 +} + +# This returns a list of "ip node" lines in $out +all_ips_on_node() +{ + local node=$@ + try_command_on_node $node "$CTDB ip -Y -n all | cut -d ':' -f1-3 | sed -e '1d' -e 's@^:@@' -e 's@:@ @g'" +} + +_select_test_node_and_ips () +{ + all_ips_on_node 0 + + test_node="" # this matches no PNN + test_node_ips="" + local ip pnn + while read ip pnn ; do + if [ -z "$test_node" -a "$pnn" != "-1" ] ; then + test_node="$pnn" + fi + if [ "$pnn" = "$test_node" ] ; then + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + echo "Selected node ${test_node} with IPs: ${test_node_ips}." + test_ip="${test_node_ips%% *}" + + [ -n "$test_node" ] || return 1 +} + +select_test_node_and_ips () +{ + local timeout=10 + while ! _select_test_node_and_ips ; do + echo "Unable to find a test node with IPs assigned" + if [ $timeout -le 0 ] ; then + echo "BAD: Too many attempts" + return 1 + fi + sleep_for 1 + timeout=$(($timeout - 1)) + done + + return 0 +} + +####################################### + +# Wait until either timeout expires or command succeeds. The command +# will be tried once per second. +wait_until () +{ + local timeout="$1" ; shift # "$@" is the command... + + local negate=false + if [ "$1" = "!" ] ; then + negate=true + shift + fi + + echo -n "<${timeout}|" + local t=$timeout + while [ $t -gt 0 ] ; do + local rc=0 + "$@" || rc=$? + if { ! $negate && [ $rc -eq 0 ] ; } || \ + { $negate && [ $rc -ne 0 ] ; } ; then + echo "|$(($timeout - $t))|" + echo "OK" + return 0 + fi + echo -n . + t=$(($t - 1)) + sleep 1 + done + + echo "*TIMEOUT*" + + return 1 +} + +sleep_for () +{ + echo -n "=${1}|" + for i in $(seq 1 $1) ; do + echo -n '.' + sleep 1 + done + echo '|' +} + +_cluster_is_healthy () +{ + $CTDB nodestatus all >/dev/null && \ + node_has_status 0 recovered +} + +cluster_is_healthy () +{ + if onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster is HEALTHY" + return 0 + else + echo "Cluster is UNHEALTHY" + if ! ${ctdb_test_restart_scheduled:-false} ; then + echo "DEBUG AT $(date '+%F %T'):" + local i + for i in "onnode -q 0 $CTDB status" "onnode -q 0 onnode all $CTDB scriptstatus" ; do + echo "$i" + $i || true + done + fi + return 1 + fi +} + +wait_until_healthy () +{ + local timeout="${1:-120}" + + echo "Waiting for cluster to become healthy..." + + wait_until 120 _cluster_is_healthy +} + +# This function is becoming nicely overloaded. Soon it will collapse! :-) +node_has_status () +{ + local pnn="$1" + local status="$2" + + local bits fpat mpat rpat + case "$status" in + (unhealthy) bits="?:?:?:1:*" ;; + (healthy) bits="?:?:?:0:*" ;; + (disconnected) bits="1:*" ;; + (connected) bits="0:*" ;; + (banned) bits="?:1:*" ;; + (unbanned) bits="?:0:*" ;; + (disabled) bits="?:?:1:*" ;; + (enabled) bits="?:?:0:*" ;; + (stopped) bits="?:?:?:?:1:*" ;; + (notstopped) bits="?:?:?:?:0:*" ;; + (frozen) fpat='^[[:space:]]+frozen[[:space:]]+1$' ;; + (unfrozen) fpat='^[[:space:]]+frozen[[:space:]]+0$' ;; + (monon) mpat='^Monitoring mode:ACTIVE \(0\)$' ;; + (monoff) mpat='^Monitoring mode:DISABLED \(1\)$' ;; + (recovered) rpat='^Recovery mode:NORMAL \(0\)$' ;; + *) + echo "node_has_status: unknown status \"$status\"" + return 1 + esac + + if [ -n "$bits" ] ; then + local out x line + + out=$($CTDB -Y status 2>&1) || return 1 + + { + read x + while read line ; do + # This needs to be done in 2 steps to avoid false matches. + local line_bits="${line#:${pnn}:*:}" + [ "$line_bits" = "$line" ] && continue + [ "${line_bits#${bits}}" != "$line_bits" ] && return 0 + done + return 1 + } <<<"$out" # Yay bash! + elif [ -n "$fpat" ] ; then + $CTDB statistics -n "$pnn" | egrep -q "$fpat" + elif [ -n "$mpat" ] ; then + $CTDB getmonmode -n "$pnn" | egrep -q "$mpat" + elif [ -n "$rpat" ] ; then + $CTDB status -n "$pnn" | egrep -q "$rpat" + else + echo 'node_has_status: unknown mode, neither $bits nor $fpat is set' + return 1 + fi +} + +wait_until_node_has_status () +{ + local pnn="$1" + local status="$2" + local timeout="${3:-30}" + local proxy_pnn="${4:-any}" + + echo "Waiting until node $pnn has status \"$status\"..." + + if ! wait_until $timeout onnode $proxy_pnn $CTDB_TEST_WRAPPER node_has_status "$pnn" "$status" ; then + for i in "onnode -q any $CTDB status" "onnode -q any onnode all $CTDB scriptstatus" ; do + echo "$i" + $i || true + done + + return 1 + fi + +} + +# Useful for superficially testing IP failover. +# IPs must be on nodes matching nodeglob. +# If the first argument is '!' then the IPs must not be on nodes +# matching nodeglob. +ips_are_on_nodeglob () +{ + local negating=false + if [ "$1" = "!" ] ; then + negating=true ; shift + fi + local nodeglob="$1" ; shift + local ips="$*" + + local out + + all_ips_on_node 1 + + for check in $ips ; do + while read ip pnn ; do + if [ "$check" = "$ip" ] ; then + case "$pnn" in + ($nodeglob) if $negating ; then return 1 ; fi ;; + (*) if ! $negating ; then return 1 ; fi ;; + esac + ips="${ips/${ip}}" # Remove from list + break + fi + # If we're negating and we didn't see the address then it + # isn't hosted by anyone! + if $negating ; then + ips="${ips/${check}}" + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + done + + ips="${ips// }" # Remove any spaces. + [ -z "$ips" ] +} + +wait_until_ips_are_on_nodeglob () +{ + echo "Waiting for IPs to fail over..." + + wait_until 60 ips_are_on_nodeglob "$@" +} + +node_has_some_ips () +{ + local node="$1" + + local out + + all_ips_on_node 1 + + while read ip pnn ; do + if [ "$node" = "$pnn" ] ; then + return 0 + fi + done <<<"$out" # bashism to avoid problem setting variable in pipeline. + + return 1 +} + +wait_until_node_has_some_ips () +{ + echo "Waiting for node to have some IPs..." + + wait_until 60 node_has_some_ips "$@" +} + +ip2ipmask () +{ + _ip="$1" + + ip addr show to "$_ip" | awk '$1 == "inet" { print $2 }' +} + +####################################### + +daemons_stop () +{ + echo "Attempting to politely shutdown daemons..." + onnode 1 $CTDB shutdown -n all || true + + echo "Sleeping for a while..." + sleep_for 1 + + local pat="ctdbd --socket=.* --nlist .* --nopublicipcheck" + if pgrep -f "$pat" >/dev/null ; then + echo "Killing remaining daemons..." + pkill -f "$pat" + + if pgrep -f "$pat" >/dev/null ; then + echo "Once more with feeling.." + pkill -9 -f "$pat" + fi + fi + + rm -rf "${TEST_VAR_DIR}/test.db" +} + +daemons_setup () +{ + mkdir -p "${TEST_VAR_DIR}/test.db/persistent" + + local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all" + local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt" + rm -f $CTDB_NODES $public_addresses_all $no_public_addresses + + # If there are (strictly) greater than 2 nodes then we'll randomly + # choose a node to have no public addresses. + local no_public_ips=-1 + [ $TEST_LOCAL_DAEMONS -gt 2 ] && no_public_ips=$(($RANDOM % $TEST_LOCAL_DAEMONS)) + echo "$no_public_ips" >$no_public_addresses + + # When running certain tests we add and remove eventscripts, so we + # need to be able to modify the events.d/ directory. Therefore, + # we use a temporary events.d/ directory under $TEST_VAR_DIR. We + # copy the actual test eventscript(s) in there from the original + # events.d/ directory that sits alongside $TEST_SCRIPT_DIR. + local top=$(dirname "$TEST_SCRIPTS_DIR") + local events_d="${top}/events.d" + mkdir -p "${TEST_VAR_DIR}/events.d" + cp -p "${events_d}/"* "${TEST_VAR_DIR}/events.d/" + + local i + for i in $(seq 1 $TEST_LOCAL_DAEMONS) ; do + if [ "${CTDB_USE_IPV6}x" != "x" ]; then + echo ::$i >>"$CTDB_NODES" + ip addr add ::$i/128 dev lo + else + echo 127.0.0.$i >>"$CTDB_NODES" + # 2 public addresses on most nodes, just to make things interesting. + if [ $(($i - 1)) -ne $no_public_ips ] ; then + echo "192.168.234.$i/24 lo" >>"$public_addresses_all" + echo "192.168.234.$(($i + $TEST_LOCAL_DAEMONS))/24 lo" >>"$public_addresses_all" + fi + fi + done +} + +daemons_start_1 () +{ + local pnn="$1" + shift # "$@" gets passed to ctdbd + + local public_addresses_all="${TEST_VAR_DIR}/public_addresses_all" + local public_addresses_mine="${TEST_VAR_DIR}/public_addresses.${pnn}" + local no_public_addresses="${TEST_VAR_DIR}/no_public_addresses.txt" + + local no_public_ips=-1 + [ -r $no_public_addresses ] && read no_public_ips <$no_public_addresses + + if [ "$no_public_ips" = $pnn ] ; then + echo "Node $no_public_ips will have no public IPs." + fi + + local node_ip=$(sed -n -e "$(($pnn + 1))p" "$CTDB_NODES") + local ctdb_options="--sloppy-start --reclock=${TEST_VAR_DIR}/rec.lock --nlist $CTDB_NODES --nopublicipcheck --listen=${node_ip} --event-script-dir=${TEST_VAR_DIR}/events.d --logfile=${TEST_VAR_DIR}/daemon.${pnn}.log -d 3 --log-ringbuf-size=10000 --dbdir=${TEST_VAR_DIR}/test.db --dbdir-persistent=${TEST_VAR_DIR}/test.db/persistent --dbdir-state=${TEST_VAR_DIR}/test.db/state" + + if [ $pnn -eq $no_public_ips ] ; then + ctdb_options="$ctdb_options --public-addresses=/dev/null" + else + cp "$public_addresses_all" "$public_addresses_mine" + ctdb_options="$ctdb_options --public-addresses=$public_addresses_mine" + fi + + # We'll use "pkill -f" to kill the daemons with + # "--socket=.* --nlist .* --nopublicipcheck" as context. + $VALGRIND ctdbd --socket="${TEST_VAR_DIR}/sock.$pnn" $ctdb_options "$@" ||return 1 +} + +daemons_start () +{ + # "$@" gets passed to ctdbd + + echo "Starting $TEST_LOCAL_DAEMONS ctdb daemons..." + + for i in $(seq 0 $(($TEST_LOCAL_DAEMONS - 1))) ; do + daemons_start_1 $i "$@" + done +} + +####################################### + +_ctdb_hack_options () +{ + local ctdb_options="$*" + + case "$ctdb_options" in + *--start-as-stopped*) + export CTDB_START_AS_STOPPED="yes" + esac +} + +_restart_ctdb () +{ + _ctdb_hack_options "$@" + + if [ -e /etc/redhat-release ] ; then + service ctdb restart + else + /etc/init.d/ctdb restart + fi +} + +_ctdb_start () +{ + _ctdb_hack_options "$@" + + /etc/init.d/ctdb start +} + +setup_ctdb () +{ + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_setup + fi +} + +# Common things to do after starting one or more nodes. +_ctdb_start_post () +{ + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || return 1 + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this blocks + # until the recovery is complete. Hopefully this will help the + # cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + + echo "ctdb is ready" +} + +# This assumes that ctdbd is not running on the given node. +ctdb_start_1 () +{ + local pnn="$1" + shift # "$@" is passed to ctdbd start. + + echo -n "Starting CTDB on node ${pnn}..." + + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_start_1 $pnn "$@" + else + onnode $pnn $CTDB_TEST_WRAPPER _ctdb_start "$@" + fi + + # If we're starting only 1 node then we're doing something weird. + ctdb_restart_when_done +} + +restart_ctdb () +{ + # "$@" is passed to ctdbd start. + + echo -n "Restarting CTDB" + if $ctdb_test_restart_scheduled ; then + echo -n " (scheduled)" + fi + echo "..." + + local i + for i in $(seq 1 5) ; do + if [ -n "$CTDB_NODES_SOCKETS" ] ; then + daemons_stop + daemons_start "$@" + else + onnode -p all $CTDB_TEST_WRAPPER _restart_ctdb "$@" + fi || { + echo "Restart failed. Trying again in a few seconds..." + sleep_for 5 + continue + } + + onnode -q 1 $CTDB_TEST_WRAPPER wait_until_healthy || { + echo "Cluster didn't become healthy. Restarting..." + continue + } + + echo "Setting RerecoveryTimeout to 1" + onnode -pq all "$CTDB setvar RerecoveryTimeout 1" + + # In recent versions of CTDB, forcing a recovery like this + # blocks until the recovery is complete. Hopefully this will + # help the cluster to stabilise before a subsequent test. + echo "Forcing a recovery..." + onnode -q 0 $CTDB recover + sleep_for 1 + + # Cluster is still healthy. Good, we're done! + if ! onnode 0 $CTDB_TEST_WRAPPER _cluster_is_healthy ; then + echo "Cluster became UNHEALTHY again [$(date)]" + onnode -p all ctdb status -Y 2>&1 + onnode -p all ctdb scriptstatus 2>&1 + echo "Restarting..." + continue + fi + + echo "Doing a sync..." + onnode -q 0 $CTDB sync + + echo "ctdb is ready" + return 0 + done + + echo "Cluster UNHEALTHY... too many attempts..." + onnode -p all ctdb status -Y 2>&1 + onnode -p all ctdb scriptstatus 2>&1 + + # Try to make the calling test fail + status=1 + return 1 +} + +ctdb_restart_when_done () +{ + ctdb_test_restart_scheduled=true +} + +get_ctdbd_command_line_option () +{ + local pnn="$1" + local option="$2" + + try_command_on_node "$pnn" "$CTDB getpid" || \ + die "Unable to get PID of ctdbd on node $pnn" + + local pid="${out#*:}" + try_command_on_node "$pnn" "ps -p $pid -o args hww" || \ + die "Unable to get command-line of PID $pid" + + # Strip everything up to and including --option + local t="${out#*--${option}}" + # Strip leading '=' or space if present + t="${t#=}" + t="${t# }" + # Strip any following options and print + echo "${t%% -*}" +} + +####################################### + +install_eventscript () +{ + local script_name="$1" + local script_contents="$2" + + if [ -z "$TEST_LOCAL_DAEMONS" ] ; then + # The quoting here is *very* fragile. However, we do + # experience the joy of installing a short script using + # onnode, and without needing to know the IP addresses of the + # nodes. + onnode all "f=\"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\" ; echo \"Installing \$f\" ; echo '${script_contents}' > \"\$f\" ; chmod 755 \"\$f\"" + else + f="${TEST_VAR_DIR}/events.d/${script_name}" + echo "$script_contents" >"$f" + chmod 755 "$f" + fi +} + +uninstall_eventscript () +{ + local script_name="$1" + + if [ -z "$TEST_LOCAL_DAEMONS" ] ; then + onnode all "rm -vf \"\${CTDB_BASE:-/etc/ctdb}/events.d/${script_name}\"" + else + rm -vf "${TEST_VAR_DIR}/events.d/${script_name}" + fi +} + +####################################### + +# This section deals with the 99.ctdb_test eventscript. + +# Metafunctions: Handle a ctdb-test file on a node. +# given event. +ctdb_test_eventscript_file_create () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn touch "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_remove () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn rm -f "/tmp/ctdb-test-${type}.${pnn}" +} + +ctdb_test_eventscript_file_exists () +{ + local pnn="$1" + local type="$2" + + try_command_on_node $pnn test -f "/tmp/ctdb-test-${type}.${pnn}" >/dev/null 2>&1 +} + + +# Handle a flag file on a node that is removed by 99.ctdb_test on the +# given event. +ctdb_test_eventscript_flag () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "flag-${event}" +} + + +# Handle a trigger that causes 99.ctdb_test to fail it's monitor +# event. +ctdb_test_eventscript_unhealthy_trigger () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-trigger" +} + +# Handle the file that 99.ctdb_test created to show that it has marked +# a node unhealthy because it detected the above trigger. +ctdb_test_eventscript_unhealthy_detected () +{ + local cmd="$1" + local pnn="$2" + + ctdb_test_eventscript_file_${cmd} "$pnn" "unhealthy-detected" +} + +# Handle a trigger that causes 99.ctdb_test to timeout it's monitor +# event. This should cause the node to be banned. +ctdb_test_eventscript_timeout_trigger () +{ + local cmd="$1" + local pnn="$2" + local event="$3" + + ctdb_test_eventscript_file_${cmd} "$pnn" "${event}-timeout" +} + +# Note that the eventscript can't use the above functions! +ctdb_test_eventscript_install () +{ + + local script='#!/bin/sh +out=$(ctdb pnn) +pnn="${out#PNN:}" + +rm -vf "/tmp/ctdb-test-flag-${1}.${pnn}" + +trigger="/tmp/ctdb-test-unhealthy-trigger.${pnn}" +detected="/tmp/ctdb-test-unhealthy-detected.${pnn}" +timeout_trigger="/tmp/ctdb-test-${1}-timeout.${pnn}" +case "$1" in + monitor) + if [ -e "$trigger" ] ; then + echo "${0}: Unhealthy because \"$trigger\" detected" + touch "$detected" + exit 1 + elif [ -e "$detected" -a ! -e "$trigger" ] ; then + echo "${0}: Healthy again, \"$trigger\" no longer detected" + rm "$detected" + fi + + ;; + *) + if [ -e "$timeout_trigger" ] ; then + echo "${0}: Sleeping for a long time because \"$timeout_trigger\" detected" + sleep 9999 + fi + ;; + *) + +esac + +exit 0 +' + install_eventscript "99.ctdb_test" "$script" +} + +ctdb_test_eventscript_uninstall () +{ + uninstall_eventscript "99.ctdb_test" +} + +# Note that this only works if you know all other monitor events will +# succeed. You also need to install the eventscript before using it. +wait_for_monitor_event () +{ + local pnn="$1" + + echo "Waiting for a monitor event on node ${pnn}..." + ctdb_test_eventscript_flag create $pnn "monitor" + + wait_until 120 ! ctdb_test_eventscript_flag exists $pnn "monitor" + +} + +####################################### + +nfs_test_setup () +{ + select_test_node_and_ips + + nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p') + + echo "Creating test subdirectory..." + try_command_on_node $test_node "mktemp -d --tmpdir=$nfs_first_export" + nfs_test_dir="$out" + try_command_on_node $test_node "chmod 777 $nfs_test_dir" + + nfs_mnt_d=$(mktemp -d) + nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE" + nfs_remote_file="${nfs_test_dir}/TEST_FILE" + + ctdb_test_exit_hook_add nfs_test_cleanup + + echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..." + mount -o timeo=1,hard,intr,vers=3 \ + ${test_ip}:${nfs_first_export} ${nfs_mnt_d} +} + +nfs_test_cleanup () +{ + rm -f "$nfs_local_file" + umount -f "$nfs_mnt_d" + rmdir "$nfs_mnt_d" + onnode -q $test_node rmdir "$nfs_test_dir" +} + + + +####################################### + +# Make sure that $CTDB is set. +: ${CTDB:=ctdb} + +local="${TEST_SUBDIR}/scripts/local.bash" +if [ -r "$local" ] ; then + . "$local" +fi diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests new file mode 100755 index 00000000000..171e8197f72 --- /dev/null +++ b/ctdb/tests/scripts/run_tests @@ -0,0 +1,273 @@ +#!/bin/bash + +usage() { + cat <<EOF +Usage: run_tests [OPTIONS] [TESTS] + +Options: + -s Print a summary of tests results after running all tests + -l Use local daemons for integration tests + -e Exit on the first test failure + -V <dir> Use <dir> as TEST_VAR_DIR + -C Clean up - kill daemons and remove TEST_VAR_DIR when done + -v Verbose - print test output for non-failures (only some tests) + -A Use "cat -A" to print test output (only some tests) + -D Show diff between failed/expected test output (some tests only) + -X Trace certain scripts run by tests using -x (only some tests) + -d Print descriptions of tests instead of filenames (dodgy!) + -H No headers - for running single test with other wrapper + -q Quiet - don't show tests being run (hint: use with -s) + -x Trace this script with the -x option +EOF + exit 1 +} + +# Print a message and exit. +die () +{ + echo "$1" >&2 ; exit ${2:-1} +} + +###################################################################### + +with_summary=false +with_desc=false +quiet=false +exit_on_fail=false +no_header=false + +export TEST_VERBOSE=false +export TEST_COMMAND_TRACE=false +export TEST_CAT_RESULTS_OPTS="" +export TEST_DIFF_RESULTS=false +export TEST_LOCAL_DAEMONS # No default, developer can "override"! +export TEST_VAR_DIR="" +export TEST_CLEANUP=false + +temp=$(getopt -n "$prog" -o "xdehlqsvV:XACDH" -l help -- "$@") + +[ $? != 0 ] && usage + +eval set -- "$temp" + +while true ; do + case "$1" in + -x) set -x; shift ;; + -d) with_desc=true ; shift ;; # 4th line of output is description + -e) exit_on_fail=true ; shift ;; + -l) TEST_LOCAL_DAEMONS="3" ; shift ;; + -q) quiet=true ; shift ;; + -s) with_summary=true ; shift ;; + -v) TEST_VERBOSE=true ; shift ;; + -V) TEST_VAR_DIR="$2" ; shift 2 ;; + -X) TEST_COMMAND_TRACE=true ; shift ;; + -A) TEST_CAT_RESULTS_OPTS="-A" ; shift ;; + -C) TEST_CLEANUP=true ; shift ;; + -D) TEST_DIFF_RESULTS=true ; shift ;; + -H) no_header=true ; shift ;; + --) shift ; break ;; + *) usage ;; + esac +done + +if $quiet ; then + show_progress() { cat >/dev/null ; } +else + show_progress() { cat ; } +fi + +###################################################################### + +ctdb_test_begin () +{ + local name="$1" + + teststarttime=$(date '+%s') + testduration=0 + + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" + echo "Running test $name ($(date '+%T'))" + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" +} + +ctdb_test_end () +{ + local name="$1" ; shift + local status="$1" ; shift + # "$@" is command-line + + local interp="SKIPPED" + local statstr=" (reason $*)" + if [ -n "$status" ] ; then + if [ $status -eq 0 ] ; then + interp="PASSED" + statstr="" + echo "ALL OK: $*" + else + interp="FAILED" + statstr=" (status $status)" + fi + fi + + testduration=$(($(date +%s)-$teststarttime)) + + echo "==========================================================================" + echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)" + echo "==========================================================================" + +} + +ctdb_test_run () +{ + local name="$1" ; shift + + [ -n "$1" ] || set -- "$name" + + $no_header || ctdb_test_begin "$name" + + local status=0 + "$@" || status=$? + + $no_header || ctdb_test_end "$name" "$status" "$*" + + return $status +} + +###################################################################### + +tests_total=0 +tests_passed=0 +tests_failed=0 +summary="" + +if ! which mktemp >/dev/null 2>&1 ; then + # Not perfect, but it will do... + mktemp () + { + _dir=false + if [ "$1" = "-d" ] ; then + _dir=true + fi + _t="${TMPDIR:-/tmp}/tmp.$$.$RANDOM" + ( + umask 077 + if $_dir ; then + mkdir "$_t" + else + >"$_t" + fi + ) + echo "$_t" + } +fi + +tf=$(mktemp) +sf=$(mktemp) + +set -o pipefail + +run_one_test () +{ + _f="$1" + + [ -x "$_f" ] || die "test \"$_f\" is not executable" + tests_total=$(($tests_total + 1)) + + ctdb_test_run "$_f" | tee "$tf" | show_progress + status=$? + if [ $status -eq 0 ] ; then + tests_passed=$(($tests_passed + 1)) + else + tests_failed=$(($tests_failed + 1)) + fi + if $with_summary ; then + if [ $status -eq 0 ] ; then + _t=" PASSED " + else + _t="*FAILED*" + fi + if $with_desc ; then + desc=$(tail -n +4 $tf | head -n 1) + _f="$desc" + fi + echo "$_t $_f" >>"$sf" + fi +} + +find_and_run_one_test () +{ + _t="$1" + _dir="$2" + + _f="${_dir}${_dir:+/}${_t}" + + if [ -d "$_f" ] ; then + for _i in $(ls "${_f%/}/"*".sh" 2>/dev/null) ; do + run_one_test "$_i" + if $exit_on_fail && [ $status -ne 0 ] ; then + break + fi + done + elif [ -f "$_f" ] ; then + run_one_test "$_f" + else + status=127 + fi +} + +[ -n "$TEST_VAR_DIR" ] || TEST_VAR_DIR=$(mktemp -d) +mkdir -p "$TEST_VAR_DIR" +# Must be absolute +TEST_VAR_DIR=$(cd "$TEST_VAR_DIR"; echo "$PWD") +echo "TEST_VAR_DIR=$TEST_VAR_DIR" + +export TEST_SCRIPTS_DIR=$(dirname "$0") + +for f ; do + find_and_run_one_test "$f" + + if [ $status -eq 127 ] ; then + # Find the the top-level tests directory + tests_dir=$(dirname $(cd $TEST_SCRIPTS_DIR; echo $PWD)) + # Strip off current directory from beginning, if there, just + # to make paths more friendly. + tests_dir=${tests_dir#$PWD/} + find_and_run_one_test "$f" "$tests_dir" + fi + + if [ $status -eq 127 ] ; then + die "test \"$f\" is not recognised" + fi + + if $exit_on_fail && [ $status -ne 0 ] ; then + break + fi +done + +rm -f "$tf" + +if $with_summary ; then + echo + cat "$sf" + echo + echo "${tests_passed}/${tests_total} tests passed" +fi + +rm -f "$sf" + +echo + +if $TEST_CLEANUP ; then + echo "Removing TEST_VAR_DIR=$TEST_VAR_DIR" + rm -rf "$TEST_VAR_DIR" +else + echo "Not cleaning up TEST_VAR_DIR=$TEST_VAR_DIR" +fi + +if $no_header || $exit_on_fail ; then + exit $status +elif [ $tests_failed -gt 0 ] ; then + exit 1 +else + exit 0 +fi diff --git a/ctdb/tests/scripts/test_wrap b/ctdb/tests/scripts/test_wrap new file mode 100755 index 00000000000..176310e9a15 --- /dev/null +++ b/ctdb/tests/scripts/test_wrap @@ -0,0 +1,21 @@ +#!/bin/bash + +# Execute the given command. The intention is that it is a function +# from "${TEST_SCRIPTS_DIR}/integration.bash". + +PATH="$(dirname $0):${PATH}" + +TEST_SCRIPTS_DIR=$(dirname $0) + +# We need the test binaries (i.e. tests/bin/) to be in $PATH. If they +# aren't already in $PATH then we know that tests/bin/ sits alongside +# tests/scripts/. +f="ctdb_bench" +if [ ! $(which $f >/dev/null 2>&1) ] ; then + d=$(dirname "$TEST_SCRIPTS_DIR")/bin + [ -x "$d/$f" ] && PATH="$d:$PATH" +fi + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +"$@" diff --git a/ctdb/tests/scripts/unit.sh b/ctdb/tests/scripts/unit.sh new file mode 100644 index 00000000000..c7c2b7a2736 --- /dev/null +++ b/ctdb/tests/scripts/unit.sh @@ -0,0 +1,141 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +. "${TEST_SCRIPTS_DIR}/common.sh" + +# Common variables and functions for CTDB unit tests. + +# Set the required result for a test. +# - Argument 1 is exit code. +# - Argument 2, if present is the required test output but "--" +# indicates empty output. +# If argument 2 is not present or null then read required test output +# from stdin. +required_result () +{ + required_rc="${1:-0}" + if [ -n "$2" ] ; then + if [ "$2" = "--" ] ; then + required_output="" + else + required_output="$2" + fi + else + if ! tty -s ; then + required_output=$(cat) + else + required_output="" + fi + fi +} + +ok () +{ + required_result 0 "$@" +} + +ok_null () +{ + ok -- +} + +result_print () +{ + _passed="$1" + _out="$2" + _rc="$3" + _extra_header="$4" + + if "$TEST_VERBOSE" || ! $_passed ; then + if [ -n "$_extra_header" ] ; then + cat <<EOF + +################################################## +$_extra_header +EOF + fi + +cat <<EOF +-------------------------------------------------- +Output (Exit status: ${_rc}): +-------------------------------------------------- +EOF + echo "$_out" | cat $TEST_CAT_RESULTS_OPTS + fi + + if ! $_passed ; then + cat <<EOF +-------------------------------------------------- +Required output (Exit status: ${required_rc}): +-------------------------------------------------- +EOF + echo "$required_output" | cat $TEST_CAT_RESULTS_OPTS + + if $TEST_DIFF_RESULTS ; then + _outr=$(mktemp) + echo "$required_output" >"$_outr" + + _outf=$(mktemp) + echo "$_fout" >"$_outf" + + cat <<EOF +-------------------------------------------------- +Diff: +-------------------------------------------------- +EOF + diff -u "$_outr" "$_outf" | cat -A + rm "$_outr" "$_outf" + fi + fi +} + +result_footer () +{ + _passed="$1" + _extra_footer="$2" + + if "$TEST_VERBOSE" || ! $_passed ; then + if [ -n "$_extra_footer" ] ; then + cat <<EOF +-------------------------------------------------- +$_extra_footer +-------------------------------------------------- +EOF + fi + fi + + if $_passed ; then + echo "PASSED" + return 0 + else + echo + echo "FAILED" + return 1 + fi +} + +result_check () +{ + _rc=$? + + _extra_header="$1" + + if [ -n "$OUT_FILTER" ] ; then + _fout=$(echo "$_out" | eval sed -r $OUT_FILTER) + else + _fout="$_out" + fi + + if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then + _passed=true + else + _passed=false + fi + + result_print "$_passed" "$_out" "$_rc" "$_extra_header" + result_footer "$_passed" +} + +local="${TEST_SUBDIR}/scripts/local.sh" +if [ -r "$local" ] ; then + . "$local" +fi diff --git a/ctdb/tests/simple/00_ctdb_init.sh b/ctdb/tests/simple/00_ctdb_init.sh new file mode 100755 index 00000000000..bd15fd7ed6e --- /dev/null +++ b/ctdb/tests/simple/00_ctdb_init.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Restart the ctdbd daemons of a CTDB cluster. + +No error if ctdbd is not already running on the cluster. + +Prerequisites: + +* Nodes must be accessible via 'onnode'. + +Steps: + +1. Restart the ctdb daemons on all nodes using a method according to + the test environment and platform. + +Expected results: + +* The cluster is healthy within a reasonable timeframe. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +setup_ctdb +restart_ctdb diff --git a/ctdb/tests/simple/00_ctdb_onnode.sh b/ctdb/tests/simple/00_ctdb_onnode.sh new file mode 100755 index 00000000000..3bc8f8b2fee --- /dev/null +++ b/ctdb/tests/simple/00_ctdb_onnode.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Use 'onnode' to confirm connectivity between all cluster nodes. + +Steps: + +1. Do a recursive "onnode all" to make sure all the nodes can connect + to each other. On a cluster this ensures that SSH keys are known + between all hosts, which will stop output being corrupted with + messages about nodes being added to the list of known hosts. + +Expected results: + +* 'onnode' works between all nodes. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + + +# + +echo "Checking connectivity between nodes..." +onnode all onnode -p all hostname + +# We're seeing some weirdness with CTDB controls timing out. We're +# wondering if time is jumping forward, so this creates a time log on +# each node that we can examine later if tests fail weirdly. +if [ -z "$TEST_LOCAL_DAEMONS" -a -n "$CTDB_TEST_TIME_LOGGING" ] ; then + echo "Starting time logging on each node..." + f="${TEST_VAR_DIR}/ctdb.test.time.log" + onnode -p all "[ -f $f ] || while : ; do date '+%s %N' ; sleep 1 ; done >$f 2>&1 </dev/null &" & +fi diff --git a/ctdb/tests/simple/01_ctdb_version.sh b/ctdb/tests/simple/01_ctdb_version.sh new file mode 100755 index 00000000000..3e1ed3e7646 --- /dev/null +++ b/ctdb/tests/simple/01_ctdb_version.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the output of the 'ctdb version' command. + +This test assumes an RPM-based installation and needs to be skipped on +non-RPM systems. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run the 'ctdb version' command on one of the cluster nodes. +3. Compare the version displayed with that listed by the rpm command + for the ctdb package. + +Expected results: + +* The 'ctdb version' command displays the ctdb version number. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +if ! try_command_on_node -v 0 "rpm -q ctdb" ; then + echo "No useful output from rpm, SKIPPING rest of test". + exit 0 +fi +rpm_ver="${out#ctdb-}" +# Some version of RPM append the architecture to the version. +# And also remove the release suffix. +arch=$(uname -m) +rpm_ver="${rpm_ver%-*.${arch}}" + +try_command_on_node -v 0 "$CTDB version" +ctdb_ver="${out#CTDB version: }" + +if [ "$ctdb_ver" = "$rpm_ver" ] ; then + echo "OK: CTDB version = RPM version" +else + echo "BAD: CTDB version != RPM version" + testfailures=1 +fi diff --git a/ctdb/tests/simple/02_ctdb_listvars.sh b/ctdb/tests/simple/02_ctdb_listvars.sh new file mode 100755 index 00000000000..2f709a8cc0d --- /dev/null +++ b/ctdb/tests/simple/02_ctdb_listvars.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb listvars' shows a list of all tunable variables. + +This test simply checks that at least 5 sane looking lines are +printed. It does not check that the list is complete or that the +values are sane. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb listvars' and verify that it shows a list of tunable + variables and their current values. + +Expected results: + +* 'ctdb listvars' works as expected. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node -v 0 "$CTDB listvars" + +sanity_check_output \ + 5 \ + '^[[:alpha:]][[:alnum:]]+[[:space:]]*=[[:space:]]*[[:digit:]]+$' \ + "$out" diff --git a/ctdb/tests/simple/03_ctdb_getvar.sh b/ctdb/tests/simple/03_ctdb_getvar.sh new file mode 100755 index 00000000000..a58aa3bf117 --- /dev/null +++ b/ctdb/tests/simple/03_ctdb_getvar.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb getvar' works correctly. + +Expands on the steps below as it actually checks the values of all +variables listed by 'ctdb listvars'. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb getvars <varname>' with a valid variable name (possibly + obtained via 'ctdb listvars'. +3. Verify that the command displays the correct value of the variable + (corroborate with the value shown by 'ctdb listvars'. + +Expected results: + +* 'ctdb getvar' shows the correct value of the variable. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node -v 0 "$CTDB listvars" + +echo "Veryifying all variable values using \"ctdb getvar\"..." + +echo "$out" | +while read var x val ; do + try_command_on_node 0 "$CTDB getvar $var" + + val2="${out#*= }" + + if [ "$val" != "$val2" ] ; then + echo "MISMATCH on $var: $val != $val2" + exit 1 + fi +done diff --git a/ctdb/tests/simple/04_ctdb_setvar.sh b/ctdb/tests/simple/04_ctdb_setvar.sh new file mode 100755 index 00000000000..5012e318cc8 --- /dev/null +++ b/ctdb/tests/simple/04_ctdb_setvar.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb setvar' works correctly. + +Doesn't strictly follow the procedure outlines below, since it doesn't +pick a variable from the output of 'ctdb listvars'. However, it +verifies the value with 'ctdb getvar' in addition to 'ctdb listvars'. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Get a list of all the ctdb tunable variables, using the 'ctdb + listvars' command. +3. Set the value of one of the variables using the 'setvar' control on + one of the nodes. E.g. 'ctdb setvar DeterministicIPs 0'. +4. Verify that the 'listvars' control now shows the new value for the + variable. + +Expected results: + +* After setting a value using 'ctdb setvar', 'ctdb listvars' shows the + modified value of the variable. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +var="RecoverTimeout" + +try_command_on_node -v 0 $CTDB getvar $var + +val="${out#*= }" + +echo "Going to try incrementing it..." + +incr=$(($val + 1)) + +try_command_on_node 0 $CTDB setvar $var $incr + +echo "That seemed to work, let's check the value..." + +try_command_on_node -v 0 $CTDB getvar $var + +newval="${out#*= }" + +if [ "$incr" != "$newval" ] ; then + echo "Nope, that didn't work..." + exit 1 +fi + +echo "Look's good! Now verifying with \"ctdb listvars\"" +try_command_on_node -v 0 "$CTDB listvars | grep '^$var'" + +check="${out#*= }" + +if [ "$incr" != "$check" ] ; then + echo "Nope, that didn't work..." + exit 1 +fi + +echo "Look's good! Putting the old value back..." +cmd="$CTDB setvar $var $val" +try_command_on_node 0 $cmd diff --git a/ctdb/tests/simple/05_ctdb_listnodes.sh b/ctdb/tests/simple/05_ctdb_listnodes.sh new file mode 100755 index 00000000000..a84e4af980f --- /dev/null +++ b/ctdb/tests/simple/05_ctdb_listnodes.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb listnodes' shows the list of nodes in a ctdb cluster. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb listnodes' on all the nodes of the cluster. +3. Verify that one all the nodes the command displays a list of + current cluster nodes. + +Expected results: + +* 'ctdb listnodes' displays the correct information. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node -v 0 "$CTDB listnodes" + +num_nodes=$(echo "$out" | wc -l) + +# Each line should look like an IP address. +sanity_check_output \ + 2 \ + '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$' \ + "$out" + +out_0="$out" + +echo "Checking other nodes..." + +n=1 +while [ $n -lt $num_nodes ] ; do + echo -n "Node ${n}: " + try_command_on_node $n "$CTDB listnodes" + if [ "$out_0" = "$out" ] ; then + echo "OK" + else + echo "DIFFERs from node 0:" + echo "$out" + testfailures=1 + fi + n=$(($n + 1)) +done diff --git a/ctdb/tests/simple/06_ctdb_getpid.sh b/ctdb/tests/simple/06_ctdb_getpid.sh new file mode 100755 index 00000000000..7152ad41710 --- /dev/null +++ b/ctdb/tests/simple/06_ctdb_getpid.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb getpid' works as expected. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb getpid -n <number>' on the nodes to check the PID of the + ctdbd process. +3. Verify that the output is valid. +4. Verify that with the '-n all' option the command shows the PIDs on + all the nodes + +Expected results: + +* 'ctdb getpid' shows valid output. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# This is an attempt at being independent of the number of nodes +# reported by "ctdb getpid -n all". +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" +echo "There are $num_nodes nodes..." + +# Call getpid a few different ways and make sure the answer is always the same. + +try_command_on_node -v 0 "onnode -q all $CTDB getpid" +pids_onnode="$out" + +try_command_on_node -v 0 "$CTDB getpid -n all" +pids_getpid_all="$out" + +cmd="" +n=0 +while [ $n -lt $num_nodes ] ; do + cmd="${cmd}${cmd:+; }$CTDB getpid -n $n" + n=$(($n + 1)) +done +try_command_on_node -v 0 "( $cmd )" +pids_getpid_n="$out" + +if [ "$pids_onnode" = "$pids_getpid_all" -a \ + "$pids_getpid_all" = "$pids_getpid_n" ] ; then + echo "They're the same... cool!" +else + echo "Error: they differ." + testfailures=1 +fi + +echo "Checking each PID for validity" + +n=0 +while [ $n -lt $num_nodes ] ; do + read line + pid=${line#Pid:} + try_command_on_node $n "ls -l /proc/${pid}/exe | sed -e 's@.*/@@'" + echo -n "Node ${n}, PID ${pid} looks to be running \"$out\" - " + if [ "$out" = "ctdbd" ] ; then + echo "GOOD!" + elif [ -n "$VALGRIND" -a "$out" = "memcheck" ] ; then + # We could check cmdline too if this isn't good enough. + echo "GOOD enough!" + else + echo "BAD!" + testfailures=1 + fi + n=$(($n + 1)) +done <<<"$pids_onnode" diff --git a/ctdb/tests/simple/07_ctdb_process_exists.sh b/ctdb/tests/simple/07_ctdb_process_exists.sh new file mode 100755 index 00000000000..83205aa07f6 --- /dev/null +++ b/ctdb/tests/simple/07_ctdb_process_exists.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb process-exists' shows correct information. + +The implementation is creative about how it gets PIDs for existing and +non-existing processes. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. On one of the cluster nodes, get the PID of an existing process + (using ps wax). +3. Run 'ctdb process-exists <pid>' on the node and verify that the + correct output is shown. +4. Run 'ctdb process-exists <pid>' with a pid of a non-existent + process and verify that the correct output is shown. + +Expected results: + +* 'ctdb process-exists' shows the correct output. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +test_node=1 + +# Create a background process on $test_node that will last for 60 seconds. +# It should still be there when we check. +try_command_on_node $test_node 'sleep 60 >/dev/null 2>&1 & echo $!' +pid="$out" + +echo "Checking for PID $pid on node $test_node" +# set -e is good, but avoid it here +status=0 +onnode 0 "$CTDB process-exists ${test_node}:${pid}" || status=$? +echo "$out" + +if [ $status -eq 0 ] ; then + echo "OK" +else + echo "BAD" + testfailures=1 +fi + +# Now just echo the PID of the shell from the onnode process on node +# 2. This PID will disappear and PIDs shouldn't roll around fast +# enough to trick the test... but there is a chance that will happen! +try_command_on_node $test_node 'echo $$' +pid="$out" + +echo "Checking for PID $pid on node $test_node" +try_command_on_node -v 0 "! $CTDB process-exists ${test_node}:${pid}" diff --git a/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh new file mode 100755 index 00000000000..138f59c6e79 --- /dev/null +++ b/ctdb/tests/simple/08_ctdb_isnotrecmaster.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the operation of 'ctdb isnotrecmaster'. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb isnotrecmaster' on each node. + +3. Verify that only 1 node shows the output 'This node is the + recmaster' and all the other nodes show the output 'This node is + not the recmaster'. + +Expected results: + +* 'ctdb isnotrecmaster' shows the correct output. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +cmd="$CTDB isnotrecmaster || true" +try_command_on_node all "$cmd" +echo "Output of \"$cmd\":" +echo "$out" + +num_all_lines=$(echo "$out" | wc -l) +num_rm_lines=$(echo "$out" | fgrep -c 'this node is the recmaster') || true +num_not_rm_lines=$(echo "$out" | fgrep -c 'this node is not the recmaster') || true + +if [ $num_rm_lines -eq 1 ] ; then + echo "OK, there is only 1 recmaster" +else + echo "BAD, there are ${num_rm_lines} nodes claiming to be the recmaster" + testfailures=1 +fi + +if [ $(($num_all_lines - $num_not_rm_lines)) -eq 1 ] ; then + echo "OK, all the other nodes claim not to be the recmaster" +else + echo "BAD, there are only ${num_not_rm_lines} nodes claiming not to be the recmaster" + testfailures=1 +fi diff --git a/ctdb/tests/simple/09_ctdb_ping.sh b/ctdb/tests/simple/09_ctdb_ping.sh new file mode 100755 index 00000000000..ab6ba146df6 --- /dev/null +++ b/ctdb/tests/simple/09_ctdb_ping.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the operation of the 'ctdb ping' command. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run the 'ctdb ping' command on one of the nodes and verify that it + shows valid and expected output. +3. Shutdown one of the cluster nodes, using the 'ctdb shutdown' + command. +4. Run the 'ctdb ping -n <node>' command from another node to this + node. +5. Verify that the command is not successful since th ctdb daemon is + not running on the node. + +Expected results: + +* The 'ctdb ping' command shows valid and expected output. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +try_command_on_node -v 0 "$CTDB ping -n 1" + +sanity_check_output \ + 1 \ + '^response from 1 time=-?[.0-9]+ sec[[:space:]]+\([[:digit:]]+ clients\)$' \ + "$out" + +try_command_on_node -v 0 "$CTDB shutdown -n 1" + +wait_until_node_has_status 1 disconnected 30 0 + +try_command_on_node -v 0 "! $CTDB ping -n 1" + +sanity_check_output \ + 1 \ + "(: ctdb_control error: ('ctdb_control to disconnected node'|'node is disconnected')|Unable to get ping response from node 1|Node 1 is DISCONNECTED|ctdb_control for getpnn failed|: Can not access node. Node is not operational\.|Node 1 has status DISCONNECTED\|UNHEALTHY\|INACTIVE$)" \ + "$out" diff --git a/ctdb/tests/simple/11_ctdb_ip.sh b/ctdb/tests/simple/11_ctdb_ip.sh new file mode 100755 index 00000000000..c1aec0e9bd7 --- /dev/null +++ b/ctdb/tests/simple/11_ctdb_ip.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb ip' shows the correct output. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb ip' on one of the nodes and verify the list of IP + addresses displayed (cross check the result with the output of + 'ip addr show' on the node). +3. Verify that colon-separated output is generated with the -Y option. + +Expected results: + +* 'ctdb ip' shows the list of public IPs being served by a node. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +echo "Getting list of public IPs..." +try_command_on_node -v 1 "$CTDB ip -n all | tail -n +2" +ips=$(echo "$out" | sed \ + -e 's@ node\[@ @' \ + -e 's@\].*$@@') +machineout=$(echo "$out" | sed -r \ + -e 's@^| |$@:@g' \ + -e 's@[[:alpha:]]+\[@@g' \ + -e 's@\]@@g') + +if [ -z "$TEST_LOCAL_DAEMONS" ]; then + while read ip pnn ; do + try_command_on_node $pnn "ip addr show" + if [ "${out/inet ${ip}\/}" != "$out" ] ; then + echo "GOOD: node $pnn appears to have $ip assigned" + else + echo "BAD: node $pnn does not appear to have $ip assigned" + testfailures=1 + fi + done <<<"$ips" # bashism to avoid problem setting variable in pipeline. +fi + +[ "$testfailures" != 1 ] && echo "Looks good!" + +cmd="$CTDB -Y ip -n all | tail -n +2" +echo "Checking that \"$cmd\" produces expected output..." + +try_command_on_node 1 "$cmd" +if [ "$out" = "$machineout" ] ; then + echo "Yep, looks good!" +else + echo "Nope, it looks like this:" + echo "$out" + echo "Should be like this:" + echo "$machineout" + testfailures=1 +fi diff --git a/ctdb/tests/simple/12_ctdb_getdebug.sh b/ctdb/tests/simple/12_ctdb_getdebug.sh new file mode 100755 index 00000000000..4a4926d0c60 --- /dev/null +++ b/ctdb/tests/simple/12_ctdb_getdebug.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb getdebug' works as expected. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Get the current debug level on a node, using 'ctdb getdebug -n <node>'. +3. Verify that colon-separated output is generated with the -Y option. +4. Verify that the '-n all' option shows the debug level on all nodes. + +Expected results: + +* 'ctdb getdebug' shows the debug level on all the nodes. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +try_command_on_node -v 1 "onnode -q all $CTDB getdebug" +getdebug_onnode="$out" + +sanity_check_output \ + $num_nodes \ + '^Node [[:digit:]]+ is at debug level [[:alpha:]]+ \([[:digit:]]+\)$' \ + "$out" + +try_command_on_node -v 1 "$CTDB getdebug -n all" +getdebug_all="$out" + +cmd="" +n=0 +while [ $n -lt $num_nodes ] ; do + cmd="${cmd}${cmd:+; }$CTDB getdebug -n $n" + n=$(($n + 1)) +done +try_command_on_node -v 1 "$cmd" +getdebug_n="$out" + +if [ "$getdebug_onnode" = "$getdebug_all" -a \ + "$getdebug_all" = "$getdebug_n" ] ; then + echo "They're the same... cool!" +else + echo "Error: they differ." + testfailures=1 +fi + +colons="" +nl=" +" +while read line ; do + t=$(echo "$line" | sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \((-?[[:digit:]]+)\)$@:\1:\2:@') + colons="${colons}${colons:+${nl}}:Name:Level:${nl}${t}" +done <<<"$getdebug_onnode" + +cmd="$CTDB -Y getdebug -n all" +echo "Checking that \"$cmd\" produces expected output..." + +try_command_on_node 1 "$cmd" +if [ "$out" = "$colons" ] ; then + echo "Yep, looks good!" +else + echo "Nope, it looks like this:" + echo "$out" + testfailures=1 +fi diff --git a/ctdb/tests/simple/13_ctdb_setdebug.sh b/ctdb/tests/simple/13_ctdb_setdebug.sh new file mode 100755 index 00000000000..d1d1f22a8d5 --- /dev/null +++ b/ctdb/tests/simple/13_ctdb_setdebug.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb setdebug' works as expected. + +This is a little superficial. It checks that CTDB thinks the debug +level has been changed but doesn't actually check that logging occurs +at the new level. + +A test should also be added to see if setting the debug value via a +numerical value works too. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Get the current debug level on a node, using 'ctdb getdebug'. +3. Change the debug level to some other value (e.g. EMERG) using + 'ctdb setdebug'. +4. Verify that the new debug level is correctly set using 'ctdb getdebug'. + +Expected results: + +* 'ctdb setdebug' correctly sets the debug level on a node. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +get_debug () +{ + # Sets; check_debug + local node="$1" + + local out + + try_command_on_node -v $node "$CTDB getdebug" + check_debug=$(echo "$out" | + sed -r -e 's@Node [[:digit:]]+ is at debug level ([[:alpha:]]+) \(-?[[:digit:]]+\)$@\1@') +} + +set_and_check_debug () +{ + local node="$1" + local level="$2" + + echo "Setting debug level on node ${node} to ${level}." + try_command_on_node $node "$CTDB setdebug ${level}" + + local check_debug + get_debug $node + + if [ "$level" = "$check_debug" ] ; then + echo "That seemed to work... cool!" + else + echo "BAD: Debug level should have changed to \"$level\" but it is \"$check_debug\"." + testfailures=1 + fi +} + +get_debug 1 +initial_debug="$check_debug" + +new_debug="EMERG" +[ "$initial_debug" = "$new_debug" ] && new_debug="ALERT" + +set_and_check_debug 1 "$new_debug" + +if [ "$testfailures" != 1 ] ; then + echo "Returning the debug level to its initial value..." + set_and_check_debug 1 "$initial_debug" +fi diff --git a/ctdb/tests/simple/14_ctdb_statistics.sh b/ctdb/tests/simple/14_ctdb_statistics.sh new file mode 100755 index 00000000000..9cc5ac15cc9 --- /dev/null +++ b/ctdb/tests/simple/14_ctdb_statistics.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb statistics' works as expected. + +This is pretty superficial and could do more validation. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb statistics' on a node, and verify that the output is + valid. +3. Repeat the command with the '-n all' option and verify that the + output is valid. + +Expected results: + +* 'ctdb statistics' shows valid output on all the nodes. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +pattern='^(CTDB version 1|Current time of statistics[[:space:]]*:.*|Statistics collected since[[:space:]]*:.*|Gathered statistics for [[:digit:]]+ nodes|[[:space:]]+[[:alpha:]_]+[[:space:]]+[[:digit:]]+|[[:space:]]+(node|client|timeouts|locks)|[[:space:]]+([[:alpha:]_]+_latency|max_reclock_[[:alpha:]]+)[[:space:]]+[[:digit:]-]+\.[[:digit:]]+[[:space:]]sec|[[:space:]]*(locks_latency|reclock_ctdbd|reclock_recd|call_latency|lockwait_latency|childwrite_latency)[[:space:]]+MIN/AVG/MAX[[:space:]]+[-.[:digit:]]+/[-.[:digit:]]+/[-.[:digit:]]+ sec out of [[:digit:]]+|[[:space:]]+(hop_count_buckets|lock_buckets):[[:space:][:digit:]]+)$' + +try_command_on_node -v 1 "$CTDB statistics" + +sanity_check_output 40 "$pattern" "$out" + +try_command_on_node -v 1 "$CTDB statistics -n all" + +sanity_check_output 40 "$pattern" "$out" diff --git a/ctdb/tests/simple/15_ctdb_statisticsreset.sh b/ctdb/tests/simple/15_ctdb_statisticsreset.sh new file mode 100755 index 00000000000..eaa60d69205 --- /dev/null +++ b/ctdb/tests/simple/15_ctdb_statisticsreset.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb statisticsreset' works as expected. + +This is pretty superficial. It just checks that a few particular +items reduce. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb statisticsreset' on all nodes and verify that it executes + successfully. + +Expected results: + +* 'ctdb statisticsreset' executes successfully. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" + +get_stat () +{ + local label="$1" + local out="$2" + + echo "$out" | sed -rn -e "s@^[[:space:]]+${label}[[:space:]]+([[:digit:]])@\1@p" | head -1 +} + +check_reduced () +{ + local label="$1" + local before="$2" + local after="$3" + + if [ $after -lt $before ] ; then + echo "GOOD: ${label} reduced from ${before} to ${after}" + else + echo "BAD: ${label} did not reduce from ${before} to ${after}" + testfailures=1 + fi +} + +n=0 +while [ $n -lt $num_nodes ] ; do + echo "Getting initial statistics for node ${n}..." + + try_command_on_node -v $n $CTDB statistics + + before_req_control=$(get_stat "req_control" "$out") + before_reply_control=$(get_stat "reply_control" "$out") + before_node_packets_recv=$(get_stat "node_packets_recv" "$out") + + try_command_on_node $n $CTDB statisticsreset + + try_command_on_node -v $n $CTDB statistics + + after_req_control=$(get_stat "req_control" "$out") + after_reply_control=$(get_stat "reply_control" "$out") + after_node_packets_recv=$(get_stat "node_packets_recv" "$out") + + check_reduced "req_control" "$before_req_control" "$after_req_control" + check_reduced "reply_control" "$before_reply_control" "$after_reply_control" + check_reduced "node_packets_recv" "$before_node_packets_recv" "$after_node_packets_recv" + + n=$(($n + 1)) +done diff --git a/ctdb/tests/simple/16_ctdb_config_add_ip.sh b/ctdb/tests/simple/16_ctdb_config_add_ip.sh new file mode 100755 index 00000000000..b770bd6a6a4 --- /dev/null +++ b/ctdb/tests/simple/16_ctdb_config_add_ip.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that an IP address can be added to a node using 'ctdb addip'. + +This test goes to some trouble to figure out which IP address to add +but assumes a 24-bit subnet mask. It does not handle IPv6. It does +not do any network level checks that the new IP address is reachable +but simply trusts 'ctdb ip' that the address has been added. There is +also an extra prerequisite that the node being added to already has +public addresses - this is difficult to avoid if the extra address is +to be sensibly chosen. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Use 'ctdb ip' on one of the nodes to list the IP addresses being + served. +3. Add an additional public address to be served by the node, using + 'ctdb addip'. +4. Verify that this IP address has been added to the list of IP + addresses being served by the node, using the 'ctdb ip' command. + +Expected results: + +* 'ctdb ip' adds an IP address to the list of public IP addresses + being served by a node. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +echo "Getting list of public IPs..." +all_ips_on_node 0 + +# When selecting test_node we just want a node that has public IPs. +# This will work and is economically semi-randomly. :-) +read x test_node <<<"$out" + +test_node_ips="" +all_ips="" +while read ip pnn ; do + all_ips="${all_ips}${all_ips:+ }${ip}" + [ "$pnn" = "$test_node" ] && \ + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" +done <<<"$out" + +echo "Selected node ${test_node} with IPs: $test_node_ips" + +# Try to find a free IP adddress. This is inefficient but should +# succeed quickly. +try_command_on_node $test_node "ip addr show" +all_test_node_ips=$(echo "$out" | sed -rn -e 's@^[[:space:]]+inet[[:space:]]+([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+/[[:digit:]]+).*[[:space:]]([^[:space:]]+)+$@\1:\2@p') + +add_ip="" + +# Use an IP already on one of the nodes, remove the last octet and +# loop through the possible IP addreses. +for i in $test_node_ips ; do + prefix="${i%.*}" + for j in $(seq 101 199) ; do + try="${prefix}.${j}" + # Try to make sure it isn't used anywhere! + + # First, make sure it isn't an existing public address on the + # cluster. + for k in $all_ips ; do + [ "$try" = "$k" ] && continue 2 + done + + # Also make sure it isn't some other address in use on the + # node. + for k in $all_test_node_ips ; do + [ "$try" = "${k%/*}" ] && continue 2 + done + + # Get the interface details for $i, which our address is a + # close relative of. This should never fail but it can't hurt + # to be careful... + try_command_on_node $test_node "ctdb ip -v -Y" + while IFS=":" read x ip pnn iface x ; do + if [ "$i" = "$ip" ]; then + add_ip="$try/32:$iface" + break 3 + fi + done <<<"$out" + done +done + +if [ -z "$add_ip" ] ; then + echo "BAD: Unable to find IP address to add." + exit 1 +fi + +echo "Adding IP: ${add_ip/:/ on interface }" +try_command_on_node $test_node $CTDB addip ${add_ip/:/ } + +echo "Waiting for IP to be added..." +if wait_until 60 ips_are_on_nodeglob $test_node ${add_ip%/*} ; then + echo "That worked!" +else + echo "BAD: IP didn't get added." + try_command_on_node $test_node $CTDB ip -n all + exit 1 +fi diff --git a/ctdb/tests/simple/17_ctdb_config_delete_ip.sh b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh new file mode 100755 index 00000000000..1ad9f334365 --- /dev/null +++ b/ctdb/tests/simple/17_ctdb_config_delete_ip.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that a node's public IP address can be deleted using 'ctdb deleteip'. + +This test does not do any network level checks that the IP address is +no longer reachable but simply trusts 'ctdb ip' that the address has +been deleted. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Use 'ctdb ip' on one of the nodes to list the IP addresses being + served. +3. Delete one public IP address being be served by the node, using + 'ctdb delip'. +4. Verify that the delete IP address is no longer listed using the + all_ips_on_node helper function. + +Expected results: + +* 'ctdb delip' removes an IP address from the list of public IP + addresses being served by a node. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +echo "Getting list of public IPs..." +all_ips_on_node -v 0 + +# Select an IP/node to remove. +num_ips=$(echo "$out" | wc -l) +num_to_remove=$(($RANDOM % $num_ips)) + +# Find the details in the list. +i=0 +while [ $i -le $num_to_remove ] ; do + read ip_to_remove test_node + i=$(($i + 1)) +done <<<"$out" + +echo "Attempting to remove ${ip_to_remove} from node ${test_node}." +try_command_on_node $test_node $CTDB delip $ip_to_remove + +echo "Sleeping..." +sleep_for 1 + +test_node_ips="" +while read ip pnn ; do + [ "$pnn" = "$test_node" ] && \ + test_node_ips="${test_node_ips}${test_node_ips:+ }${ip}" +done <<<"$out" # bashism to avoid problem setting variable in pipeline. + +if [ "${test_node_ips/${ip_to_remove}}" = "$test_node_ips" ] ; then + echo "GOOD: That worked!" +else + echo "BAD: The remove IP address is still there!" + testfailures=1 +fi diff --git a/ctdb/tests/simple/18_ctdb_reloadips.sh b/ctdb/tests/simple/18_ctdb_reloadips.sh new file mode 100755 index 00000000000..760e4766a7b --- /dev/null +++ b/ctdb/tests/simple/18_ctdb_reloadips.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that IPs can be rearrranged using 'ctdb reloadips'. + +Various sub-tests that remove addresses from the public_addresses file +on a node or delete the entire contents of the public_addresses file. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Expected results: + +* When addresses are deconfigured "ctdb ip" no longer reports them and + when added they are seen again. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Emptying public addresses file on $test_node" + +addresses=$(get_ctdbd_command_line_option $test_node "public-addresses") +echo "Public addresses file on node $test_node is \"$addresses\"" +backup="${addresses}.$$" + +restore_public_addresses () +{ + try_command_on_node $test_node "mv $backup $addresses >/dev/null 2>&1 || true" +} +ctdb_test_exit_hook_add restore_public_addresses + +try_command_on_node $test_node "mv $addresses $backup && touch $addresses" + +try_command_on_node any $CTDB reloadips all + +echo "Getting list of public IPs on node $test_node" +try_command_on_node $test_node "$CTDB ip | tail -n +2" + +if [ -n "$out" ] ; then + cat <<EOF +BAD: node $test_node still has ips: +$out +EOF + exit 1 +fi + +echo "GOOD: no IPs left on node $test_node" + +echo "Restoring addresses" +restore_public_addresses + +try_command_on_node any $CTDB reloadips all + +echo "Getting list of public IPs on node $test_node" +try_command_on_node $test_node "$CTDB ip | tail -n +2" + +if [ -z "$out" ] ; then + echo "BAD: node $test_node has no ips" + exit 1 +fi + +cat <<EOF +GOOD: node $test_node has these addresses: +$out +EOF + +try_command_on_node any $CTDB sync + +select_test_node_and_ips + +echo "Removing IP $test_ip from node $test_node" + +try_command_on_node $test_node "mv $addresses $backup && grep -v '^${test_ip}/' $backup >$addresses" + +try_command_on_node any $CTDB reloadips all + +try_command_on_node $test_node $CTDB ip + +if grep "^${test_ip} " <<<"$out" ; then + cat <<EOF +BAD: node $test_node can still host IP $test_ip: +$out +EOF + exit 1 +fi + +cat <<EOF +GOOD: node $test_node is no longer hosting IP $test_ip: +$out +EOF diff --git a/ctdb/tests/simple/20_delip_iface_gc.sh b/ctdb/tests/simple/20_delip_iface_gc.sh new file mode 100755 index 00000000000..bc43567a57b --- /dev/null +++ b/ctdb/tests/simple/20_delip_iface_gc.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that an interface is deleted when all IPs on it are deleted. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +echo "Getting public IPs information..." +try_command_on_node -v any "$CTDB ip -v -n all -Y | tail -n +2" +ip_info="$out" + +# Select the first node and find out its interfaces +test_node=$(awk -F: 'NR == 1 { print $3}' <<<"$ip_info") +ifaces=$(awk -F: -v tn=$test_node '$3 == tn { print $6 }' <<<"$ip_info" | sed 's@, @ @g' | xargs -n 1 | sort -u) +echo "Selected test node ${test_node} with interfaces: ${ifaces}" + +# Delete all IPs on each interface... deleting IPs from one interface +# can cause other interfaces to disappear, so we need to be careful... +for i in $ifaces ; do + try_command_on_node $test_node "$CTDB ifaces -Y" + info=$(awk -F: -v iface="$i" '$2 == iface { print $0 }' <<<"$out") + + if [ -z "$info" ] ; then + echo "Interface ${i} missing... assuming already deleted!" + continue + fi + + echo "Deleting IPs on interface ${i}, with this information:" + echo " $info" + + try_command_on_node $test_node "$CTDB ip -v -Y | tail -n +2" + awk -F: -v i="$i" \ + '$6 == i { print $2 }' <<<"$out" | + while read ip ; do + echo " $ip" + try_command_on_node $test_node "$CTDB delip $ip" + done + + try_command_on_node $test_node "$CTDB ifaces -Y" + info=$(awk -F: -v iface="$i" '$2 == iface { print $0 }' <<<"$out") + + if [ -z "$info" ] ; then + echo "GOOD: Interface ${i} has been garbage collected" + else + echo "BAD: Interface ${i} still exists" + echo "$out" + exit 1 + fi +done diff --git a/ctdb/tests/simple/23_ctdb_moveip.sh b/ctdb/tests/simple/23_ctdb_moveip.sh new file mode 100755 index 00000000000..7c09e585030 --- /dev/null +++ b/ctdb/tests/simple/23_ctdb_moveip.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb moveip' allows movement of public IPs between cluster nodes. + +To work, this test unsets DeterministicIPs and sets NoIPFailback. + +This test does not do any network level checks that the IP address is +no longer reachable but simply trusts 'ctdb ip' that the address has +been deleted. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Use 'ctdb ip' on one of the nodes to list the IP addresses being + served. +3. Use 'ctdb moveip' to move an address from one node to another. +4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node. + +Expected results: + +* 'ctdb moveip' allows an IP address to be moved between cluster nodes. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" +echo "There are $num_nodes nodes..." + +if [ $num_nodes -lt 2 ] ; then + echo "Less than 2 nodes!" + exit 1 +fi + +echo "Getting list of public IPs..." +all_ips_on_node -v 0 + +sanity_check_ips "$out" + +# Select an IP/node to move. +num_ips=$(echo "$out" | wc -l) +num_to_move=$(($RANDOM % $num_ips)) + +# Find the details in the list. +i=0 +while [ $i -le $num_to_move ] ; do + read ip_to_move test_node + i=$(($i + 1)) +done <<<"$out" + +# Can only move address to a node that is willing to host $ip_to_move. +# This inefficient but shouldn't take long or get stuck. +to_node=$test_node +while [ $test_node -eq $to_node ] ; do + n=$(($RANDOM % $num_ips)) + i=0 + while [ $i -le $n ] ; do + read x to_node + i=$(($i + 1)) + done <<<"$out" +done + +echo "Turning off DeterministicIPs..." +try_command_on_node 0 $CTDB setvar DeterministicIPs 0 -n all + +echo "Turning on NoIPFailback..." +try_command_on_node 0 $CTDB setvar NoIPFailback 1 -n all + +echo "Attempting to move ${ip_to_move} from node ${test_node} to node ${to_node}." +try_command_on_node $test_node $CTDB moveip $ip_to_move $to_node + +if wait_until_ips_are_on_nodeglob "[!${test_node}]" $ip_to_move ; then + echo "IP moved from ${test_node}." +else + echo "BAD: IP didn't move from ${test_node}." + exit 1 +fi + +if wait_until_ips_are_on_nodeglob "$to_node" $ip_to_move ; then + echo "IP moved to ${to_node}." +else + echo "BAD: IP didn't move to ${to_node}." + exit 1 +fi diff --git a/ctdb/tests/simple/24_ctdb_getdbmap.sh b/ctdb/tests/simple/24_ctdb_getdbmap.sh new file mode 100755 index 00000000000..9bed5903914 --- /dev/null +++ b/ctdb/tests/simple/24_ctdb_getdbmap.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb getdbmap' operates as expected. + +This test creates some test databases using 'ctdb attach'. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Get the database on using 'ctdb getdbmap'. +3. Verify that the output is valid. + +Expected results: + +* 'ctdb getdbmap' shows a valid listing of databases. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +make_temp_db_filename () +{ + dd if=/dev/urandom count=1 bs=512 2>/dev/null | + md5sum | + awk '{printf "%s.tdb\n", $1}' +} + +try_command_on_node -v 0 "$CTDB getdbmap" + +db_map_pattern='^(Number of databases:[[:digit:]]+|dbid:0x[[:xdigit:]]+ name:[^[:space:]]+ path:[^[:space:]]+)$' + +sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out" + +num_db_init=$(echo "$out" | sed -n -e '1s/.*://p') + +for i in $(seq 1 5) ; do + f=$(make_temp_db_filename) + echo "Creating test database: $f" + try_command_on_node 0 $CTDB attach "$f" + try_command_on_node 0 $CTDB getdbmap + sanity_check_output $(($num_db_init + 1)) "$dbmap_pattern" "$out" + num=$(echo "$out" | sed -n -e '1s/^.*://p') + if [ $num = $(($num_db_init + $i)) ] ; then + echo "OK: correct number of additional databases" + else + echo "BAD: no additional database" + exit 1 + fi + if [ "${out/name:${f} /}" != "$out" ] ; then + echo "OK: getdbmap knows about \"$f\"" + else + echo "BAD: getdbmap does not know about \"$f\"" + exit 1 + fi +done diff --git a/ctdb/tests/simple/25_dumpmemory.sh b/ctdb/tests/simple/25_dumpmemory.sh new file mode 100755 index 00000000000..4082da1d130 --- /dev/null +++ b/ctdb/tests/simple/25_dumpmemory.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb dumpmemory' shows expected output. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run 'ctdb dumpmemory' and verify that it shows expected output +3. Verify that the command takes the '-n all' option and that it + causes output for all nodes to be displayed. + +Expected results: + +* 'ctdb dumpmemory' sows valid output. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node -v 0 "$CTDB dumpmemory" + +pat='^([[:space:]].+[[:space:]]+contains[[:space:]]+[[:digit:]]+ bytes in[[:space:]]+[[:digit:]]+ blocks \(ref [[:digit:]]+\)[[:space:]]+0x[[:xdigit:]]+|[[:space:]]+reference to: .+|full talloc report on .+ \(total[[:space:]]+[[:digit:]]+ bytes in [[:digit:]]+ blocks\))$' + +sanity_check_output 10 "$pat" "$out" + +echo "Checking output using '-n all'..." + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +try_command_on_node 0 "$CTDB dumpmemory" -n all +sanity_check_output 10 "$pat" "$out" + +if [ $(fgrep -c 'full talloc report on' <<<"$out") -eq $num_nodes ] ; then + echo "OK: there looks to be output for all $num_nodes nodes" +else + echo "BAD: there not look to be output for all $num_nodes nodes" + exit 1 +fi diff --git a/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh new file mode 100755 index 00000000000..6642b1759f9 --- /dev/null +++ b/ctdb/tests/simple/26_ctdb_config_check_error_on_unreachable_ctdb.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify an error occurs if a ctdb command is run against a node without a ctdbd. + +That is, check that an error message is printed if an attempt is made +to execute a ctdb command against a node that is not running ctdbd. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Shutdown ctdb on a node using 'ctdb shutdown -n <node>'. +3. Verify that the status of the node changes to 'DISCONNECTED'. +4. Now run 'ctdb ip -n <node>' from another node. +5. Verify that an error message is printed stating that the node is + disconnected. +6. Execute some other commands against the shutdown node. For example, + disable, enable, ban, unban, listvars. +7. For each command, verify that an error message is printed stating + that the node is disconnected. + +Expected results: + +* For a node on which ctdb is not running, all commands display an + error message stating that the node is disconnected. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +test_node=1 + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) +echo "There are $num_nodes nodes." + +echo "Shutting down node ${test_node}..." +try_command_on_node $test_node $CTDB shutdown + +wait_until_node_has_status $test_node disconnected 30 0 + +wait_until_node_has_status 0 recovered 30 0 + +pat="ctdb_control error: 'ctdb_control to disconnected node'|ctdb_control error: 'node is disconnected'|Node $test_node is DISCONNECTED|Node $test_node has status DISCONNECTED\|UNHEALTHY\|INACTIVE" + +for i in ip disable enable "ban 0" unban listvars ; do + try_command_on_node -v 0 ! $CTDB $i -n $test_node + + if egrep -q "$pat" <<<"$out" ; then + echo "OK: \"ctdb ${i}\" fails with expected \"disconnected node\" message" + else + echo "BAD: \"ctdb ${i}\" does not fail with expected \"disconnected node\" message" + exit 1 + fi +done diff --git a/ctdb/tests/simple/31_ctdb_disable.sh b/ctdb/tests/simple/31_ctdb_disable.sh new file mode 100755 index 00000000000..d0214549a48 --- /dev/null +++ b/ctdb/tests/simple/31_ctdb_disable.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the operation of 'ctdb disable'. + +This is a superficial test of the 'ctdb disable' command. It trusts +information from CTDB that indicates that the IP failover has happened +correctly. Another test should check that the failover has actually +happened at the networking level. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Disable one of the nodes using 'ctdb disable -n <node>'. +3. Verify that the status of the node changes to 'disabled'. +4. Verify that the IP addreses served by the disabled node are failed + over to other nodes. + +Expected results: + +* The status of the disabled node changes as expected and IP addresses + failover as expected. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Disabling node $test_node" + +try_command_on_node 1 $CTDB disable -n $test_node + +# Avoid a potential race condition... +wait_until_node_has_status $test_node disabled + +if wait_until_ips_are_on_nodeglob "[!${test_node}]" $test_node_ips ; then + echo "All IPs moved." +else + echo "Some IPs didn't move." + testfailures=1 +fi diff --git a/ctdb/tests/simple/32_ctdb_enable.sh b/ctdb/tests/simple/32_ctdb_enable.sh new file mode 100755 index 00000000000..7cc3da31291 --- /dev/null +++ b/ctdb/tests/simple/32_ctdb_enable.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the operation of 'ctdb enable'. + +This is a superficial test of the 'ctdb enable' command. It trusts +information from CTDB that indicates that the IP failover has happened +correctly. Another test should check that the failover has actually +happened at the networking level. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Disable one of the nodes using 'ctdb disable -n <node>'. +3. Verify that the status of the node changes to 'disabled'. +4. Verify that the public IP addreses served by the disabled node are + failed over to other nodes. +5. Enable the disabled node using 'ctdb enable -n '<node>'. +6. Verify that the status changes back to 'OK'. +7. Verify that some public IP addreses are failed back to the node. + + +Expected results: + +* The status of a re-enabled node changes as expected and IP addresses + fail back as expected. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +######################################## + +set -e + +cluster_is_healthy + +select_test_node_and_ips + +echo "Disabling node $test_node" +try_command_on_node 1 $CTDB disable -n $test_node + +wait_until_node_has_status $test_node disabled + +if wait_until_ips_are_on_nodeglob "[!${test_node}]" $test_node_ips ; then + echo "All IPs moved." +else + echo "Some IPs didn't move." + testfailures=1 +fi + +echo "Reenabling node $test_node" +try_command_on_node 1 $CTDB enable -n $test_node + +wait_until_node_has_status $test_node enabled + +wait_until_node_has_some_ips "$test_node" diff --git a/ctdb/tests/simple/41_ctdb_stop.sh b/ctdb/tests/simple/41_ctdb_stop.sh new file mode 100755 index 00000000000..1a45d8f0c94 --- /dev/null +++ b/ctdb/tests/simple/41_ctdb_stop.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the operation of the 'ctdb stop' command. + +This is a superficial test of the 'ctdb stop' command. It trusts +information from CTDB that indicates that the IP failover has +happened correctly. Another test should check that the failover +has actually happened at the networking level. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Stop one of the nodes using the 'ctdb stop' command. +3. Verify that the status of the node changes to 'stopped'. +4. Verify that the public IP addresses that were being served by + the node are failed over to one of the other nodes. + +Expected results: + +* The status of the stopped nodes changes as expected and IP addresses + failover as expected. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Stopping node ${test_node}..." +try_command_on_node 1 $CTDB stop -n $test_node + +wait_until_node_has_status $test_node stopped + +if wait_until_ips_are_on_nodeglob "[!${test_node}]" $test_node_ips ; then + echo "All IPs moved." +else + echo "Some IPs didn't move." + testfailures=1 +fi diff --git a/ctdb/tests/simple/42_ctdb_continue.sh b/ctdb/tests/simple/42_ctdb_continue.sh new file mode 100755 index 00000000000..381baf55f06 --- /dev/null +++ b/ctdb/tests/simple/42_ctdb_continue.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify the operation of the 'ctdb continue' command. + +This is a superficial test of the 'ctdb continue' command. It trusts +information from CTDB that indicates that the IP failover and failback +has happened correctly. Another test should check that the failover +and failback has actually happened at the networking level. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Stop one of the nodes using the 'ctdb stop' command. +3. Verify that the status of the node changes to 'stopped'. +4. Verify that the public IP addresses that were being served by + the node are failed over to one of the other nodes. +5. Use 'ctdb continue' to bring the node back online. +6. Verify that the status of the node changes back to 'OK' and that + some public IP addresses move back to the node. + +Expected results: + +* The 'ctdb continue' command successfully brings a stopped node online. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Stopping node ${test_node}..." +try_command_on_node 1 $CTDB stop -n $test_node + +wait_until_node_has_status $test_node stopped + +if wait_until_ips_are_on_nodeglob "[!${test_node}]" $test_node_ips ; then + echo "All IPs moved." +else + echo "Some IPs didn't move." + testfailures=1 +fi + +echo "Continuing node $test_node" +try_command_on_node 1 $CTDB continue -n $test_node + +wait_until_node_has_status $test_node notstopped + +wait_until_node_has_some_ips "$test_node" diff --git a/ctdb/tests/simple/43_stop_recmaster_yield.sh b/ctdb/tests/simple/43_stop_recmaster_yield.sh new file mode 100755 index 00000000000..e7a825002e5 --- /dev/null +++ b/ctdb/tests/simple/43_stop_recmaster_yield.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb stop' causes a node to yield the recovery master role. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Determine which node is the recmaster. +2. Stop this node using the 'ctdb stop' command. +3. Verify that the status of the node changes to 'stopped'. +4. Verify that this node no longer has the recovery master role. + +Expected results: + +* The 'ctdb stop' command causes a node to yield the recmaster role. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +echo "Finding out which node is the recovery master..." +try_command_on_node -v 0 "$CTDB recmaster" +test_node=$out + +echo "Stopping node ${test_node} - it is the current recmaster..." +try_command_on_node 1 $CTDB stop -n $test_node + +wait_until_node_has_status $test_node stopped + +echo "Checking which node is the recovery master now..." +try_command_on_node -v 0 "$CTDB recmaster" +recmaster=$out + +if [ "$recmaster" != "$test_node" ] ; then + echo "OK: recmaster moved to node $recmaster" +else + echo "BAD: recmaster did not move" + exit 1 +fi diff --git a/ctdb/tests/simple/51_ctdb_bench.sh b/ctdb/tests/simple/51_ctdb_bench.sh new file mode 100755 index 00000000000..d4f7c542d16 --- /dev/null +++ b/ctdb/tests/simple/51_ctdb_bench.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Run the ctdb_bench test and sanity check the output. + +This doesn't test for performance regressions or similarly anything +useful. Only vague sanity checking of results is done. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run ctdb_bench on all nodes with default options. +3. Ensure that the number of +ve and -ive messages are within 1% of + each other. +4. Ensure that the number of messages per second is greater than 10. + +Expected results: + +* ctdb_bench runs without error and prints reasonable results. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +echo "Running ctdb_bench on all $num_nodes nodes." +try_command_on_node -v -pq all $CTDB_TEST_WRAPPER $VALGRIND ctdb_bench -n $num_nodes + +# Get the last line of output. +while read line ; do + prev=$line +done <<<"$out" + +pat='^(Ring: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec \(\+ve=[[:digit:]]+ -ve=[[:digit:]]+\)[[:space:]]?|Waiting for cluster[[:space:]]?)+$' +sanity_check_output 1 "$pat" "$out" + +# $prev should look like this: +# Ring: 10670.93 msgs/sec (+ve=53391 -ve=53373) +stuff="${prev##*Ring: }" +mps="${stuff% msgs/sec*}" + +if [ ${mps%.*} -ge 10 ] ; then + echo "OK: $mps msgs/sec >= 10 msgs/sec" +else + echo "BAD: $mps msgs/sec < 10 msgs/sec" + exit 1 +fi + +stuff="${stuff#*msgs/sec (+ve=}" +positive="${stuff%% *}" + +if [ $positive -gt 0 ] ; then + echo "OK: +ive ($positive) > 0" +else + echo "BAD: +ive ($positive) = 0" + exit 1 +fi + +stuff="${stuff#*-ve=}" +negative="${stuff%)}" + +if [ $negative -gt 0 ] ; then + echo "OK: -ive ($negative) > 0" +else + echo "BAD: -ive ($negative) = 0" + exit 1 +fi + +perc_diff=$(( ($positive - $negative) * 100 / $positive )) +perc_diff=${perc_diff#-} + +check_percent=5 +if [ $perc_diff -le $check_percent ] ; then + echo "OK: percentage difference between +ive and -ive ($perc_diff%) <= $check_percent%" +else + echo "BAD: percentage difference between +ive and -ive ($perc_diff%) > $check_percent%" + exit 1 +fi diff --git a/ctdb/tests/simple/52_ctdb_fetch.sh b/ctdb/tests/simple/52_ctdb_fetch.sh new file mode 100755 index 00000000000..54405d0d9d3 --- /dev/null +++ b/ctdb/tests/simple/52_ctdb_fetch.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Run the ctdb_fetch test and sanity check the output. + +This doesn't test for performance regressions or similarly anything +useful. Only vague sanity checking of results is done. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run ctdb_fetch on all nodes with default options. +3. Ensure that the number of +ve and -ive messages are within 1% of + each other. +4. Ensure that the number of messages per second is greater than 10. + +Expected results: + +* ctdb_fetch runs without error and prints reasonable results. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +echo "Running ctdb_fetch on all $num_nodes nodes." +try_command_on_node -v -pq all $CTDB_TEST_WRAPPER $VALGRIND ctdb_fetch -n $num_nodes + +pat='^(Fetch: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec[[:space:]]?|msg_count=[[:digit:]]+ on node [[:digit:]]|Fetching final record|DATA:|Test data|Waiting for cluster[[:space:]]?|.*: Reqid wrap!|Sleeping for [[:digit:]]+ seconds|)+$' +sanity_check_output 1 "$pat" "$out" + +# Filter out the performance figures: +out_fetch=$(echo "$out" | egrep '^(Fetch: .*)+$') + +# Get the last line of output. +while read line ; do + prev=$line +done <<<"$out_fetch" + +# $prev should look like this: +# Fetch: 10670.93 msgs/sec +stuff="${prev##*Fetch: }" +mps="${stuff% msgs/sec*}" + +if [ ${mps%.*} -ge 10 ] ; then + echo "OK: $mps msgs/sec >= 10 msgs/sec" +else + echo "BAD: $mps msgs/sec < 10 msgs/sec" + exit 1 +fi diff --git a/ctdb/tests/simple/53_ctdb_transaction.sh b/ctdb/tests/simple/53_ctdb_transaction.sh new file mode 100755 index 00000000000..b99b3a9fe42 --- /dev/null +++ b/ctdb/tests/simple/53_ctdb_transaction.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that the ctdb_transaction test succeeds. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run two copies of ctdb_transaction on each node with a 30 second + timeout. +3. Ensure that all ctdb_transaction processes complete successfully. + +Expected results: + +* ctdb_transaction runs without error. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +if test "x${CTDB_TEST_TIMELIMIT}" == "x" ; then + CTDB_TEST_TIMELIMIT=30 +fi + +t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}" + +echo "Running ctdb_transaction on all $num_nodes nodes." +try_command_on_node -v -pq all "$t & $t" diff --git a/ctdb/tests/simple/54_ctdb_transaction_recovery.sh b/ctdb/tests/simple/54_ctdb_transaction_recovery.sh new file mode 100755 index 00000000000..d796e94d8ed --- /dev/null +++ b/ctdb/tests/simple/54_ctdb_transaction_recovery.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that the ctdb_transaction test succeeds. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Run two copies of ctdb_transaction on each node with a 30 second + timeout. +3. Ensure that all ctdb_transaction processes complete successfully. + +Expected results: + +* ctdb_transaction runs without error. +EOF +} + +recovery_loop() +{ + local COUNT=1 + + while true ; do + echo Recovery $COUNT + try_command_on_node 0 $CTDB recover + sleep 2 + COUNT=$((COUNT + 1)) + done +} + +recovery_loop_start() +{ + recovery_loop > /tmp/recloop.out & + RECLOOP_PID=$! + ctdb_test_exit_hook_add "kill $RECLOOP_PID >/dev/null 2>&1" +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +if test "x${CTDB_TEST_TIMELIMIT}" == "x" ; then + CTDB_TEST_TIMELIMIT=30 +fi + +t="$CTDB_TEST_WRAPPER $VALGRIND ctdb_transaction --timelimit=${CTDB_TEST_TIMELIMIT}" + +echo "Starting recovery loop" +recovery_loop_start + +echo "Running ctdb_transaction on all $num_nodes nodes." +try_command_on_node -v -pq all "$t & $t" + diff --git a/ctdb/tests/simple/60_recoverd_missing_ip.sh b/ctdb/tests/simple/60_recoverd_missing_ip.sh new file mode 100755 index 00000000000..0734aeef8a1 --- /dev/null +++ b/ctdb/tests/simple/60_recoverd_missing_ip.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that the reconvery daemon handles unhosted IPs properly. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +select_test_node_and_ips + +echo "Running test against node $test_node and IP $test_ip" + +# Find the interface +try_command_on_node $test_node "$CTDB ip -v -Y | awk -F: -v ip=$test_ip '\$2 == ip { print \$4 }'" +iface="$out" + +if [ -z "$TEST_LOCAL_DAEMONS" ] ; then + # Find the netmask + try_command_on_node $test_node ip addr show to $test_ip + mask="${out##*/}" + mask="${mask%% *}" +else + mask="24" +fi + +echo "$test_ip/$mask is on $iface" + +# Push out the next monitor event so it is less likely to be cancelled +# and result in services not being restarted properly. +try_command_on_node $test_node $CTDB eventscript monitor + +echo "Deleting IP $test_ip from all nodes" +try_command_on_node -v $test_node $CTDB delip -n all $test_ip + +wait_until_ips_are_on_nodeglob '!' $test_node $test_ip + +try_command_on_node -v all $CTDB ip + +my_exit_hook () +{ + if [ -z "$TEST_LOCAL_DAEMONS" ] ; then + onnode -q all $CTDB enablescript "10.interface" + fi +} + +ctdb_test_exit_hook_add my_exit_hook + +# This forces us to wait until the ipreallocated associated with the +# delips is complete. +try_command_on_node $test_node $CTDB sync + +# This effectively cancels any monitor event that is in progress and +# runs a new one +try_command_on_node $test_node $CTDB eventscript monitor + +if [ -z "$TEST_LOCAL_DAEMONS" ] ; then + # Stop monitor events from bringing up the link status of an interface + try_command_on_node $test_node $CTDB disablescript 10.interface +fi + +echo "Marking interface $iface down on node $test_node" +try_command_on_node $test_node $CTDB setifacelink $iface down + +try_command_on_node $test_node $CTDB clearlog recoverd + +echo "Adding IP $test_ip to node $test_node" +try_command_on_node $test_node $CTDB addip $test_ip/$mask $iface + +# Give the recovery daemon enough time to start doing IP verification +sleep_for 15 + +try_command_on_node $test_node $CTDB getlog recoverd + +msg="Public IP '$test_ip' is not assigned and we could serve it" + +if grep "$msg" <<<"$out" ; then + echo "BAD: the recovery daemon noticed that the IP was unhosted" + exit 1 +else + echo "GOOD: the recovery daemon did not notice that the IP was unhosted" +fi + +try_command_on_node $test_node $CTDB clearlog recoverd + +echo "Marking interface $iface up on node $test_node" +try_command_on_node $test_node $CTDB setifacelink $iface up + +wait_until_ips_are_on_nodeglob $test_node $test_ip + +try_command_on_node -v $test_node $CTDB getlog recoverd + +if grep "$msg" <<<"$out" ; then + echo "GOOD: the recovery daemon noticed that the IP was unhosted" +else + echo "BAD: the recovery daemon did not notice that the IP was unhosted" + exit 1 +fi diff --git a/ctdb/tests/simple/70_recoverpdbbyseqnum.sh b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh new file mode 100755 index 00000000000..612366c310f --- /dev/null +++ b/ctdb/tests/simple/70_recoverpdbbyseqnum.sh @@ -0,0 +1,232 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +The tunable RecoverPDBBySeqNum controls how we perform recovery +on persistent databases. +The default is that persistent databases are recovered exactly the same +way as normal databases. That is that we recover record by record. + +If RecoverPDBBySeqNum is set to 1 AND if a record with the key +"__db_sequence_number__" can be found in the database, then instead we will +perform the recovery by picking the copy of the database from the node +that has the highest sequence number and ignore the content on all other +nodes. + + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. create a persistent test database +3. test that RecoveryPDBBySeqNum==0 and no seqnum record blends the database + during recovery +4. test that RecoveryPDBBySeqNum==0 and seqnum record blends the database + during recovery +5. test that RecoveryPDBBySeqNum==1 and no seqnum record blends the database + during recovery +6. test that RecoveryPDBBySeqNum==1 and seqnum record does not blend the database + during recovery + +Expected results: + +* that 3,4,5 will blend the databases and that 6 will recovery the highest seqnum + database + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +# create a temporary persistent database to test with +echo create persistent test database persistent_test.tdb +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach persistent_test.tdb persistent + + +# set RecoverPDBBySeqNum=0 +echo "setting RecoverPDBBySeqNum to 0" +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 0 + + + +# 3, +# If RecoverPDBBySeqNum==0 and no __db_sequence_number__ +# recover record by record +# +# wipe database +echo +echo test that RecoverPDBBySeqNum==0 and no __db_sequence_number__ blends the database during recovery +echo wipe the test database +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb + +# add one record to node 0 key==ABC data==ABC +TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(ABC) data(ABC) on node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243 +# +# add one record to node 1 key==DEF data==DEF +TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(DEF) data(DEF) on node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546 + +# force a recovery +echo force a recovery +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +# check that we now have both records on node 0 +num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l) +[ $num_records != "2" ] && { + echo "BAD: we did not end up with the expected two records after the recovery" + exit 1 +} +echo "OK. databases were blended" + + + +# 4, +# If RecoverPDBBySeqNum==0 and __db_sequence_number__ +# recover record by record +# +# wipe database +echo +echo test that RecoverPDBBySeqNum==0 and __db_sequence_number__ blends the database during recovery +echo wipe the test database +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb + +echo "add __db_sequence_number__==5 record to all nodes" +try_command_on_node -v 0 $CTDB_TEST_WRAPPER ctdb nodestatus all | grep pnn | sed -e"s/^pnn://" -e "s/ .*//" | while read PNN; do + TDB=`try_command_on_node -v -q $PNN $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` + try_command_on_node -q $PNN $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000500000000000000 +done + +# add one record to node 0 key==ABC data==ABC +TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(ABC) data(ABC) on node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243 +echo "add __db_sequence_number__==7 record to node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000700000000000000 + +# add one record to node 1 key==DEF data==DEF +TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(DEF) data(DEF) on node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546 +echo "add __db_sequence_number__==8 record to node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000800000000000000 + +# force a recovery +echo force a recovery +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +# check that we now have both records on node 0 +num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l) +[ $num_records != "2" ] && { + echo "BAD: we did not end up with the expected two records after the recovery" + exit 1 +} +echo "OK. databases were blended" + + + +# set RecoverPDBBySeqNum=1 +echo +echo "setting RecoverPDBBySeqNum to 1" +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 1 + + + +# 5, +# If RecoverPDBBySeqNum==1 and no __db_sequence_number__ +# recover record by record +# +# wipe database +echo +echo test that RecoverPDBBySeqNum==1 and no __db_sequence_number__ blends the database during recovery +echo wipe the test database +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb + +# add one record to node 0 key==ABC data==ABC +TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(ABC) data(ABC) on node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243 + +# add one record to node 1 key==DEF data==DEF +TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(DEF) data(DEF) on node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546 + +# force a recovery +echo force a recovery +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +# check that we now have both records on node 0 +num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l) +[ $num_records != "2" ] && { + echo "BAD: we did not end up with the expected two records after the recovery" + exit 1 +} +echo "OK. databases were blended" + + + +# 6, +# If RecoverPDBBySeqNum==1 and __db_sequence_number__ +# recover whole database +# +# wipe database +echo +echo test that RecoverPDBBySeqNum==1 and __db_sequence_number__ does not blend the database during recovery +echo wipe the test database +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb + +echo "add __db_sequence_number__==5 record to all nodes" +try_command_on_node -v 0 $CTDB_TEST_WRAPPER ctdb nodestatus all | grep pnn | sed -e"s/^pnn://" -e "s/ .*//" | while read PNN; do + TDB=`try_command_on_node -v -q $PNN $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` + try_command_on_node -q $PNN $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000500000000000000 +done + + +# add one record to node 0 key==ABC data==ABC +TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(ABC) data(ABC) on node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243 +echo "add __db_sequence_number__==7 record to node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000700000000000000 + +# add one record to node 1 key==DEF data==DEF +TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(DEF) data(DEF) on node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546 +echo "add __db_sequence_number__==8 record to node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x5f5f64625f73657175656e63655f6e756d6265725f5f00 0x0700000000000000000000000000000000000000000000000800000000000000 + +# force a recovery +echo force a recovery +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +# check that we now have both records on node 0 +num_records=$(try_command_on_node -v -pq 0 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | egrep "ABC|DEF" | wc -l) +[ $num_records != "1" ] && { + echo "BAD: we did not end up with the expected single record after the recovery" + exit 1 +} + +echo "OK. databases were not blended" + + + +# set RecoverPDBBySeqNum=1 +echo "setting RecoverPDBBySeqNum back to 0" +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 0 diff --git a/ctdb/tests/simple/71_ctdb_wipedb.sh b/ctdb/tests/simple/71_ctdb_wipedb.sh new file mode 100755 index 00000000000..0cd07ccbca6 --- /dev/null +++ b/ctdb/tests/simple/71_ctdb_wipedb.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +The command 'ctdb wipedb' is used to clear a database across the whole +cluster. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. create a persistent test database +3, add some records to node #0 and node #1 +4, perform wipedb on node #0 and verify the database is empty on both node 0 and 1 + +Expected results: + +* that 4 will result in empty database + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +# create a temporary persistent database to test with +echo create persistent test database persistent_test.tdb +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach persistent_test.tdb persistent + + +# 3, +# add one record to node 0 key==ABC data==ABC +TDB=`try_command_on_node -v -q 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(ABC) data(ABC) on node 0" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x414243 0x070000000000000000000000000000000000000000000000414243 +# +# add one record to node 1 key==DEF data==DEF +TDB=`try_command_on_node -v -q 1 $CTDB_TEST_WRAPPER ctdb getdbmap | grep persistent_test.tdb | sed -e "s/.*path://" -e "s/ .*//"` +echo "store key(DEF) data(DEF) on node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb tstore $TDB 0x444546 0x070000000000000000000000000000000000000000000000444546 + + +# 4, +echo wipe the persistent test database +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb persistent_test.tdb +echo force a recovery +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +# check that the database is wiped +num_records=$(try_command_on_node -v -pq 1 $CTDB_TEST_WRAPPER ctdb cattdb persistent_test.tdb | grep key | wc -l) +[ $num_records != "0" ] && { + echo "BAD: we did not end up with an empty database" + exit 1 +} +echo "OK. database was wiped" + diff --git a/ctdb/tests/simple/72_update_record_persistent.sh b/ctdb/tests/simple/72_update_record_persistent.sh new file mode 100755 index 00000000000..254ce195c59 --- /dev/null +++ b/ctdb/tests/simple/72_update_record_persistent.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +UPDATE_RECORD control should be able to create new records and update +existing records in a persistent database. + +Prerequisites: + +* An active CTDB cluster with at least one active node. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. create a persistent test database +3, wipe the database to make sure it is empty +4, create a new record +5, update the record + +Expected results: + +* 4 created record found in the tdb +* 5 updated record found in the tdb + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +TDB=persistent_test.tdb + +# create a temporary persistent database to test with +echo create persistent test database $TDB +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach $TDB persistent + + +# 3, +echo wipe the persistent test database +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb $TDB +echo force a recovery +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +# check that the database is wiped +num_records=$(try_command_on_node -v -pq 1 $CTDB_TEST_WRAPPER ctdb cattdb $TDB | grep key | wc -l) +[ $num_records != "0" ] && { + echo "BAD: we did not end up with an empty database" + exit 1 +} +echo "OK. database was wiped" + +# 4, +echo Create a new record in the persistent database using UPDATE_RECORD +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb_update_record_persistent --database=$TDB --record=Update_Record_Persistent --value=FirstValue + +try_command_on_node -q 0 "ctdb cattdb $TDB | grep 'FirstValue' | wc -l" +[ $out != 1 ] && { + echo "BAD: we did find the record after the create/update" + exit 1 +} + +# 5, +echo Modify an existing record in the persistent database using UPDATE_RECORD +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb_update_record_persistent --database=$TDB --record=Update_Record_Persistent --value=SecondValue + +try_command_on_node -q 0 "ctdb cattdb $TDB | grep 'FirstValue' | wc -l" +[ $out != 0 ] && { + echo "BAD: we still found the old record after the modify/update" + exit 1 +} + +try_command_on_node -q 0 "ctdb cattdb $TDB | grep 'SecondValue' | wc -l" +[ $out != 1 ] && { + echo "BAD: could not find the record after the modify/update" + exit 1 +} + + +echo wipe the persistent test databases and clean up +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb $TDB diff --git a/ctdb/tests/simple/73_tunable_NoIPTakeover.sh b/ctdb/tests/simple/73_tunable_NoIPTakeover.sh new file mode 100755 index 00000000000..eee3da9db2a --- /dev/null +++ b/ctdb/tests/simple/73_tunable_NoIPTakeover.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Verify that 'ctdb setvar NoIPTakeover 1' stops ip addresses from being failed +over onto the node. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. Use 'ctdb ip' on one of the nodes to list the IP addresses being + served. +3. Use 'ctdb moveip' to move an address from one node to another. +4. Verify that the IP is no longer being hosted by the first node and is now being hosted by the second node. + +Expected results: + +* 'ctdb moveip' allows an IP address to be moved between cluster nodes. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +try_command_on_node 0 "$CTDB listnodes | wc -l" +num_nodes="$out" +echo "There are $num_nodes nodes..." + +if [ $num_nodes -lt 2 ] ; then + echo "Less than 2 nodes!" + exit 1 +fi + + +echo "Wait until the ips are reallocated" +sleep 30 +try_command_on_node -q 0 "$CTDB ipreallocate" + +num=`try_command_on_node -v 1 "$CTDB ip" | grep -v Public | egrep " 1$" | wc -l` +echo "Number of addresses on node 1 : $num" + + +echo "Turning on NoIPTakeover on node 1" +try_command_on_node -q 1 "$CTDB setvar NoIPTakeover 1" +try_command_on_node -q 1 "$CTDB ipreallocate" + +echo Disable node 1 +try_command_on_node -q 1 "$CTDB disable" +try_command_on_node -q 1 "$CTDB ipreallocate" +num=`try_command_on_node -v 1 "$CTDB ip" | grep -v Public | egrep " 1$" | wc -l` +echo "Number of addresses on node 1 : $num" +[ "$num" != "0" ] && { + echo "BAD: node 1 still hosts ip addresses" + exit 1 +} + + +echo "Enable node 1 again" +try_command_on_node -q 1 "$CTDB enable" +sleep 30 +try_command_on_node -q 1 "$CTDB ipreallocate" +try_command_on_node -q 1 "$CTDB ipreallocate" +num=`try_command_on_node -v 1 "$CTDB ip" | grep -v Public | egrep " 1$" | wc -l` +echo "Number of addresses on node 1 : $num" +[ "$num" != "0" ] && { + echo "BAD: node took over ip addresses" + exit 1 +} + + +echo "OK. ip addresses were not taken over" +exit 0 diff --git a/ctdb/tests/simple/75_readonly_records_basic.sh b/ctdb/tests/simple/75_readonly_records_basic.sh new file mode 100755 index 00000000000..f243ea1e21c --- /dev/null +++ b/ctdb/tests/simple/75_readonly_records_basic.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Readonly records can be activated at runtime using a ctdb command. +If readonly records are not activated, then any attempt to fetch a readonly +copy should be automatically upgraded to a read-write fetch_lock(). + +If readonly delegations are present, then any attempt to aquire a read-write +fetch_lock will trigger all delegations to be revoked before the fetch lock +completes. + + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Verify that the status on all of the ctdb nodes is 'OK'. +2. create a test database and some records +3. try to fetch readonly records, this should not result in any delegations +4. activate readonly support +5. try to fetch readonly records, this should result in delegations +6. do a fetchlock and the delegations should be revoked +7. try to fetch readonly records, this should result in delegations +8. do a recovery and the delegations should be revoked + +Expected results: + +3. No delegations created when db is not in readonly mode +4. It is possible to activate readonly support for a database +5. Delegations should be created +6. Delegations should be revoked +8. Delegations should be revoked + + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + + +# create a temporary database to test with +echo create test database test.tdb +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach test.tdb + + +# create some records +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb_update_record + +# +# 3 +# try readonly requests +echo Try some readonly fetches, these should all be upgraded to full fetchlocks +try_command_on_node -q 0,1,2 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" + +# no delegations should have been created +numreadonly=`try_command_on_node -v all $CTDB_TEST_WRAPPER ctdb cattdb test.tdb | grep READONLY | wc -l` +[ "$numreadonly" != "0" ] && { + echo "BAD: readonly delegations were created, but the feature is not activated on the database" + exit 1 +} + + +# +# 4 +# + +echo Activating ReadOnly record support for test.tdb ... +# activate readonly support +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setdbreadonly test.tdb +numreadonly=`try_command_on_node -v 0 $CTDB_TEST_WRAPPER ctdb getdbmap | grep READONLY | wc -l` +[ "$numreadonly" != "1" ] && { + echo BAD: could not activate readonly support for the test database + exit 1 +} + + + +# +# 5 +# + +echo Create some readonly delegations ... +# fetch record to node 0 and make it dmaster +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb_update_record + +# fetch readonly to node 1 +try_command_on_node -v 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" + +numreadonly=`try_command_on_node -v all $CTDB_TEST_WRAPPER ctdb cattdb test.tdb | grep RO_HAVE | wc -l` +[ "$numreadonly" != "2" ] && { + echo BAD: could not create readonly delegation + exit 1 +} + + + + +# +# 6 +# + +echo verify that a fetchlock will revoke the delegations ... +# fetch record to node 0 and make it dmaster +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb_update_record + +numreadonly=`try_command_on_node -v all $CTDB_TEST_WRAPPER ctdb cattdb test.tdb | grep RO_HAVE | wc -l` +[ "$numreadonly" != "0" ] && { + echo BAD: fetchlock did not revoke delegations + exit 1 +} + + +# +# 7 +# + +echo Create some readonly delegations ... +# fetch record to node 0 and make it dmaster +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb_update_record + +# fetch readonly to node 1 +try_command_on_node -v 0 $CTDB_TEST_WRAPPER "ctdb_fetch_readonly_once </dev/null" + +numreadonly=`try_command_on_node -v all $CTDB_TEST_WRAPPER ctdb cattdb test.tdb | grep RO_HAVE | wc -l` +[ "$numreadonly" != "2" ] && { + echo BAD: could not create readonly delegation + exit 1 +} + + + + +# +# 8 +# + +echo verify that a recovery will revoke the delegations ... +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb recover + +numreadonly=`try_command_on_node -v all $CTDB_TEST_WRAPPER ctdb cattdb test.tdb | grep RO_HAVE | wc -l` +[ "$numreadonly" != "0" ] && { + echo BAD: recovery did not revoke delegations + exit 1 +} + +echo OK. test completed successfully +exit 0 diff --git a/ctdb/tests/simple/76_ctdb_pdb_recovery.sh b/ctdb/tests/simple/76_ctdb_pdb_recovery.sh new file mode 100755 index 00000000000..096b9d5c244 --- /dev/null +++ b/ctdb/tests/simple/76_ctdb_pdb_recovery.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +The recovery process based on RSN for persistent databases is defective. +For persistent databases sequence number based recovery method should be +used. This test checks for the defect in the RSN based recovery method +for persistent databases and confirms that the same issue is not observed +when using sequence number based recovery method. + +Steps: + +1. Create a persistent database +2. Add a record and update it few times. +3. Delete the record +4. Turn off one of the nodes +5. Add a record with same key. +6. Turn on the stopped node + +Expected results: + +* Check that the record is deleted (RSN based recovery) and record is + present (sequence number based recovery) + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +do_test() +{ +# Wipe Test database +echo "wipe test database" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb $TESTDB + +# Add a record key=test1 data=value1 +# and update values +for value in value1 value2 value3 value4 value5 ; do + echo "store key(test1) data($value)" + try_command_on_node -q 0 "(echo -ne $value > /tmp/test_data)" + try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb pstore $TESTDB test1 /tmp/test_data +done + +# Delete record +echo "delete key(test1)" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb pdelete $TESTDB test1 + +# Stop a node +echo "stop node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb stop + +wait_until_node_has_status 1 stopped + +# Add a record key=test1 data=value2 +echo "store key(test1) data(newvalue1)" +try_command_on_node -q 0 "(echo -ne newvalue1 > /tmp/test_data)" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb pstore $TESTDB test1 /tmp/test_data + +# Continue node +echo "contine node 1" +try_command_on_node -q 1 $CTDB_TEST_WRAPPER ctdb continue + +wait_until_node_has_status 1 notstopped + +} + +# +# Main test +# +TESTDB="persistent_test.tdb" + +status=0 + +# Create a temporary persistent database to test with +echo "create persistent test database $TESTDB" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach $TESTDB persistent + +echo "set RecoverPDBBySeqNum to 0" +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 0 + +do_test +if try_command_on_node 0 $CTDB_TEST_WRAPPER ctdb pfetch $TESTDB test1 ; then + echo "GOOD: Record was not deleted (recovery by RSN worked)" +else + echo "BAD: Record was not deleted" + status=1 +fi + +# Set RecoverPDBBySeqNum = 1 +echo "set RecoverPDBBySeqNum to 1" +try_command_on_node -q all $CTDB_TEST_WRAPPER ctdb setvar RecoverPDBBySeqNum 1 + +do_test +if try_command_on_node 0 $CTDB_TEST_WRAPPER ctdb pfetch $TESTDB test1 ; then + echo "GOOD: Record was not deleted (recovery by sequnce number worked)" +else + echo "BAD: Record was deleted" + status=1 +fi + +exit $status diff --git a/ctdb/tests/simple/77_ctdb_db_recovery.sh b/ctdb/tests/simple/77_ctdb_db_recovery.sh new file mode 100755 index 00000000000..00fa0965e00 --- /dev/null +++ b/ctdb/tests/simple/77_ctdb_db_recovery.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Recovery can under certain circumstances lead to old record copies +resurrecting: Recovery selects the newest record copy purely by RSN. At +the end of the recovery, the recovery master is the dmaster for all +records in all (non-persistent) databases. And the other nodes locally +hold the complete copy of the databases. The bug is that the recovery +process does not increment the RSN on the recovery master at the end of +the recovery. Now clients acting directly on the Recovery master will +directly change a record's content on the recmaster without migration +and hence without RSN bump. So a subsequent recovery can not tell that +the recmaster's copy is newer than the copies on the other nodes, since +their RSN is the same. Hence, if the recmaster is not node 0 (or more +precisely not the active node with the lowest node number), the recovery +will choose copies from nodes with lower number and stick to these. + +Steps: + +1. Create a test database +2. Add a record with value value1 on recovery master +3. Force a recovery +4. Update the record with value value2 on recovery master +5. Force a recovery +6. Fetch the record + +Expected results: + +* The record should have value value2 and not value1 + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +# +# Main test +# +TESTDB="rec_test.tdb" + +status=0 + +# Make sure node 0 is not the recovery master +echo "find out which node is recmaster" +try_command_on_node -q any $CTDB_TEST_WRAPPER ctdb recmaster +recmaster="$out" +if [ "$recmaster" = "0" ]; then + echo "node 0 is recmaster, disable recmasterrole on node 0" + # + # Note: + # It should be sufficient to run "ctdb setrecmasterrole off" + # on node 0 and wait for election and recovery to finish. + # But there were problems related to this in this automatic + # test, so for now use "ctdb stop" and "ctdb continue". + # + echo "stop node 0" + try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb stop + wait_until_node_has_status 0 stopped + echo "continue node 0" + try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb continue + wait_until_node_has_status 0 notstopped + + try_command_on_node -q any $CTDB_TEST_WRAPPER ctdb recmaster + recmaster="$out" + if [ "$recmaster" = "0" ]; then + echo "failed to move recmaster to different node" + exit 1 + fi +fi + +echo "Recmaster:$recmaster" + +# Create a temporary non-persistent database to test with +echo "create test database $TESTDB" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb attach $TESTDB + +# Wipe Test database +echo "wipe test database" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb wipedb $TESTDB + +# Add a record key=test1 data=value1 +echo "store key(test1) data(value1)" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb writekey $TESTDB test1 value1 + +# Fetch a record key=test1 +echo "read key(test1)" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb readkey $TESTDB test1 +echo "$out" + +# Do a recovery +echo "force recovery" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb recover + +wait_until_node_has_status $recmaster recovered + +# Add a record key=test1 data=value2 +echo "store key(test1) data(value2)" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb writekey $TESTDB test1 value2 + +# Fetch a record key=test1 +echo "read key(test1)" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb readkey $TESTDB test1 +echo "$out" + +# Do a recovery +echo "force recovery" +try_command_on_node -q $recmaster $CTDB_TEST_WRAPPER ctdb recover + +wait_until_node_has_status $recmaster recovered + +# Verify record key=test1 +echo "read key(test1)" +try_command_on_node $recmaster $CTDB_TEST_WRAPPER ctdb readkey $TESTDB test1 +echo "$out" +if [ "$out" = "Data: size:6 ptr:[value2]" ]; then + echo "GOOD: Recovery did not corrupt database" +else + echo "BAD: Recovery corrupted database" + status=1 +fi + +exit $status diff --git a/ctdb/tests/simple/80_ctdb_traverse.sh b/ctdb/tests/simple/80_ctdb_traverse.sh new file mode 100755 index 00000000000..65a991af315 --- /dev/null +++ b/ctdb/tests/simple/80_ctdb_traverse.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +Test CTDB cluster wide traverse code. + +Prerequisites: + +* An active CTDB cluster with at least 2 active nodes. + +Steps: + +1. Create a test database +2. Add records on different nodes +3. Run traverse + +Expected results: + +* All records are retrieved. + +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +ctdb_test_init "$@" + +set -e + +cluster_is_healthy + +# Reset configuration +ctdb_restart_when_done + +try_command_on_node 0 "$CTDB listnodes" +num_nodes=$(echo "$out" | wc -l) + +num_records=1000 + +TESTDB="traverse_test.tdb" + +echo "create test database $TESTDB" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb attach $TESTDB + +echo "wipe test database $TESTDB" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb wipedb $TESTDB + +echo "Add $num_records records to database" +i=0 +while [ $i -lt $num_records ]; do + key=$(printf "key-%04x" $i) + value="value-$i" + + n=$[ $i % $num_nodes ] + try_command_on_node -q $n $CTDB_TEST_WRAPPER ctdb writekey $TESTDB $key $value + + i=$[ $i + 1 ] +done + +echo "Start a traverse and collect records" +try_command_on_node -q 0 $CTDB_TEST_WRAPPER ctdb catdb $TESTDB + +num_read=$(echo "$out" | tail -n 1 | cut -d\ -f2) +if [ $num_read -eq $num_records ]; then + echo "GOOD: All $num_records records retrieved" + status=0 +else + echo "BAD: Only $num_read/$num_records records retrieved" + status=1 +fi + +exit $status diff --git a/ctdb/tests/simple/99_daemons_shutdown.sh b/ctdb/tests/simple/99_daemons_shutdown.sh new file mode 100755 index 00000000000..3583828b71a --- /dev/null +++ b/ctdb/tests/simple/99_daemons_shutdown.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +test_info() +{ + cat <<EOF +If we running local daemons and TEST_CLEANUP is true then shutdown the daemons. + +No error if ctdbd is not already running on the cluster. + +Prerequisites: + +* Nodes must be accessible via 'onnode'. +EOF +} + +. "${TEST_SCRIPTS_DIR}/integration.bash" + +# Do not call ctdb_test_init() here. It will setup ctdb_test_exit() +# to run and that will find the daemons missing and restart them! + +if [ -n "$TEST_LOCAL_DAEMONS" ] && $TEST_CLEANUP ; then + daemons_stop +fi diff --git a/ctdb/tests/simple/README b/ctdb/tests/simple/README new file mode 100644 index 00000000000..3ac738dc6ce --- /dev/null +++ b/ctdb/tests/simple/README @@ -0,0 +1,2 @@ +Simple integration tests. These can be run against a pool of CTDB +daemons running on the local machine - aka "local daemons". diff --git a/ctdb/tests/src/ctdb_bench.c b/ctdb/tests/src/ctdb_bench.c new file mode 100644 index 00000000000..3323589b34c --- /dev/null +++ b/ctdb/tests/src/ctdb_bench.c @@ -0,0 +1,262 @@ +/* + simple ctdb benchmark + + Copyright (C) Andrew Tridgell 2006 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" +#include "ctdb_client.h" +#include "ctdb_private.h" + +#include <sys/time.h> +#include <time.h> + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + + +static int timelimit = 10; +static int num_records = 10; +static int num_nodes; + +enum my_functions {FUNC_INCR=1, FUNC_FETCH=2}; + +/* + ctdb call function to increment an integer +*/ +static int incr_func(struct ctdb_call_info *call) +{ + if (call->record_data.dsize == 0) { + call->new_data = talloc(call, TDB_DATA); + if (call->new_data == NULL) { + return CTDB_ERR_NOMEM; + } + call->new_data->dptr = talloc_size(call, 4); + call->new_data->dsize = 4; + *(uint32_t *)call->new_data->dptr = 0; + } else { + call->new_data = &call->record_data; + } + (*(uint32_t *)call->new_data->dptr)++; + return 0; +} + +/* + ctdb call function to fetch a record +*/ +static int fetch_func(struct ctdb_call_info *call) +{ + call->reply_data = &call->record_data; + return 0; +} + + +static int msg_count; +static int msg_plus, msg_minus; + +/* + handler for messages in bench_ring() +*/ +static void ring_message_handler(struct ctdb_context *ctdb, uint64_t srvid, + TDB_DATA data, void *private_data) +{ + int incr = *(int *)data.dptr; + int *count = (int *)private_data; + int dest; + + (*count)++; + dest = (ctdb_get_pnn(ctdb) + num_nodes + incr) % num_nodes; + ctdb_client_send_message(ctdb, dest, srvid, data); + if (incr == 1) { + msg_plus++; + } else { + msg_minus++; + } +} + + +static void send_start_messages(struct ctdb_context *ctdb, int incr) +{ + /* two messages are injected into the ring, moving + in opposite directions */ + int dest; + TDB_DATA data; + + data.dptr = (uint8_t *)&incr; + data.dsize = sizeof(incr); + + dest = (ctdb_get_pnn(ctdb) + num_nodes + incr) % num_nodes; + ctdb_client_send_message(ctdb, dest, 0, data); +} + +static void each_second(struct event_context *ev, struct timed_event *te, + struct timeval t, void *private_data) +{ + struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context); + + /* we kickstart the ring into action by inserting messages from node + with pnn 0. + it may happen that some other node does not yet have ctdb_bench + running in which case the ring is broken and the messages are lost. + if so, once every second try again to restart the ring + */ + if (msg_plus == 0) { +// printf("no messages recevied, try again to kickstart the ring in forward direction...\n"); + send_start_messages(ctdb, 1); + } + if (msg_minus == 0) { +// printf("no messages recevied, try again to kickstart the ring in reverse direction...\n"); + send_start_messages(ctdb, -1); + } + event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb); +} + +static void dummy_event(struct event_context *ev, struct timed_event *te, + struct timeval t, void *private_data) +{ + struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context); + event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), dummy_event, ctdb); +} + +/* + benchmark sending messages in a ring around the nodes +*/ +static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev) +{ + int pnn=ctdb_get_pnn(ctdb); + + if (pnn == 0) { + event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb); + } else { + event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(1, 0), dummy_event, ctdb); + } + + start_timer(); + while (end_timer() < timelimit) { + if (pnn == 0 && msg_count % 10000 == 0 && end_timer() > 0) { + printf("Ring: %.2f msgs/sec (+ve=%d -ve=%d)\r", + msg_count/end_timer(), msg_plus, msg_minus); + fflush(stdout); + } + event_loop_once(ev); + } + + printf("Ring: %.2f msgs/sec (+ve=%d -ve=%d)\n", + msg_count/end_timer(), msg_plus, msg_minus); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" }, + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + { NULL, 'n', POPT_ARG_INT, &num_nodes, 0, "num_nodes", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + int ret; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + if (num_nodes == 0) { + printf("You must specify the number of nodes\n"); + exit(1); + } + + ev = event_context_init(NULL); + + /* initialise ctdb */ + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + if (ctdb == NULL) { + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb", + false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + /* setup a ctdb call function */ + ret = ctdb_set_call(ctdb_db, incr_func, FUNC_INCR); + if (ret != 0) { + DEBUG(DEBUG_DEBUG,("ctdb_set_call() failed, ignoring return code %d\n", ret)); + } + ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH); + if (ret != 0) { + DEBUG(DEBUG_DEBUG,("ctdb_set_call() failed, ignoring return code %d\n", ret)); + } + + if (ctdb_client_set_message_handler(ctdb, 0, ring_message_handler,&msg_count)) + goto error; + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + bench_ring(ctdb, ev); + +error: + return 0; +} diff --git a/ctdb/tests/src/ctdb_fetch.c b/ctdb/tests/src/ctdb_fetch.c new file mode 100644 index 00000000000..b900efa7c3c --- /dev/null +++ b/ctdb/tests/src/ctdb_fetch.c @@ -0,0 +1,278 @@ +/* + simple ctdb benchmark + + Copyright (C) Andrew Tridgell 2006 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + + +static int timelimit = 10; +static int num_records = 10; +static int num_nodes; +static int msg_count; + +#define TESTKEY "testkey" + +/* + fetch a record + store a expanded record + send a message to next node to tell it to do the same +*/ +static void bench_fetch_1node(struct ctdb_context *ctdb) +{ + TDB_DATA key, data, nulldata; + struct ctdb_db_context *ctdb_db; + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + int dest, ret; + struct ctdb_record_handle *h; + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + + ctdb_db = ctdb_db_handle(ctdb, "test.tdb"); + + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + return; + } + + if (data.dsize > 1000) { + data.dsize = 0; + } + + if (data.dsize == 0) { + data.dptr = (uint8_t *)talloc_asprintf(tmp_ctx, "Test data\n"); + } + data.dptr = (uint8_t *)talloc_asprintf_append((char *)data.dptr, + "msg_count=%d on node %d\n", + msg_count, ctdb_get_pnn(ctdb)); + if (data.dptr == NULL) { + printf("Failed to create record\n"); + talloc_free(tmp_ctx); + return; + } + data.dsize = strlen((const char *)data.dptr)+1; + + ret = ctdb_record_store(h, data); + talloc_free(h); + if (ret != 0) { + printf("Failed to store record\n"); + } + + talloc_free(tmp_ctx); + + /* tell the next node to do the same */ + nulldata.dptr = NULL; + nulldata.dsize = 0; + + dest = (ctdb_get_pnn(ctdb) + 1) % num_nodes; + ctdb_client_send_message(ctdb, dest, 0, nulldata); +} + +/* + handler for messages in bench_ring() +*/ +static void message_handler(struct ctdb_context *ctdb, uint64_t srvid, + TDB_DATA data, void *private_data) +{ + msg_count++; + bench_fetch_1node(ctdb); +} + + +/* + * timeout handler - noop + */ +static void timeout_handler(struct event_context *ev, struct timed_event *timer, + struct timeval curtime, void *private_data) +{ + return; +} + +/* + benchmark the following: + + fetch a record + store a expanded record + send a message to next node to tell it to do the same + +*/ +static void bench_fetch(struct ctdb_context *ctdb, struct event_context *ev) +{ + int pnn=ctdb_get_pnn(ctdb); + + if (pnn == num_nodes - 1) { + bench_fetch_1node(ctdb); + } + + start_timer(); + event_add_timed(ev, ctdb, timeval_current_ofs(timelimit,0), timeout_handler, NULL); + + while (end_timer() < timelimit) { + if (pnn == 0 && msg_count % 100 == 0 && end_timer() > 0) { + printf("Fetch: %.2f msgs/sec\r", msg_count/end_timer()); + fflush(stdout); + } + if (event_loop_once(ev) != 0) { + printf("Event loop failed!\n"); + break; + } + } + + printf("Fetch: %.2f msgs/sec\n", msg_count/end_timer()); +} + +/* + handler for reconfigure message +*/ +static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid, + TDB_DATA data, void *private_data) +{ + int *ready = (int *)private_data; + *ready = 1; +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" }, + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + { NULL, 'n', POPT_ARG_INT, &num_nodes, 0, "num_nodes", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + TDB_DATA key, data; + struct ctdb_record_handle *h; + int cluster_ready=0; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* talloc_enable_leak_report_full(); */ + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + if (num_nodes == 0) { + printf("You must specify the number of nodes\n"); + exit(1); + } + + ev = event_context_init(NULL); + tevent_loop_allow_nesting(ev); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + + if (ctdb == NULL) { + printf("failed to connect to ctdb daemon.\n"); + exit(1); + } + + ctdb_client_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler, + &cluster_ready); + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb", + false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + ctdb_client_set_message_handler(ctdb, 0, message_handler, &msg_count); + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + /* This test has a race condition. If CTDB receives the message from previous + * node, before this node has registered for that message, this node will never + * receive that message and will block on receive. Sleeping for some time will + * hopefully ensure that the test program on all the nodes register for messages. + */ + printf("Sleeping for %d seconds\n", num_nodes); + sleep(num_nodes); + bench_fetch(ctdb, ev); + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + + printf("Fetching final record\n"); + + h = ctdb_fetch_lock(ctdb_db, ctdb, key, &data); + + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + exit(1); + } + + printf("DATA:\n%s\n", (char *)data.dptr); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_fetch_one.c b/ctdb/tests/src/ctdb_fetch_one.c new file mode 100644 index 00000000000..ba0e183fe57 --- /dev/null +++ b/ctdb/tests/src/ctdb_fetch_one.c @@ -0,0 +1,145 @@ +/* + simple ctdb benchmark + This test just fetch_locks a record and releases it in a loop. + + Copyright (C) Ronnie Sahlberg 2009 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> + +static int timelimit = 10; +static int lock_count = 0; + +static struct ctdb_db_context *ctdb_db; + +#define TESTKEY "testkey" + + +static void alarm_handler(int sig) +{ + printf("Locks:%d\n", lock_count); + lock_count=0; + + timelimit--; + if (timelimit <= 0) { + exit(0); + } + alarm(1); +} + +/* + Just try locking/unlocking the same record over and over +*/ +static void bench_fetch_one_loop(struct ctdb_context *ctdb, struct event_context *ev) +{ + TDB_DATA key, data; + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + + + while (1) { + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + struct ctdb_record_handle *h; + + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + continue; + } + + talloc_free(tmp_ctx); + lock_count++; + } +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + + if (ctdb == NULL) { + printf("failed to connect to ctdb daemon.\n"); + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb", + false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + signal(SIGALRM, alarm_handler); + alarm(1); + + bench_fetch_one_loop(ctdb, ev); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_fetch_readonly_loop.c b/ctdb/tests/src/ctdb_fetch_readonly_loop.c new file mode 100644 index 00000000000..5944fb7bdbb --- /dev/null +++ b/ctdb/tests/src/ctdb_fetch_readonly_loop.c @@ -0,0 +1,145 @@ +/* + simple ctdb test tool + This test just fetch_locks a record and releases it in a loop. + + Copyright (C) Ronnie Sahlberg 2009 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "system/time.h" +#include "popt.h" +#include "cmdline.h" +#include "ctdb_private.h" + +static struct ctdb_db_context *ctdb_db; + +const char *TESTKEY = "testkey"; +static int count; + +/* + Just try locking/unlocking a single record once +*/ +static void fetch_lock_once(struct ctdb_context *ctdb, struct event_context *ev) +{ + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + TDB_DATA key, data; + struct ctdb_record_handle *h; + static time_t t = 0, t2; + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + +// printf("Trying to fetch lock the record ...\n"); + + h = ctdb_fetch_readonly_lock(ctdb_db, tmp_ctx, key, &data, true); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + exit(10); + } + + count++; + t2 = time(NULL); + if (t != 0 && t != t2) { + static int last_count = 0; + + printf("count : %d\n", count - last_count); + last_count = count; + } + t = t2; + + talloc_free(tmp_ctx); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + TDB_DATA key; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "record", 'r', POPT_ARG_STRING, &TESTKEY, 0, "record", "string" }, + POPT_TABLEEND + }; + int opt, ret; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(5, 0)); + if (ctdb == NULL) { + exit(1); + } + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + + ret = ctdb_ctrl_getvnnmap(ctdb, timeval_zero(), CTDB_CURRENT_NODE, ctdb, &ctdb->vnn_map); + if (ret != 0) { + printf("failed to get vnnmap\n"); + exit(10); + } + printf("Record:%s\n", TESTKEY); + printf("Lmaster : %d\n", ctdb_lmaster(ctdb, &key)); + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(5, 0), "test.tdb", false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + while (1) { + fetch_lock_once(ctdb, ev); + } + + return 0; +} diff --git a/ctdb/tests/src/ctdb_fetch_readonly_once.c b/ctdb/tests/src/ctdb_fetch_readonly_once.c new file mode 100644 index 00000000000..5dc64e0e063 --- /dev/null +++ b/ctdb/tests/src/ctdb_fetch_readonly_once.c @@ -0,0 +1,117 @@ +/* + simple ctdb test tool + This test just fetch_locks a record and releases it once. + + Copyright (C) Ronnie Sahlberg 2009 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include <poll.h> + +const char *TESTKEY = "testkey"; + +/* + Just try locking/unlocking a single record once +*/ +static void fetch_readonly_once(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key) +{ + TDB_DATA data; + struct ctdb_record_handle *h; + + printf("Trying to readonly fetch lock the record ...\n"); + + h = ctdb_fetch_readonly_lock(ctdb_db, ctdb, key, &data, 1); + if (h == NULL) { + fprintf(stderr, "Failed to get readonly lock\n"); + exit(1); + } + + talloc_free(h); + printf("Record released.\n"); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + struct event_context *ev; + + TDB_DATA key; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + { "record", 'r', POPT_ARG_STRING, &TESTKEY, 0, "record", "string" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + if (ctdb == NULL) { + printf("failed to connect to ctdb daemon.\n"); + exit(1); + } + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(3, 0), "test.tdb", + false, 0); + if (!ctdb_db) { + fprintf(stderr, "ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(10); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + fetch_readonly_once(ctdb, ctdb_db, key); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_functest.c b/ctdb/tests/src/ctdb_functest.c new file mode 100644 index 00000000000..16ca4fddd64 --- /dev/null +++ b/ctdb/tests/src/ctdb_functest.c @@ -0,0 +1,189 @@ +/* + Tests for tools/ctdb.c and CTDB client stubs + + Copyright (C) Martin Schwenke 2011 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#define CTDB_TEST_OVERRIDE_MAIN +#include "ctdb_test.c" + +static void test_read_nodemap(void) +{ + struct ctdb_context *ctdb = talloc_zero(NULL, struct ctdb_context); + + ctdb_test_stubs_read_nodemap(ctdb); + ctdb_test_stubs_print_nodemap(ctdb); + + talloc_free(ctdb); +} + +static void test_read_ifaces(void) +{ + struct ctdb_context *ctdb = talloc_zero(NULL, struct ctdb_context); + + ctdb_test_stubs_read_ifaces(ctdb); + ctdb_test_stubs_print_ifaces(ctdb); + + talloc_free(ctdb); +} + +static void test_read_vnnmap(void) +{ + struct ctdb_context *ctdb = talloc_zero(NULL, struct ctdb_context); + + ctdb_test_stubs_read_vnnmap(ctdb); + ctdb_test_stubs_print_vnnmap(ctdb); + + talloc_free(ctdb); +} + +static void test_fake_setup(void) +{ + bool first = true; + struct ctdb_context *ctdb = talloc_zero(NULL, struct ctdb_context); + + ctdb_test_stubs_fake_setup(ctdb); + + if (ctdb->nodes != NULL) { + if (!first) { + printf("\n"); + } + printf("NODEMAP\n"); + ctdb_test_stubs_print_nodemap(ctdb); + first = false; + } + + if (ctdb->ifaces != NULL) { + if (!first) { + printf("\n"); + } + printf("IFACES\n"); + ctdb_test_stubs_print_ifaces(ctdb); + first = false; + } + + if (ctdb->vnn_map != NULL) { + if (!first) { + printf("\n"); + } + printf("VNNMAP\n"); + ctdb_test_stubs_print_vnnmap(ctdb); + first = false; + } + + talloc_free(ctdb); +} + +static const char * decode_pnn_mode(uint32_t pnn_mode) +{ + int i; + static const struct { + uint32_t mode; + const char *name; + } pnn_modes[] = { + { CTDB_CURRENT_NODE, "CURRENT_NODE" }, + { CTDB_BROADCAST_ALL, "BROADCAST_ALL" }, + { CTDB_BROADCAST_VNNMAP, "BROADCAST_VNNMAP" }, + { CTDB_BROADCAST_CONNECTED, "BROADCAST_CONNECTED" }, + { CTDB_MULTICAST, "MULTICAST" }, + }; + + for (i = 0; i < ARRAY_SIZE(pnn_modes); i++) { + if (pnn_mode == pnn_modes[i].mode) { + return pnn_modes[i].name; + } + } + + return "PNN"; +} + +static void print_nodes(uint32_t *nodes, uint32_t pnn_mode) +{ + int i; + + printf("NODES:"); + for (i = 0; i < talloc_array_length(nodes); i++) { + printf(" %lu", (unsigned long) nodes[i]); + } + printf("\n"); + + printf("PNN MODE: %s (%lu)\n", + decode_pnn_mode(pnn_mode), (unsigned long) pnn_mode); +} + +static void test_parse_nodestring(const char *nodestring_s, + const char *dd_ok_s) +{ + const char *nodestring; + bool dd_ok; + struct ctdb_context *ctdb; + uint32_t *nodes; + uint32_t pnn_mode; + + nodestring = strcmp("", nodestring_s) == 0 ? NULL : nodestring_s; + + if (strcasecmp(dd_ok_s, "yes") == 0 || + strcmp(dd_ok_s, "true") == 0) { + dd_ok = true; + } else { + dd_ok = false; + } + + ctdb = talloc_zero(NULL, struct ctdb_context); + + ctdb_test_stubs_read_nodemap(ctdb); + + if (parse_nodestring(ctdb, NULL, nodestring, CTDB_CURRENT_NODE, dd_ok, + &nodes, &pnn_mode)) { + print_nodes(nodes, pnn_mode); + } + + talloc_free(ctdb); +} + +static void usage(void) +{ + fprintf(stderr, "usage: ctdb_tool_functest <op>\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + LogLevel = DEBUG_DEBUG; + if (getenv("CTDB_TEST_LOGLEVEL")) { + LogLevel = atoi(getenv("CTDB_TEST_LOGLEVEL")); + } + + if (argc < 2) { + usage(); + } + + if (argc == 2 && strcmp(argv[1], "read_nodemap") == 0) { + test_read_nodemap(); + } else if (argc == 2 && strcmp(argv[1], "read_ifaces") == 0) { + test_read_ifaces(); + } else if (argc == 2 && strcmp(argv[1], "read_vnnmap") == 0) { + test_read_vnnmap(); + } else if (argc == 2 && strcmp(argv[1], "fake_setup") == 0) { + test_fake_setup(); + } else if (argc == 4 && strcmp(argv[1], "parse_nodestring") == 0) { + test_parse_nodestring(argv[2], argv[3]); + } else { + usage(); + } + + return 0; +} diff --git a/ctdb/tests/src/ctdb_lock_tdb.c b/ctdb/tests/src/ctdb_lock_tdb.c new file mode 100644 index 00000000000..ad2a3297d31 --- /dev/null +++ b/ctdb/tests/src/ctdb_lock_tdb.c @@ -0,0 +1,42 @@ +#include <stdio.h> +#include <fcntl.h> + +#include "includes.h" + +const char *tdb_file; +TDB_CONTEXT *tdb; + +void signal_handler(int signum) +{ + tdb_close(tdb); +} + + +int +main(int argc, char *argv[]) +{ + if (argc != 2) { + printf("Usage: %s <tdb file>\n", argv[0]); + exit(1); + } + + tdb_file = argv[1]; + + tdb = tdb_open(tdb_file, 0, 0, O_RDWR, 0); + if (tdb == NULL) { + fprintf(stderr, "Failed to open TDB file %s\n", tdb_file); + exit(1); + } + + signal(SIGINT, signal_handler); + + if (tdb_lockall(tdb) != 0) { + fprintf(stderr, "Failed to lock database %s\n", tdb_file); + tdb_close(tdb); + exit(1); + } + + sleep(999999); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_persistent.c b/ctdb/tests/src/ctdb_persistent.c new file mode 100644 index 00000000000..0bf92b345e8 --- /dev/null +++ b/ctdb/tests/src/ctdb_persistent.c @@ -0,0 +1,268 @@ +/* + simple tool to test persistent databases + + Copyright (C) Andrew Tridgell 2006-2007 + Copyright (c) Ronnie sahlberg 2007 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + +static int timelimit = 10; + +static unsigned int pnn; + +static TDB_DATA old_data; + +static int success = true; + +static void each_second(struct event_context *ev, struct timed_event *te, + struct timeval t, void *private_data) +{ + struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context); + int i; + uint32_t *old_counters; + + + printf("[%4u] Counters: ", getpid()); + old_counters = (uint32_t *)old_data.dptr; + for (i=0;i<old_data.dsize/sizeof(uint32_t); i++) { + printf("%6u ", old_counters[i]); + } + printf("\n"); + + event_add_timed(ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb); +} + +static void check_counters(struct ctdb_context *ctdb, TDB_DATA data) +{ + int i; + uint32_t *counters, *old_counters; + unsigned char *tmp_dptr; + + counters = (uint32_t *)data.dptr; + old_counters = (uint32_t *)old_data.dptr; + + /* check that all the counters are monotonic increasing */ + for (i=0; i<old_data.dsize/sizeof(uint32_t); i++) { + if (counters[i]<old_counters[i]) { + printf("[%4u] ERROR: counters has decreased for node %u From %u to %u\n", + getpid(), i, old_counters[i], counters[i]); + success = false; + } + } + + if (old_data.dsize != data.dsize) { + old_data.dsize = data.dsize; + tmp_dptr = talloc_realloc_size(ctdb, old_data.dptr, old_data.dsize); + if (tmp_dptr == NULL) { + printf("[%4u] ERROR: talloc_realloc_size failed.\n", getpid()); + success = false; + return; + } else { + old_data.dptr = tmp_dptr; + } + } + + memcpy(old_data.dptr, data.dptr, data.dsize); +} + + + +static void test_store_records(struct ctdb_context *ctdb, struct event_context *ev) +{ + TDB_DATA key; + struct ctdb_db_context *ctdb_db; + + ctdb_db = ctdb_db_handle(ctdb, "persistent.tdb"); + + key.dptr = discard_const("testkey"); + key.dsize = strlen((const char *)key.dptr)+1; + + start_timer(); + while (end_timer() < timelimit) { + TDB_DATA data; + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + struct ctdb_transaction_handle *h; + int ret; + uint32_t *counters; + + h = ctdb_transaction_start(ctdb_db, tmp_ctx); + if (h == NULL) { + printf("Failed to start transaction on node %d\n", + ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + return; + } + + ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data); + if (ret != 0) { + DEBUG(DEBUG_ERR,("Failed to fetch record\n")); + exit(1); + } + + if (data.dsize < sizeof(uint32_t) * (pnn+1)) { + unsigned char *ptr = data.dptr; + + data.dptr = talloc_zero_size(tmp_ctx, sizeof(uint32_t) * (pnn+1)); + memcpy(data.dptr, ptr, data.dsize); + talloc_free(ptr); + + data.dsize = sizeof(uint32_t) * (pnn+1); + } + + if (data.dptr == NULL) { + printf("Failed to realloc array\n"); + talloc_free(tmp_ctx); + return; + } + + counters = (uint32_t *)data.dptr; + + /* bump our counter */ + counters[pnn]++; + + ret = ctdb_transaction_store(h, key, data); + if (ret != 0) { + DEBUG(DEBUG_ERR,("Failed to store record\n")); + exit(1); + } + + ret = ctdb_transaction_commit(h); + if (ret != 0) { + DEBUG(DEBUG_ERR,("Failed to commit transaction\n")); + //exit(1); + } + + /* store the counters and verify that they are sane */ + if (pnn == 0) { + check_counters(ctdb, data); + } + + talloc_free(tmp_ctx); + } + +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + int unsafe_writes = 0; + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" }, + { "unsafe-writes", 'u', POPT_ARG_NONE, &unsafe_writes, 0, "do not use tdb transactions when writing", NULL }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + setlinebuf(stdout); + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + if (ctdb == NULL) { + printf("Could not attach to daemon\n"); + return 1; + } + + /* attach to a specific database */ + if (unsafe_writes == 1) { + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), + "persistent.tdb", true, TDB_NOSYNC); + } else { + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), + "persistent.tdb", true, 0); + } + + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + pnn = ctdb_get_pnn(ctdb); + printf("Starting test on node %u. running for %u seconds\n", pnn, timelimit); + + if (pnn == 0) { + event_add_timed(ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb); + } + + test_store_records(ctdb, ev); + + if (pnn == 0) { + if (success != true) { + printf("The test FAILED\n"); + return 1; + } else { + printf("SUCCESS!\n"); + } + } + return 0; +} diff --git a/ctdb/tests/src/ctdb_porting_tests.c b/ctdb/tests/src/ctdb_porting_tests.c new file mode 100644 index 00000000000..0c434518def --- /dev/null +++ b/ctdb/tests/src/ctdb_porting_tests.c @@ -0,0 +1,305 @@ +/* + Test porting lib (common/system_*.c) + + Copyright (C) Mathieu Parent 2013 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "include/ctdb_private.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +static struct { + const char *socketname; + const char *debuglevel; + pid_t helper_pid; + int socket; + int successcount; + int testcount; +} globals = { + .socketname = "/tmp/test.sock" +}; + + + +/* + Socket functions +*/ +/* + create a unix domain socket and bind it + return a file descriptor open on the socket +*/ +static int socket_server_create(void) +{ + struct sockaddr_un addr; + + globals.socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (globals.socket == -1) { + DEBUG(DEBUG_CRIT,("Unable to create server socket: %s\n", strerror(errno))); + return -1; + } + + set_close_on_exec(globals.socket); + //set_nonblocking(globals.socket); + + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, globals.socketname, sizeof(addr.sun_path)); + + if (bind(globals.socket, (struct sockaddr *)&addr, sizeof(addr)) == -1) { + DEBUG(DEBUG_CRIT,("Unable to bind on socket '%s': %s\n", globals.socketname, strerror(errno))); + goto failed; + } + + if (chown(globals.socketname, geteuid(), getegid()) != 0 || + chmod(globals.socketname, 0700) != 0) { + DEBUG(DEBUG_CRIT,("Unable to secure socket '%s': %s\n", globals.socketname, strerror(errno))); + goto failed; + } + + + if (listen(globals.socket, 100) != 0) { + DEBUG(DEBUG_CRIT,("Unable to listen on socket '%s': %s\n", globals.socketname, strerror(errno))); + goto failed; + } + return 0; + +failed: + close(globals.socket); + globals.socket = -1; + return -1; +} + +static int socket_server_wait_peer(void) +{ + struct sockaddr_un addr; + socklen_t len; + int fd; + + memset(&addr, 0, sizeof(addr)); + len = sizeof(addr); + fd = accept(globals.socket, (struct sockaddr *)&addr, &len); + if (fd == -1) { + DEBUG(DEBUG_CRIT,("Unable to accept on ctdb socket '%s': %s\n", globals.socketname, strerror(errno))); + return -1; + } + + //set_nonblocking(fd); + set_close_on_exec(fd); + return fd; +} + +static int socket_server_close(void) +{ + if (close(globals.socket) == -1) { + DEBUG(DEBUG_CRIT,("Unable to close server socket: %s\n", strerror(errno))); + return -1; + } + if (unlink(globals.socketname) == -1) { + DEBUG(DEBUG_CRIT,("Unable to remove server socket: %s\n", strerror(errno))); + return -1; + } + return 0; +} + +static int socket_client_connect(void) +{ + struct sockaddr_un addr; + int client = 0; + + client = socket(AF_UNIX, SOCK_STREAM, 0); + if (client == -1) { + DEBUG(DEBUG_CRIT,("Unable to create client socket: %s\n", strerror(errno))); + return -1; + } + + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, globals.socketname, sizeof(addr.sun_path)); + if (connect(client, (struct sockaddr *)&addr, sizeof(addr))==-1) { + DEBUG(DEBUG_CRIT,("Unable to connect to '%s': %s\n", globals.socketname, strerror(errno))); + close(client); + return -1; + } + + return client; +} + +static int socket_client_write(int client) +{ + if (write(client, "\0", 1) == -1) { + DEBUG(DEBUG_CRIT,("Unable to write to client socket: %s\n", strerror(errno))); + return -1; + } + return 0; +} + +static int socket_client_close(int client) +{ + if (close(client) == -1) { + DEBUG(DEBUG_CRIT,("Unable to close client socket: %s\n", strerror(errno))); + return -1; + } + return 0; +} + +/* + forked program +*/ +static int fork_helper(void) +{ + pid_t pid; + int i, client, max_rounds = 10; + + pid = fork(); + if (pid == -1) { + DEBUG(DEBUG_CRIT,("Unable to fork: %s\n", strerror(errno))); + return -1; + } + if (pid == 0) { // Child + client = socket_client_connect(); + socket_client_write(client); + for (i = 1 ; i <= max_rounds ; i++ ) { + DEBUG(DEBUG_DEBUG,("Child process waiting ( %d/%d)\n", i, max_rounds)); + sleep(1); + } + socket_client_close(client); + exit(0); + } else { + globals.helper_pid = pid; + } + return 0; +} + +/* + tests +*/ +int test_ctdb_sys_check_iface_exists(void) +{ + const char *fakename; + bool test; + globals.testcount++; + fakename = strdup("fake"); + if (fakename == NULL) { + DEBUG(DEBUG_CRIT,("Unable to allocate memory\n")); + return -1; + } + test = ctdb_sys_check_iface_exists(fakename); + if(test == true) { + DEBUG(DEBUG_CRIT,("Test failed: Fake interface detected: %s\n", fakename)); + return -1; + } + DEBUG(DEBUG_INFO,("Test OK: Fake interface not detected: %s\n", fakename)); + globals.successcount++; + return 0; +} + +int test_ctdb_get_peer_pid(void) +{ + int ret; + int fd; + pid_t peer_pid = 0; + globals.testcount++; + fd = socket_server_wait_peer(); + ret = ctdb_get_peer_pid(fd, &peer_pid); + if (ret == -1) { + DEBUG(DEBUG_CRIT,("Test failed: Unable to get peer process id\n")); + return -1; + } + if (peer_pid <= 0) { + DEBUG(DEBUG_CRIT,("Test failed: Invalid peer process id: %d\n", peer_pid)); + return -1; + } + DEBUG(DEBUG_INFO,("Test OK: Peer process id: %d\n", peer_pid)); + globals.successcount++; + return 0; +} + +int test_ctdb_get_process_name(void) +{ + char *process_name = NULL; + globals.testcount++; + process_name = ctdb_get_process_name(globals.helper_pid); + if ((process_name == NULL) || !strcmp(process_name, "unknown")) { + DEBUG(DEBUG_CRIT,("Test failed: Invalid process name of %d: %s\n", globals.helper_pid, process_name)); + return -1; + } + DEBUG(DEBUG_INFO,("Test OK: Name of PID=%d: %s\n", globals.helper_pid, process_name)); + globals.successcount++; + return 0; +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct poptOption popt_options[] = { + POPT_AUTOHELP + { "socket", 0, POPT_ARG_STRING, &globals.socketname, 0, "local socket name", "filename" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + + LogLevel = DEBUG_INFO; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + if (globals.socketname == NULL) { + DEBUG(DEBUG_CRIT,("Socket name is undefined\n")); + exit(1); + } + if (socket_server_create()) { + DEBUG(DEBUG_CRIT,("Socket error: exiting\n")); + exit(1); + } + if (fork_helper()) { + DEBUG(DEBUG_CRIT,("Forking error: exiting\n")); + exit(1); + } + /* FIXME: Test tcp_checksum6, tcp_checksum */ + /* FIXME: Test ctdb_sys_send_arp, ctdb_sys_send_tcp */ + /* FIXME: Test ctdb_sys_{open,close}_capture_socket, ctdb_sys_read_tcp_packet */ + test_ctdb_sys_check_iface_exists(); + test_ctdb_get_peer_pid(); + test_ctdb_get_process_name(); + /* FIXME: Test ctdb_get_lock_info, ctdb_get_blocker_pid*/ + + socket_server_close(); + + DEBUG(DEBUG_INFO,("%d/%d tests successfull\n", globals.successcount, globals.testcount)); + return 0; +} diff --git a/ctdb/tests/src/ctdb_randrec.c b/ctdb/tests/src/ctdb_randrec.c new file mode 100644 index 00000000000..60d233bed81 --- /dev/null +++ b/ctdb/tests/src/ctdb_randrec.c @@ -0,0 +1,201 @@ +/* + create a lot of random records, both current records and deleted records + + Copyright (C) Andrew Tridgell 2008 + Ronnie sahlberg 2007 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" +#include "ctdb_private.h" + +#include <sys/time.h> +#include <time.h> + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + +static int num_records = 10; +static int delete_pct = 75; +static int base_rec; + +static void store_records(struct ctdb_context *ctdb, struct event_context *ev) +{ + TDB_DATA key, data; + struct ctdb_db_context *ctdb_db; + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + int ret; + struct ctdb_record_handle *h; + uint32_t i=0; + + ctdb_db = ctdb_db_handle(ctdb, "test.tdb"); + + srandom(time(NULL) ^ getpid()); + + start_timer(); + + printf("working with %d records\n", num_records); + while (1) { + unsigned r = random() % num_records; + key.dptr = (uint8_t *)&r; + key.dsize = sizeof(r); + + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + return; + } + + if (random() % 100 < delete_pct) { + data.dptr = NULL; + data.dsize = 0; + } else { + data.dptr = talloc_zero_size(h, data.dsize + sizeof(r)); + data.dsize += sizeof(r); + } + + ret = ctdb_record_store(h, data); + if (ret != 0) { + printf("Failed to store record\n"); + } + + if (data.dptr == NULL && data.dsize == 0) { + struct ctdb_control_schedule_for_deletion *dd; + TDB_DATA indata; + int32_t status; + + indata.dsize = offsetof(struct ctdb_control_schedule_for_deletion, key) + key.dsize; + indata.dptr = talloc_zero_array(ctdb, uint8_t, indata.dsize); + if (indata.dptr == NULL) { + printf("out of memory\n"); + exit(1); + } + dd = (struct ctdb_control_schedule_for_deletion *)(void *)indata.dptr; + dd->db_id = ctdb_db->db_id; + dd->hdr = *ctdb_header_from_record_handle(h); + dd->keylen = key.dsize; + memcpy(dd->key, key.dptr, key.dsize); + + ret = ctdb_control(ctdb, + CTDB_CURRENT_NODE, + ctdb_db->db_id, + CTDB_CONTROL_SCHEDULE_FOR_DELETION, + 0, /* flags */ + indata, + NULL, /* mem_ctx */ + NULL, /* outdata */ + &status, + NULL, /* timeout : NULL == wait forever */ + NULL); /* error message */ + + talloc_free(indata.dptr); + + if (ret != 0 || status != 0) { + DEBUG(DEBUG_ERR, (__location__ " Error sending " + "SCHEDULE_FOR_DELETION " + "control.\n")); + } + } + + talloc_free(h); + + if (i % 1000 == 0) { + printf("%7.0f recs/second %u total\r", 1000.0 / end_timer(), i); + fflush(stdout); + start_timer(); + } + i++; + } + + talloc_free(tmp_ctx); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + { "base-rec", 'b', POPT_ARG_INT, &base_rec, 0, "base_rec", "integer" }, + { "delete-pct", 'p', POPT_ARG_INT, &delete_pct, 0, "delete_pct", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + + if (ctdb == NULL) { + printf("failed to connect to daemon\n"); + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb", + false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + store_records(ctdb, ev); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_store.c b/ctdb/tests/src/ctdb_store.c new file mode 100644 index 00000000000..69203434d6e --- /dev/null +++ b/ctdb/tests/src/ctdb_store.c @@ -0,0 +1,163 @@ +/* + simple tool to create a lot of records on a tdb and to read them out + + Copyright (C) Andrew Tridgell 2006 + Ronnie sahlberg 2007 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> + +static int num_records = 10; +static int base_rec; + +static void store_records(struct ctdb_context *ctdb, struct event_context *ev) +{ + TDB_DATA key, data; + struct ctdb_db_context *ctdb_db; + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + int ret; + struct ctdb_record_handle *h; + uint32_t i; + + ctdb_db = ctdb_db_handle(ctdb, "test.tdb"); + + printf("creating %d records\n", num_records); + for (i=0;i<num_records;i++) { + int r = base_rec + i; + key.dptr = (uint8_t *)&r; + key.dsize = sizeof(uint32_t); + + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + return; + } + + data.dptr = (uint8_t *)&i; + data.dsize = sizeof(uint32_t); + + ret = ctdb_record_store(h, data); + talloc_free(h); + if (ret != 0) { + printf("Failed to store record\n"); + } + if (i % 1000 == 0) { + printf("%u\r", i); + fflush(stdout); + } + } + + printf("fetching all %d records\n", num_records); + while (1) { + for (i=0;i<num_records;i++) { + int r = base_rec + i; + key.dptr = (uint8_t *)&r; + key.dsize = sizeof(uint32_t); + + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + return; + } + talloc_free(h); + } + sleep(1); + printf("."); + fflush(stdout); + } + + talloc_free(tmp_ctx); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + { "base-rec", 'b', POPT_ARG_INT, &base_rec, 0, "base_rec", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* talloc_enable_leak_report_full(); */ + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + + if (ctdb == NULL) { + printf("failed to connect to ctdb daemon.\n"); + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb", false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + store_records(ctdb, ev); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_takeover_tests.c b/ctdb/tests/src/ctdb_takeover_tests.c new file mode 100644 index 00000000000..7fd989eaf6e --- /dev/null +++ b/ctdb/tests/src/ctdb_takeover_tests.c @@ -0,0 +1,637 @@ +/* + Tests for ctdb_takeover.c + + Copyright (C) Martin Schwenke 2011 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "ctdbd_test.c" + +/* This is lazy... but it is test code! */ +#define CTDB_TEST_MAX_NODES 256 +#define CTDB_TEST_MAX_IPS 1024 + +/* Format of each line is "IP pnn" - the separator has to be at least + * 1 space (not a tab or whatever - a space!). + */ +static struct ctdb_public_ip_list * +read_ctdb_public_ip_list(TALLOC_CTX *ctx) +{ + char line[1024]; + ctdb_sock_addr addr; + char *t; + int pnn; + struct ctdb_public_ip_list *last = NULL; + + struct ctdb_public_ip_list *ret = NULL; + + while (fgets(line, sizeof(line), stdin) != NULL) { + + if ((t = strchr(line, ' ')) != NULL) { + /* Make line contain just the address */ + *t = '\0'; + /* Point to PNN or leading whitespace... */ + t++; + pnn = (int) strtol(t, (char **) NULL, 10); + } else { + /* Assume just an IP address, default to PNN -1 */ + if ((t = strchr(line, '\n')) != NULL) { + *t = '\0'; + } + pnn = -1; + } + + if (parse_ip(line, NULL, 0, &addr)) { + if (last == NULL) { + last = talloc(ctx, struct ctdb_public_ip_list); + } else { + last->next = talloc(ctx, struct ctdb_public_ip_list); + last = last->next; + } + last->next = NULL; + last->pnn = pnn; + memcpy(&(last->addr), &addr, sizeof(addr)); + if (ret == NULL) { + ret = last; + } + } else { + DEBUG(DEBUG_ERR, (__location__ " ERROR, bad address :%s\n", line)); + } + } + + return ret; +} + +void print_ctdb_public_ip_list(struct ctdb_public_ip_list * ips) +{ + while (ips) { + printf("%s %d\n", ctdb_addr_to_str(&(ips->addr)), ips->pnn); + ips = ips->next; + } +} + +/* Read some IPs from stdin, 1 per line, parse them and then print + * them back out. */ +void ctdb_test_read_ctdb_public_ip_list(void) +{ + struct ctdb_public_ip_list *l; + + TALLOC_CTX *tmp_ctx = talloc_new(NULL); + + l = read_ctdb_public_ip_list(tmp_ctx); + + print_ctdb_public_ip_list(l); + + talloc_free(tmp_ctx); +} + +/* Format of each line is "IP CURRENT_PNN ALLOWED_PNN,...". + */ +static bool +read_ctdb_public_ip_info(TALLOC_CTX *ctx, + int numnodes, + struct ctdb_public_ip_list ** all_ips, + struct ctdb_all_public_ips *** avail) +{ + char line[1024]; + ctdb_sock_addr addr; + char *t, *tok; + struct ctdb_public_ip_list * ta; + int pnn, numips, curr, n, i; + struct ctdb_all_public_ips * a; + + struct ctdb_public_ip_list *last = NULL; + + *avail = talloc_array_size(ctx, sizeof(struct ctdb_all_public_ips *), CTDB_TEST_MAX_NODES); + memset(*avail, 0, + sizeof(struct ctdb_all_public_ips *) * CTDB_TEST_MAX_NODES); + + numips = 0; + *all_ips = NULL; + while (fgets(line, sizeof(line), stdin) != NULL) { + + /* Get rid of pesky newline */ + if ((t = strchr(line, '\n')) != NULL) { + *t = '\0'; + } + + /* Exit on an empty line */ + if (line[0] == '\0') { + break; + } + + /* Get the IP address */ + tok = strtok(line, " \t"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line ignored :%s\n", line)); + continue; + } + + if (!parse_ip(tok, NULL, 0, &addr)) { + DEBUG(DEBUG_ERR, (__location__ " ERROR, bad address :%s\n", tok)); + continue; + } + + numips++; + if (numips > CTDB_TEST_MAX_IPS) { + DEBUG(DEBUG_ERR, ("ERROR: Exceeding CTDB_TEST_MAX_IPS: %d\n", CTDB_TEST_MAX_IPS)); + exit(1); + } + + /* Get the PNN */ + pnn = -1; + tok = strtok(NULL, " \t"); + if (tok != NULL) { + pnn = (int) strtol(tok, (char **) NULL, 10); + } + + /* Add address + pnn to all_ips */ + if (last == NULL) { + last = talloc(ctx, struct ctdb_public_ip_list); + } else { + last->next = talloc(ctx, struct ctdb_public_ip_list); + last = last->next; + } + last->next = NULL; + last->pnn = pnn; + memcpy(&(last->addr), &addr, sizeof(addr)); + if (*all_ips == NULL) { + *all_ips = last; + } + + tok = strtok(NULL, " \t#"); + if (tok == NULL) { + continue; + } + + /* Handle allowed nodes for addr */ + t = strtok(tok, ","); + while (t != NULL) { + n = (int) strtol(t, (char **) NULL, 10); + if ((*avail)[n] == NULL) { + (*avail)[n] = talloc_array(ctx, struct ctdb_all_public_ips, CTDB_TEST_MAX_IPS); + (*avail)[n]->num = 0; + } + curr = (*avail)[n]->num; + (*avail)[n]->ips[curr].pnn = pnn; + memcpy(&((*avail)[n]->ips[curr].addr), + &addr, sizeof(addr)); + (*avail)[n]->num++; + t = strtok(NULL, ","); + } + + } + + /* Build list of all allowed IPs */ + a = talloc_array(ctx, struct ctdb_all_public_ips, CTDB_TEST_MAX_IPS); + a->num = numips; + for (ta = *all_ips, i=0; ta != NULL && i < numips ; ta = ta->next, i++) { + a->ips[i].pnn = ta->pnn; + memcpy(&(a->ips[i].addr), &(ta->addr), sizeof(ta->addr)); + } + + /* Assign it to any nodes that don't have a list assigned */ + for (n = 0; n < numnodes; n++) { + if ((*avail)[n] == NULL) { + (*avail)[n] = a; + } + } + + return true; +} + +void print_ctdb_available_ips(int numnodes, struct ctdb_all_public_ips **avail) +{ + int n, i; + + for (n = 0; n < numnodes; n++) { + if ((avail[n] != NULL) && (avail[n]->num > 0)) { + printf("%d:", n); + for (i = 0; i < avail[n]->num; i++) { + printf("%s%s", + (i == 0) ? " " : ", ", + ctdb_addr_to_str(&(avail[n]->ips[i].addr))); + } + printf("\n"); + } + } +} + +void ctdb_test_read_ctdb_public_ip_info(const char nodestates[]) +{ + int numnodes; + struct ctdb_public_ip_list *l; + struct ctdb_all_public_ips **avail; + char *tok, *ns; + + TALLOC_CTX *tmp_ctx = talloc_new(NULL); + + /* Avoid that const */ + ns = talloc_strdup(tmp_ctx, nodestates); + + numnodes = 0; + tok = strtok(ns, ","); + while (tok != NULL) { + numnodes++; + if (numnodes > CTDB_TEST_MAX_NODES) { + DEBUG(DEBUG_ERR, ("ERROR: Exceeding CTDB_TEST_MAX_NODES: %d\n", CTDB_TEST_MAX_NODES)); + exit(1); + } + tok = strtok(NULL, ","); + } + + read_ctdb_public_ip_info(tmp_ctx, numnodes, &l, &avail); + + print_ctdb_public_ip_list(l); + print_ctdb_available_ips(numnodes, avail); + + talloc_free(tmp_ctx); +} + +/* Read 2 IPs from stdin, calculate the IP distance and print it. */ +void ctdb_test_ip_distance(void) +{ + struct ctdb_public_ip_list *l; + uint32_t distance; + + TALLOC_CTX *tmp_ctx = talloc_new(NULL); + + l = read_ctdb_public_ip_list(tmp_ctx); + + if (l && l->next) { + distance = ip_distance(&(l->addr), &(l->next->addr)); + printf ("%lu\n", (unsigned long) distance); + } + + talloc_free(tmp_ctx); +} + +/* Read some IPs from stdin, calculate the sum of the squares of the + * IP distances between the 1st argument and those read that are on + * the given node. The given IP must one of the ones in the list. */ +void ctdb_test_ip_distance_2_sum(const char ip[], int pnn) +{ + struct ctdb_public_ip_list *l; + struct ctdb_public_ip_list *t; + ctdb_sock_addr addr; + uint32_t distance; + + TALLOC_CTX *tmp_ctx = talloc_new(NULL); + + + l = read_ctdb_public_ip_list(tmp_ctx); + + if (l && parse_ip(ip, NULL, 0, &addr)) { + /* find the entry for the specified IP */ + for (t=l; t!=NULL; t=t->next) { + if (ctdb_same_ip(&(t->addr), &addr)) { + break; + } + } + + if (t == NULL) { + fprintf(stderr, "IP NOT PRESENT IN LIST"); + exit(1); + } + + distance = ip_distance_2_sum(&(t->addr), l, pnn); + printf ("%lu\n", (unsigned long) distance); + } else { + fprintf(stderr, "BAD INPUT"); + exit(1); + } + + talloc_free(tmp_ctx); +} + +/* Read some IPs from stdin, calculate the sume of the squares of the + * IP distances between the first and the rest, and print it. */ +void ctdb_test_lcp2_imbalance(int pnn) +{ + struct ctdb_public_ip_list *l; + uint32_t imbalance; + + TALLOC_CTX *tmp_ctx = talloc_new(NULL); + + l = read_ctdb_public_ip_list(tmp_ctx); + + imbalance = lcp2_imbalance(l, pnn); + printf ("%lu\n", (unsigned long) imbalance); + + talloc_free(tmp_ctx); +} + +static uint32_t *get_tunable_values(TALLOC_CTX *tmp_ctx, + int numnodes, + const char *tunable) +{ + int i; + char *tok; + uint32_t *tvals = talloc_zero_array(tmp_ctx, uint32_t, numnodes); + char *t = getenv(tunable); + + if (t) { + if (strcmp(t, "1") == 0) { + for (i=0; i<numnodes; i++) { + tvals[i] = 1; + } + } else { + tok = strtok(t, ","); + i = 0; + while (tok != NULL) { + tvals[i] = + (uint32_t) strtol(tok, NULL, 0); + i++; + tok = strtok(NULL, ","); + } + if (i != numnodes) { + fprintf(stderr, "ERROR: Wrong number of values in %s\n", tunable); + exit(1); + } + } + } + + return tvals; +} + +static enum ctdb_runstate *get_runstate(TALLOC_CTX *tmp_ctx, + int numnodes) +{ + int i; + uint32_t *tvals; + enum ctdb_runstate *runstate = + talloc_zero_array(tmp_ctx, enum ctdb_runstate, numnodes); + char *t = getenv("CTDB_TEST_RUNSTATE"); + + if (t == NULL) { + for (i=0; i<numnodes; i++) { + runstate[i] = CTDB_RUNSTATE_RUNNING; + } + } else { + tvals = get_tunable_values(tmp_ctx, numnodes, "CTDB_TEST_RUNSTATE"); + for (i=0; i<numnodes; i++) { + runstate[i] = (enum ctdb_runstate) tvals[i]; + } + talloc_free(tvals); + } + + return runstate; +} + +/* Fake up enough CTDB state to be able to run the IP allocation + * algorithm. Usually this sets up some standard state, sets the node + * states from the command-line and reads the current IP layout from + * stdin. + * + * However, if read_ips_for_multiple_nodes is true then each node's + * idea of the IP layout is read separately from stdin. In this mode + * is doesn't make much sense to use read_ctdb_public_ip_info's + * optional ALLOWED_PNN,... list in the input, since each node is + * being handled separately anyway. IPs for each node are separated + * by a blank line. This mode is for testing weird behaviours where + * the IP layouts differs across nodes and we want to improve + * create_merged_ip_list(), so should only be used in tests of + * ctdb_takeover_run_core(). Yes, it is a hack... :-) + */ +void ctdb_test_init(const char nodestates[], + struct ctdb_context **ctdb, + struct ctdb_public_ip_list **all_ips, + struct ctdb_ipflags **ipflags, + bool read_ips_for_multiple_nodes) +{ + struct ctdb_all_public_ips **avail; + int i, numnodes; + uint32_t nodeflags[CTDB_TEST_MAX_NODES]; + char *tok, *ns, *t; + struct ctdb_node_map *nodemap; + uint32_t *tval_noiptakeover; + uint32_t *tval_noiptakeoverondisabled; + enum ctdb_runstate *runstate; + + *ctdb = talloc_zero(NULL, struct ctdb_context); + + /* Avoid that const */ + ns = talloc_strdup(*ctdb, nodestates); + + numnodes = 0; + tok = strtok(ns, ","); + while (tok != NULL) { + nodeflags[numnodes] = (uint32_t) strtol(tok, NULL, 0); + numnodes++; + if (numnodes > CTDB_TEST_MAX_NODES) { + DEBUG(DEBUG_ERR, ("ERROR: Exceeding CTDB_TEST_MAX_NODES: %d\n", CTDB_TEST_MAX_NODES)); + exit(1); + } + tok = strtok(NULL, ","); + } + + /* Fake things up... */ + (*ctdb)->num_nodes = numnodes; + + /* Default to LCP2 */ + (*ctdb)->tunable.lcp2_public_ip_assignment = 1; + (*ctdb)->tunable.deterministic_public_ips = 0; + (*ctdb)->tunable.disable_ip_failover = 0; + (*ctdb)->tunable.no_ip_failback = 0; + + if ((t = getenv("CTDB_IP_ALGORITHM"))) { + if (strcmp(t, "lcp2") == 0) { + (*ctdb)->tunable.lcp2_public_ip_assignment = 1; + } else if (strcmp(t, "nondet") == 0) { + (*ctdb)->tunable.lcp2_public_ip_assignment = 0; + } else if (strcmp(t, "det") == 0) { + (*ctdb)->tunable.lcp2_public_ip_assignment = 0; + (*ctdb)->tunable.deterministic_public_ips = 1; + } else { + fprintf(stderr, "ERROR: unknown IP algorithm %s\n", t); + exit(1); + } + } + + tval_noiptakeover = get_tunable_values(*ctdb, numnodes, + "CTDB_SET_NoIPTakeover"); + tval_noiptakeoverondisabled = + get_tunable_values(*ctdb, numnodes, + "CTDB_SET_NoIPHostOnAllDisabled"); + + runstate = get_runstate(*ctdb, numnodes); + + nodemap = talloc_array(*ctdb, struct ctdb_node_map, numnodes); + nodemap->num = numnodes; + + if (!read_ips_for_multiple_nodes) { + read_ctdb_public_ip_info(*ctdb, numnodes, all_ips, &avail); + } + + (*ctdb)->nodes = talloc_array(*ctdb, struct ctdb_node *, numnodes); // FIXME: bogus size, overkill + + for (i=0; i < numnodes; i++) { + nodemap->nodes[i].pnn = i; + nodemap->nodes[i].flags = nodeflags[i]; + /* nodemap->nodes[i].sockaddr is uninitialised */ + + if (read_ips_for_multiple_nodes) { + read_ctdb_public_ip_info(*ctdb, numnodes, + all_ips, &avail); + } + + (*ctdb)->nodes[i] = talloc(*ctdb, struct ctdb_node); + (*ctdb)->nodes[i]->pnn = i; + (*ctdb)->nodes[i]->flags = nodeflags[i]; + (*ctdb)->nodes[i]->available_public_ips = avail[i]; + (*ctdb)->nodes[i]->known_public_ips = avail[i]; + } + + *ipflags = set_ipflags_internal(*ctdb, *ctdb, nodemap, + tval_noiptakeover, + tval_noiptakeoverondisabled, + runstate); +} + +/* IP layout is read from stdin. */ +void ctdb_test_lcp2_allocate_unassigned(const char nodestates[]) +{ + struct ctdb_context *ctdb; + struct ctdb_public_ip_list *all_ips; + struct ctdb_ipflags *ipflags; + + uint32_t *lcp2_imbalances; + bool *newly_healthy; + + ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false); + + lcp2_init(ctdb, ipflags, all_ips, NULL, + &lcp2_imbalances, &newly_healthy); + + lcp2_allocate_unassigned(ctdb, ipflags, + all_ips, lcp2_imbalances); + + print_ctdb_public_ip_list(all_ips); + + talloc_free(ctdb); +} + +/* IP layout is read from stdin. */ +void ctdb_test_lcp2_failback(const char nodestates[]) +{ + struct ctdb_context *ctdb; + struct ctdb_public_ip_list *all_ips; + struct ctdb_ipflags *ipflags; + + uint32_t *lcp2_imbalances; + bool *newly_healthy; + + ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false); + + lcp2_init(ctdb, ipflags, all_ips, NULL, + &lcp2_imbalances, &newly_healthy); + + lcp2_failback(ctdb, ipflags, + all_ips, lcp2_imbalances, newly_healthy); + + print_ctdb_public_ip_list(all_ips); + + talloc_free(ctdb); +} + +/* IP layout is read from stdin. */ +void ctdb_test_lcp2_failback_loop(const char nodestates[]) +{ + struct ctdb_context *ctdb; + struct ctdb_public_ip_list *all_ips; + struct ctdb_ipflags *ipflags; + + uint32_t *lcp2_imbalances; + bool *newly_healthy; + + ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, false); + + lcp2_init(ctdb, ipflags, all_ips, NULL, + &lcp2_imbalances, &newly_healthy); + + lcp2_failback(ctdb, ipflags, + all_ips, lcp2_imbalances, newly_healthy); + + print_ctdb_public_ip_list(all_ips); + + talloc_free(ctdb); +} + +/* IP layout is read from stdin. See comment for ctdb_test_init() for + * explanation of read_ips_for_multiple_nodes. + */ +void ctdb_test_ctdb_takeover_run_core(const char nodestates[], + bool read_ips_for_multiple_nodes) +{ + struct ctdb_context *ctdb; + struct ctdb_public_ip_list *all_ips; + struct ctdb_ipflags *ipflags; + + ctdb_test_init(nodestates, &ctdb, &all_ips, &ipflags, + read_ips_for_multiple_nodes); + + ctdb_takeover_run_core(ctdb, ipflags, &all_ips, NULL); + + print_ctdb_public_ip_list(all_ips); + + talloc_free(ctdb); +} + +void usage(void) +{ + fprintf(stderr, "usage: ctdb_takeover_tests <op>\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + LogLevel = DEBUG_DEBUG; + if (getenv("CTDB_TEST_LOGLEVEL")) { + LogLevel = atoi(getenv("CTDB_TEST_LOGLEVEL")); + } + + if (argc < 2) { + usage(); + } + + if (strcmp(argv[1], "ip_list") == 0) { + ctdb_test_read_ctdb_public_ip_list(); + } else if (argc == 3 && strcmp(argv[1], "ip_info") == 0) { + ctdb_test_read_ctdb_public_ip_info(argv[2]); + } else if (strcmp(argv[1], "ip_distance") == 0) { + ctdb_test_ip_distance(); + } else if (argc == 4 && strcmp(argv[1], "ip_distance_2_sum") == 0) { + ctdb_test_ip_distance_2_sum(argv[2], atoi(argv[3])); + } else if (argc >= 3 && strcmp(argv[1], "lcp2_imbalance") == 0) { + ctdb_test_lcp2_imbalance(atoi(argv[2])); + } else if (argc == 3 && strcmp(argv[1], "lcp2_allocate_unassigned") == 0) { + ctdb_test_lcp2_allocate_unassigned(argv[2]); + } else if (argc == 3 && strcmp(argv[1], "lcp2_failback") == 0) { + ctdb_test_lcp2_failback(argv[2]); + } else if (argc == 3 && strcmp(argv[1], "lcp2_failback_loop") == 0) { + ctdb_test_lcp2_failback_loop(argv[2]); + } else if (argc == 3 && + strcmp(argv[1], "ctdb_takeover_run_core") == 0) { + ctdb_test_ctdb_takeover_run_core(argv[2], false); + } else if (argc == 4 && + strcmp(argv[1], "ctdb_takeover_run_core") == 0 && + strcmp(argv[3], "multi") == 0) { + ctdb_test_ctdb_takeover_run_core(argv[2], true); + } else { + usage(); + } + + return 0; +} diff --git a/ctdb/tests/src/ctdb_test.c b/ctdb/tests/src/ctdb_test.c new file mode 100644 index 00000000000..bbb51bd877c --- /dev/null +++ b/ctdb/tests/src/ctdb_test.c @@ -0,0 +1,104 @@ +/* + ctdb test include file + + Copyright (C) Martin Schwenke 2011 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#ifndef _CTDBD_TEST_C +#define _CTDBD_TEST_C + +#ifdef CTDB_TEST_OVERRIDE_MAIN + +/* Define our own main() and usage() functions */ +#define main(argc, argv) main_foobar(argc, argv) +#define usage usage_foobar + +#endif /* CTDB_TEST_USE_MAIN */ + +#define ctdb_cmdline_client(x, y) \ + ctdb_cmdline_client_stub(x, y) +#define ctdb_ctrl_getnodemap(ctdb, timelimit, pnn, tmp_ctx, nodemap) \ + ctdb_ctrl_getnodemap_stub(ctdb, timelimit, pnn, tmp_ctx, nodemap) +#define ctdb_ctrl_get_ifaces(ctdb, timelimit, pnn, tmp_ctx, ifaces) \ + ctdb_ctrl_get_ifaces_stub(ctdb, timelimit, pnn, tmp_ctx, ifaces) +#define ctdb_ctrl_getpnn(ctdb, timelimit, pnn) \ + ctdb_ctrl_getpnn_stub(ctdb, timelimit, pnn) +#define ctdb_ctrl_getrecmode(ctdb, tmp_ctx, timelimit, pnn, recmode) \ + ctdb_ctrl_getrecmode_stub(ctdb, tmp_ctx, timelimit, pnn, recmode) +#define ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, timelimit, pnn, recmaster) \ + ctdb_ctrl_getrecmaster_stub(ctdb, tmp_ctx, timelimit, pnn, recmaster) +#define ctdb_ctrl_getvnnmap(ctdb, timelimit, pnn, tmp_ctx, vnnmap) \ + ctdb_ctrl_getvnnmap_stub(ctdb, timelimit, pnn, tmp_ctx, vnnmap) +#define ctdb_ctrl_getdebseqnum(ctdb, timelimit, pnn, db_id, seqnum) \ + ctdb_ctrl_getvnnmap_stub(ctdb, timelimit, pnn, db_id, seqnum) +#define ctdb_client_check_message_handlers(ctdb, ids, argc, result) \ + ctdb_client_check_message_handlers_stub(ctdb, ids, argc, result) +#define ctdb_ctrl_getcapabilities(ctdb, timeout, destnode, capabilities) \ + ctdb_ctrl_getcapabilities_stub(ctdb, timeout, destnode, capabilities) + +#include "tools/ctdb.c" + +#ifndef CTDB_TEST_USE_MAIN +#undef main +#undef usage +#endif /* CTDB_TEST_USE_MAIN */ + +#undef ctdb_cmdline_client + +#include "common/cmdline.c" + +#undef ctdb_ctrl_getnodemap +#undef ctdb_ctrl_get_ifaces +#undef ctdb_ctrl_getpnn +#undef ctdb_ctrl_getrecmode +#undef ctdb_ctrl_getrecmaster +#undef ctdb_ctrl_getvnnmap +#undef ctdb_ctrl_getdebseqnum +#undef ctdb_client_check_message_handlers +#undef ctdb_ctrl_getcapabilities + +#undef TIMELIMIT +#include "tools/ctdb_vacuum.c" + +/* UTIL_OBJ */ +#include "lib/util/idtree.c" +#include "lib/util/db_wrap.c" +#include "lib/util/strlist.c" +#include "lib/util/util.c" +#include "lib/util/util_time.c" +#include "lib/util/util_file.c" +#include "lib/util/fault.c" +#include "lib/util/substitute.c" +#include "lib/util/signal.c" + +/* CTDB_COMMON_OBJ */ +#include "common/ctdb_io.c" +#include "common/ctdb_util.c" +#include "common/ctdb_ltdb.c" +#include "common/ctdb_message.c" +#include "lib/util/debug.c" +#include "common/rb_tree.c" +#include "common/system_common.c" +#include "common/ctdb_logging.c" +#include "common/ctdb_fork.c" + +/* CTDB_CLIENT_OBJ */ +#include "client/ctdb_client.c" + +/* TEST STUBS */ +#include "ctdb_test_stubs.c" + +#endif /* _CTDBD_TEST_C */ diff --git a/ctdb/tests/src/ctdb_test_stubs.c b/ctdb/tests/src/ctdb_test_stubs.c new file mode 100644 index 00000000000..456b1fdc0ce --- /dev/null +++ b/ctdb/tests/src/ctdb_test_stubs.c @@ -0,0 +1,529 @@ +/* + Test stubs and support functions for some CTDB client functions + + Copyright (C) Martin Schwenke 2011 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +/* Read a nodemap from stdin. Each line looks like: + * <PNN> <FLAGS> [RECMASTER] [CURRENT] + * EOF or a blank line terminates input. + */ +void ctdb_test_stubs_read_nodemap(struct ctdb_context *ctdb) +{ + char line[1024]; + + TALLOC_FREE(ctdb->nodes); + ctdb->pnn = -1; + ctdb->num_nodes = 0; + + ctdb->nodes = NULL; + + while ((fgets(line, sizeof(line), stdin) != NULL) && + (line[0] != '\n')) { + uint32_t pnn, flags; + char *tok, *t; + const char *ip; + ctdb_sock_addr saddr; + + /* Get rid of pesky newline */ + if ((t = strchr(line, '\n')) != NULL) { + *t = '\0'; + } + + /* Get PNN */ + tok = strtok(line, " \t"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line (PNN) ignored \"%s\"\n", line)); + continue; + } + pnn = (uint32_t)strtoul(tok, NULL, 0); + + /* Get IP */ + tok = strtok(NULL, " \t"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line (no IP) ignored \"%s\"\n", line)); + continue; + } + if (!parse_ip(tok, NULL, 0, &saddr)) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line (IP) ignored \"%s\"\n", line)); + continue; + } + ip = talloc_strdup(ctdb, tok); + + /* Get flags */ + tok = strtok(NULL, " \t"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line (flags) ignored \"%s\"\n", line)); + continue; + } + flags = (uint32_t)strtoul(tok, NULL, 0); + + tok = strtok(NULL, " \t"); + while (tok != NULL) { + if (strcmp(tok, "CURRENT") == 0) { + ctdb->pnn = pnn; + } else if (strcmp(tok, "RECMASTER") == 0) { + ctdb->recovery_master = pnn; + } + tok = strtok(NULL, " \t"); + } + + ctdb->nodes = talloc_realloc(ctdb, ctdb->nodes, struct ctdb_node *, ctdb->num_nodes + 1); + if (ctdb->nodes == NULL) { + DEBUG(DEBUG_ERR, ("OOM allocating nodes array\n")); + exit (1); + } + ctdb->nodes[ctdb->num_nodes] = talloc_zero(ctdb, struct ctdb_node); + if (ctdb->nodes[ctdb->num_nodes] == NULL) { + DEBUG(DEBUG_ERR, ("OOM allocating node structure\n")); + exit (1); + } + + ctdb->nodes[ctdb->num_nodes]->ctdb = ctdb; + ctdb->nodes[ctdb->num_nodes]->name = "fakectdb"; + ctdb->nodes[ctdb->num_nodes]->pnn = pnn; + ctdb->nodes[ctdb->num_nodes]->address.address = ip; + ctdb->nodes[ctdb->num_nodes]->address.port = 0; + ctdb->nodes[ctdb->num_nodes]->flags = flags; + ctdb->num_nodes++; + } +} + +void ctdb_test_stubs_print_nodemap(struct ctdb_context *ctdb) +{ + int i; + + for (i = 0; i < ctdb->num_nodes; i++) { + printf("%ld\t0x%lx%s%s\n", + (unsigned long) ctdb->nodes[i]->pnn, + (unsigned long) ctdb->nodes[i]->flags, + ctdb->nodes[i]->pnn == ctdb->pnn ? "\tCURRENT" : "", + ctdb->nodes[i]->pnn == ctdb->recovery_master ? "\tRECMASTER" : ""); + } +} + +/* Read interfaces information. Same format as "ctdb ifaces -Y" + * output: + * :Name:LinkStatus:References: + * :eth2:1:4294967294 + * :eth1:1:4294967292 + */ + +struct ctdb_iface { + struct ctdb_iface *prev, *next; + const char *name; + bool link_up; + uint32_t references; +}; + +void ctdb_test_stubs_read_ifaces(struct ctdb_context *ctdb) +{ + char line[1024]; + struct ctdb_iface *iface; + + while ((fgets(line, sizeof(line), stdin) != NULL) && + (line[0] != '\n')) { + uint16_t link_state; + uint32_t references; + char *tok, *t, *name; + + /* Get rid of pesky newline */ + if ((t = strchr(line, '\n')) != NULL) { + *t = '\0'; + } + + if (strcmp(line, ":Name:LinkStatus:References:") == 0) { + continue; + } + + /* name */ + //tok = strtok(line, ":"); /* Leading colon... */ + tok = strtok(line, ":"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line ignored \"%s\"\n", line)); + continue; + } + name = tok; + + /* link_state */ + tok = strtok(NULL, ":"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line ignored \"%s\"\n", line)); + continue; + } + link_state = (uint16_t)strtoul(tok, NULL, 0); + + /* references... */ + tok = strtok(NULL, ":"); + if (tok == NULL) { + DEBUG(DEBUG_ERR, (__location__ " WARNING, bad line ignored \"%s\"\n", line)); + continue; + } + references = (uint32_t)strtoul(tok, NULL, 0); + + iface = talloc_zero(ctdb, struct ctdb_iface); + + if (iface == NULL) { + DEBUG(DEBUG_ERR, ("OOM allocating iface\n")); + exit (1); + } + + iface->name = talloc_strdup(iface, name); + iface->link_up = link_state; + iface->references = references; + + DLIST_ADD(ctdb->ifaces, iface); + } +} + +void ctdb_test_stubs_print_ifaces(struct ctdb_context *ctdb) +{ + struct ctdb_iface *iface; + + printf(":Name:LinkStatus:References:\n"); + for (iface = ctdb->ifaces; iface != NULL; iface = iface->next) { + printf(":%s:%u:%u:\n", + iface->name, + iface->link_up, + iface->references); + } +} + +/* Read vnn map. + * output: + * <GENERATION> + * <LMASTER0> + * <LMASTER1> + * ... + */ + +/* +struct ctdb_vnn_map { + uint32_t generation; + uint32_t size; + uint32_t *map; +}; +*/ +void ctdb_test_stubs_read_vnnmap(struct ctdb_context *ctdb) +{ + char line[1024]; + + TALLOC_FREE(ctdb->vnn_map); + + ctdb->vnn_map = talloc_zero(ctdb, struct ctdb_vnn_map); + if (ctdb->vnn_map == NULL) { + DEBUG(DEBUG_ERR, ("OOM allocating vnnmap\n")); + exit (1); + } + ctdb->vnn_map->generation = INVALID_GENERATION; + ctdb->vnn_map->size = 0; + ctdb->vnn_map->map = NULL; + + while ((fgets(line, sizeof(line), stdin) != NULL) && + (line[0] != '\n')) { + uint32_t n; + char *t; + + /* Get rid of pesky newline */ + if ((t = strchr(line, '\n')) != NULL) { + *t = '\0'; + } + + n = (uint32_t) strtol(line, NULL, 0); + + /* generation */ + if (ctdb->vnn_map->generation == INVALID_GENERATION) { + ctdb->vnn_map->generation = n; + continue; + } + + ctdb->vnn_map->map = talloc_realloc(ctdb, ctdb->vnn_map->map, uint32_t, ctdb->vnn_map->size + 1); + if (ctdb->vnn_map->map == NULL) { + DEBUG(DEBUG_ERR, ("OOM allocating vnn_map->map\n")); + exit (1); + } + + ctdb->vnn_map->map[ctdb->vnn_map->size] = n; + ctdb->vnn_map->size++; + } +} + +void ctdb_test_stubs_print_vnnmap(struct ctdb_context *ctdb) +{ + int i; + + printf("%d\n", ctdb->vnn_map->generation); + for (i = 0; i < ctdb->vnn_map->size; i++) { + printf("%d\n", ctdb->vnn_map->map[i]); + } +} + +void ctdb_test_stubs_fake_setup(struct ctdb_context *ctdb) +{ + char line[1024]; + + while (fgets(line, sizeof(line), stdin) != NULL) { + char *t; + + /* Get rid of pesky newline */ + if ((t = strchr(line, '\n')) != NULL) { + *t = '\0'; + } + + if (strcmp(line, "NODEMAP") == 0) { + ctdb_test_stubs_read_nodemap(ctdb); + } else if (strcmp(line, "IFACES") == 0) { + ctdb_test_stubs_read_ifaces(ctdb); + } else if (strcmp(line, "VNNMAP") == 0) { + ctdb_test_stubs_read_vnnmap(ctdb); + } else { + printf("Unknown line %s\n", line); + exit(1); + } + } +} + +/* Support... */ +static bool current_node_is_connected (struct ctdb_context *ctdb) +{ + int i; + for (i = 0; i < ctdb->num_nodes; i++) { + if (ctdb->nodes[i]->pnn == ctdb->pnn) { + if (ctdb->nodes[i]->flags & + (NODE_FLAGS_DISCONNECTED | NODE_FLAGS_DELETED)) { + return false; + } else { + return true; + } + } + } + + /* Shouldn't really happen, so fag an error */ + return false; +} + +/* Stubs... */ + +struct ctdb_context *ctdb_cmdline_client_stub(struct tevent_context *ev, + struct timeval req_timeout) +{ + struct ctdb_context *ctdb; + + ctdb = talloc_zero(NULL, struct ctdb_context); + + ctdb_set_socketname(ctdb, "fake"); + + ctdb_test_stubs_fake_setup(ctdb); + + return ctdb; +} + +/* Copied from ctdb_recover.c */ +int +ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata) +{ + uint32_t i, num_nodes; + struct ctdb_node_map *node_map; + + CHECK_CONTROL_DATA_SIZE(0); + + num_nodes = ctdb->num_nodes; + + outdata->dsize = offsetof(struct ctdb_node_map, nodes) + num_nodes*sizeof(struct ctdb_node_and_flags); + outdata->dptr = (unsigned char *)talloc_zero_size(outdata, outdata->dsize); + if (!outdata->dptr) { + DEBUG(DEBUG_ALERT, (__location__ " Failed to allocate nodemap array\n")); + exit(1); + } + + node_map = (struct ctdb_node_map *)outdata->dptr; + node_map->num = num_nodes; + for (i=0; i<num_nodes; i++) { + if (parse_ip(ctdb->nodes[i]->address.address, + NULL, /* TODO: pass in the correct interface here*/ + 0, + &node_map->nodes[i].addr) == 0) + { + DEBUG(DEBUG_ERR, (__location__ " Failed to parse %s into a sockaddr\n", ctdb->nodes[i]->address.address)); + } + + node_map->nodes[i].pnn = ctdb->nodes[i]->pnn; + node_map->nodes[i].flags = ctdb->nodes[i]->flags; + } + + return 0; +} + +int +ctdb_ctrl_getnodemap_stub(struct ctdb_context *ctdb, + struct timeval timeout, uint32_t destnode, + TALLOC_CTX *mem_ctx, + struct ctdb_node_map **nodemap) +{ + int ret; + + TDB_DATA indata; + TDB_DATA *outdata; + + if (!current_node_is_connected(ctdb)) { + return -1; + } + + indata.dsize = 0; + indata.dptr = NULL; + + outdata = talloc_zero(ctdb, TDB_DATA); + + ret = ctdb_control_getnodemap(ctdb, CTDB_CONTROL_GET_NODEMAP, + indata, outdata); + + if (ret == 0) { + *nodemap = (struct ctdb_node_map *) outdata->dptr; + } + + return ret; +} + +int +ctdb_ctrl_getvnnmap_stub(struct ctdb_context *ctdb, + struct timeval timeout, uint32_t destnode, + TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap) +{ + *vnnmap = talloc(ctdb, struct ctdb_vnn_map); + if (*vnnmap == NULL) { + DEBUG(DEBUG_ERR, (__location__ "OOM\n")); + exit (1); + } + (*vnnmap)->map = talloc_array(*vnnmap, uint32_t, ctdb->vnn_map->size); + + (*vnnmap)->generation = ctdb->vnn_map->generation; + (*vnnmap)->size = ctdb->vnn_map->size; + memcpy((*vnnmap)->map, ctdb->vnn_map->map, sizeof(uint32_t) * (*vnnmap)->size); + + return 0; +} + +int +ctdb_ctrl_getrecmode_stub(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, + struct timeval timeout, uint32_t destnode, + uint32_t *recmode) +{ + *recmode = ctdb->recovery_mode; + + return 0; +} + +int +ctdb_ctrl_getrecmaster_stub(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, + struct timeval timeout, uint32_t destnode, + uint32_t *recmaster) +{ + *recmaster = ctdb->recovery_master; + + return 0; +} + +int +ctdb_ctrl_getpnn_stub(struct ctdb_context *ctdb, struct timeval timeout, + uint32_t destnode) +{ + if (!current_node_is_connected(ctdb)) { + return -1; + } + + if (destnode == CTDB_CURRENT_NODE) { + return ctdb->pnn; + } else { + return destnode; + } +} + +/* From ctdb_takeover.c */ +int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb, + struct ctdb_req_control *c, + TDB_DATA *outdata) +{ + int i, num, len; + struct ctdb_control_get_ifaces *ifaces; + struct ctdb_iface *cur; + + /* count how many public ip structures we have */ + num = 0; + for (cur=ctdb->ifaces;cur;cur=cur->next) { + num++; + } + + len = offsetof(struct ctdb_control_get_ifaces, ifaces) + + num*sizeof(struct ctdb_control_iface_info); + ifaces = talloc_zero_size(outdata, len); + CTDB_NO_MEMORY(ctdb, ifaces); + + i = 0; + for (cur=ctdb->ifaces;cur;cur=cur->next) { + strcpy(ifaces->ifaces[i].name, cur->name); + ifaces->ifaces[i].link_state = cur->link_up; + ifaces->ifaces[i].references = cur->references; + i++; + } + ifaces->num = i; + len = offsetof(struct ctdb_control_get_ifaces, ifaces) + + i*sizeof(struct ctdb_control_iface_info); + + outdata->dsize = len; + outdata->dptr = (uint8_t *)ifaces; + + return 0; +} + +int +ctdb_ctrl_get_ifaces_stub(struct ctdb_context *ctdb, + struct timeval timeout, uint32_t destnode, + TALLOC_CTX *mem_ctx, + struct ctdb_control_get_ifaces **ifaces) +{ + TDB_DATA *outdata; + int ret; + + if (!current_node_is_connected(ctdb)) { + return -1; + } + + outdata = talloc(mem_ctx, TDB_DATA); + + ret = ctdb_control_get_ifaces(ctdb, NULL, outdata); + + if (ret == 0) { + *ifaces = (struct ctdb_control_get_ifaces *)outdata->dptr; + } + + return ret; +} + +int ctdb_client_check_message_handlers_stub(struct ctdb_context *ctdb, + uint64_t *ids, uint32_t num, + uint8_t *result) +{ + DEBUG(DEBUG_ERR, (__location__ " NOT IMPLEMENTED\n")); + return -1; +} + +int ctdb_ctrl_getcapabilities_stub(struct ctdb_context *ctdb, + struct timeval timeout, uint32_t destnode, + uint32_t *capabilities) +{ + *capabilities = CTDB_CAP_RECMASTER|CTDB_CAP_LMASTER|CTDB_CAP_NATGW; + return 0; +} diff --git a/ctdb/tests/src/ctdb_trackingdb_test.c b/ctdb/tests/src/ctdb_trackingdb_test.c new file mode 100644 index 00000000000..ee473c01247 --- /dev/null +++ b/ctdb/tests/src/ctdb_trackingdb_test.c @@ -0,0 +1,135 @@ +/* + simple trackingdb test tool + + This program is used to test the funcitons to manipulate and enumerate + the trackingdb records : + ctdb_trackingdb_add_pnn() + ctdb_trackingdb_traverse() + + Copyright (C) Ronnie Sahlberg 2009 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "system/time.h" +#include "popt.h" +#include "cmdline.h" +#include "ctdb_private.h" +#include "db_wrap.h" + +#define MAXINDEX 64 +char indices[MAXINDEX]; + +void vn_cb(struct ctdb_context *ctdb, uint32_t pnn, void *private_data) +{ + char *ind = private_data; + + printf("Callback for node %d\n", pnn); + if (ind[pnn] == 0) { + printf("ERROR, node %d from callback was never added\n", pnn); + exit(10); + } + ind[pnn] = 0; +} + +void verify_nodes(struct ctdb_context *ctdb, TDB_DATA data) +{ + int i; + + printf("Verify the nodes\n"); + ctdb_trackingdb_traverse(ctdb, data, vn_cb, indices); + for(i = 0; i < MAXINDEX; i++) { + if (indices[i] != 0) { + printf("Callback for %d was never invoked\n", i); + exit(0); + } + } +} + + + +void add_node(struct ctdb_context *ctdb, TDB_DATA *data, int pnn) +{ + printf("Add node %d\n", pnn); + if (ctdb_trackingdb_add_pnn(ctdb, data, pnn)) { + printf("Failed to add tracking db data\n"); + exit(10); + } + indices[pnn] = 1; +} + +static void trackdb_test(struct ctdb_context *ctdb) +{ + TDB_DATA data = {NULL,0}; + int i; + + printf("Add 10 nodes\n"); + srandom(time(NULL)); + for(i=0; i<10; i++) { + add_node(ctdb, &data, random()%MAXINDEX); + } + + verify_nodes(ctdb, data); + printf("OK all seems well\n"); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(5, 0)); + if (ctdb == NULL) { + exit(1); + } + + trackdb_test(ctdb); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_transaction.c b/ctdb/tests/src/ctdb_transaction.c new file mode 100644 index 00000000000..78a63f1f126 --- /dev/null +++ b/ctdb/tests/src/ctdb_transaction.c @@ -0,0 +1,300 @@ +/* + simple tool to test persistent databases + + Copyright (C) Andrew Tridgell 2006-2007 + Copyright (c) Ronnie sahlberg 2007 + Copyright (C) Michael Adam 2009 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + +static int timelimit = 10; +static int delay = 0; +static int verbose = 0; + +static unsigned int pnn; + +static TDB_DATA old_data; + +static bool success = false; + +static void print_counters(void) +{ + int i; + uint32_t *old_counters; + + printf("[%4u] Counters: ", getpid()); + old_counters = (uint32_t *)old_data.dptr; + for (i=0;i<old_data.dsize/sizeof(uint32_t); i++) { + printf("%6u ", old_counters[i]); + } + printf("\n"); +} + +static void each_second(struct event_context *ev, struct timed_event *te, + struct timeval t, void *private_data) +{ + struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context); + + print_counters(); + + event_add_timed(ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb); +} + +static void check_counters(struct ctdb_context *ctdb, TDB_DATA data) +{ + int i; + uint32_t *counters, *old_counters; + bool monotonous = true; + + counters = (uint32_t *)data.dptr; + old_counters = (uint32_t *)old_data.dptr; + + /* check that all the counters are monotonic increasing */ + for (i=0; i<old_data.dsize/sizeof(uint32_t); i++) { + if (counters[i]<old_counters[i]) { + printf("[%4u] ERROR: counters has decreased for node %u From %u to %u\n", + getpid(), i, old_counters[i], counters[i]); + monotonous = false; + } + } + + if (old_data.dsize != data.dsize) { + old_data.dsize = data.dsize; + old_data.dptr = talloc_realloc_size(ctdb, old_data.dptr, old_data.dsize); + } + + memcpy(old_data.dptr, data.dptr, data.dsize); + if (verbose) print_counters(); + + success = monotonous; +} + + +static void do_sleep(unsigned int sec) +{ + unsigned int i; + for (i=0; i<sec; i++) { + if (verbose) printf("."); + sleep(1); + } + if (verbose) printf("\n"); +} + +static void test_store_records(struct ctdb_context *ctdb, struct event_context *ev) +{ + TDB_DATA key; + struct ctdb_db_context *ctdb_db; + int ret; + uint32_t *counters; + ctdb_db = ctdb_db_handle(ctdb, "transaction.tdb"); + + key.dptr = discard_const("testkey"); + key.dsize = strlen((const char *)key.dptr)+1; + + start_timer(); + while ((timelimit == 0) || (end_timer() < timelimit)) { + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + TDB_DATA data; + struct ctdb_transaction_handle *h; + + if (verbose) DEBUG(DEBUG_ERR, ("starting transaction\n")); + h = ctdb_transaction_start(ctdb_db, tmp_ctx); + if (h == NULL) { + DEBUG(DEBUG_ERR, ("Failed to start transaction on node %d\n", + ctdb_get_pnn(ctdb))); + talloc_free(tmp_ctx); + return; + } + if (verbose) DEBUG(DEBUG_ERR, ("transaction started\n")); + do_sleep(delay); + + if (verbose) DEBUG(DEBUG_ERR, ("calling transaction_fetch\n")); + ret = ctdb_transaction_fetch(h, tmp_ctx, key, &data); + if (ret != 0) { + DEBUG(DEBUG_ERR,("Failed to fetch record\n")); + exit(1); + } + if (verbose) DEBUG(DEBUG_ERR, ("fetched data ok\n")); + do_sleep(delay); + + if (data.dsize < sizeof(uint32_t) * (pnn+1)) { + unsigned char *ptr = data.dptr; + + data.dptr = talloc_zero_size(tmp_ctx, sizeof(uint32_t) * (pnn+1)); + memcpy(data.dptr, ptr, data.dsize); + talloc_free(ptr); + + data.dsize = sizeof(uint32_t) * (pnn+1); + } + + if (data.dptr == NULL) { + DEBUG(DEBUG_ERR, ("Failed to realloc array\n")); + talloc_free(tmp_ctx); + return; + } + + counters = (uint32_t *)data.dptr; + + /* bump our counter */ + counters[pnn]++; + + if (verbose) DEBUG(DEBUG_ERR, ("calling transaction_store\n")); + ret = ctdb_transaction_store(h, key, data); + if (ret != 0) { + DEBUG(DEBUG_ERR,("Failed to store record\n")); + exit(1); + } + if (verbose) DEBUG(DEBUG_ERR, ("stored data ok\n")); + do_sleep(delay); + + if (verbose) DEBUG(DEBUG_ERR, ("calling transaction_commit\n")); + ret = ctdb_transaction_commit(h); + if (ret != 0) { + DEBUG(DEBUG_ERR,("Failed to commit transaction\n")); + check_counters(ctdb, data); + exit(1); + } + if (verbose) DEBUG(DEBUG_ERR, ("transaction committed\n")); + + /* store the counters and verify that they are sane */ + if (verbose || (pnn == 0)) { + check_counters(ctdb, data); + } + + do_sleep(delay); + + talloc_free(tmp_ctx); + } + +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + int unsafe_writes = 0; + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" }, + { "delay", 'D', POPT_ARG_INT, &delay, 0, "delay (in seconds) between operations", "integer" }, + { "verbose", 'v', POPT_ARG_NONE, &verbose, 0, "switch on verbose mode", NULL }, + { "unsafe-writes", 'u', POPT_ARG_NONE, &unsafe_writes, 0, "do not use tdb transactions when writing", NULL }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + if (verbose) { + setbuf(stdout, (char *)NULL); /* don't buffer */ + } else { + setlinebuf(stdout); + } + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + if (ctdb == NULL) { + DEBUG(DEBUG_ERR, ("Could not attach to daemon\n")); + return 1; + } + + /* attach to a specific database */ + if (unsafe_writes == 1) { + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), + "transaction.tdb", true, TDB_NOSYNC); + } else { + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), + "transaction.tdb", true, 0); + } + + if (!ctdb_db) { + DEBUG(DEBUG_ERR, ("ctdb_attach failed - %s\n", ctdb_errstr(ctdb))); + exit(1); + } + + DEBUG(DEBUG_ERR, ("Waiting for cluster\n")); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + pnn = ctdb_get_pnn(ctdb); + printf("Starting test on node %u. running for %u seconds. sleep delay: %u seconds.\n", pnn, timelimit, delay); + + if (!verbose && (pnn == 0)) { + event_add_timed(ev, ctdb, timeval_current_ofs(1, 0), each_second, ctdb); + } + + test_store_records(ctdb, ev); + + if (verbose || (pnn == 0)) { + if (success != true) { + printf("The test FAILED\n"); + return 1; + } else { + printf("SUCCESS!\n"); + } + } + return 0; +} diff --git a/ctdb/tests/src/ctdb_traverse.c b/ctdb/tests/src/ctdb_traverse.c new file mode 100644 index 00000000000..5b37ed9d724 --- /dev/null +++ b/ctdb/tests/src/ctdb_traverse.c @@ -0,0 +1,116 @@ +/* + simple tool to traverse a ctdb database over and over and over + + Copyright (C) Andrew Tridgell 2006 + Ronnie sahlberg 2007 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> + +static const char *dbname = "test.tdb"; + +static int traverse_callback(struct ctdb_context *ctdb, TDB_DATA key, TDB_DATA data, void *private_data) +{ + uint32_t *count = private_data; + + (*count)++; + return 0; +} + +static void traverse_loop(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, struct event_context *ev) +{ + uint32_t count; + + printf("traversing database\n"); + count = 0; + ctdb_traverse(ctdb_db, traverse_callback, &count); + printf("traversed %d records\n", count); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + struct ctdb_db_context *ctdb_db; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "database", 0, POPT_ARG_STRING, &dbname, 0, "database to traverse", "name" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* talloc_enable_leak_report_full(); */ + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0)); + if (ctdb == NULL) { + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), dbname, false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + while (1) { + traverse_loop(ctdb, ctdb_db, ev); + } + + return 0; +} diff --git a/ctdb/tests/src/ctdb_update_record.c b/ctdb/tests/src/ctdb_update_record.c new file mode 100644 index 00000000000..6eff1d04d97 --- /dev/null +++ b/ctdb/tests/src/ctdb_update_record.c @@ -0,0 +1,160 @@ +/* + simple ctdb test tool + This test just fetch_locks a record bumps the RSN and then writes new content + + Copyright (C) Ronnie Sahlberg 2009 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" +#include "ctdb_private.h" + +static struct ctdb_db_context *ctdb_db; + +#define TESTKEY "testkey" + + +/* + Just try locking/unlocking a single record once +*/ +static void fetch_lock_once(struct ctdb_context *ctdb, struct event_context *ev, uint32_t generation) +{ + TALLOC_CTX *tmp_ctx = talloc_new(ctdb); + TDB_DATA key, data; + struct ctdb_record_handle *h; + struct ctdb_ltdb_header *header; + int ret; + + key.dptr = discard_const(TESTKEY); + key.dsize = strlen(TESTKEY); + + printf("Trying to fetch lock the record ...\n"); + + h = ctdb_fetch_readonly_lock(ctdb_db, tmp_ctx, key, &data, false); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + exit(10); + } + + printf("Record fetchlocked.\n"); + header = talloc_memdup(tmp_ctx, ctdb_header_from_record_handle(h), sizeof(*header)); + printf("RSN:%d\n", (int)header->rsn); + talloc_free(h); + printf("Record released.\n"); + + printf("Write new record with RSN+10\n"); + header->rsn += 10; + data.dptr = (void *)talloc_asprintf(tmp_ctx, "%d", (int)header->rsn); + data.dsize = strlen((char *)data.dptr); + + ret = ctdb_ctrl_updaterecord(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, ctdb_db, key, header, data); + if (ret != 0) { + printf("Failed to writerecord, ret==%d\n", ret); + exit(1); + } + + printf("re-fetch the record\n"); + h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data); + if (h == NULL) { + printf("Failed to fetch record '%s' on node %d\n", + (const char *)key.dptr, ctdb_get_pnn(ctdb)); + talloc_free(tmp_ctx); + exit(10); + } + + printf("Record fetchlocked.\n"); + header = talloc_memdup(tmp_ctx, ctdb_header_from_record_handle(h), sizeof(*header)); + printf("RSN:%d\n", (int)header->rsn); + talloc_free(h); + printf("Record released.\n"); + + talloc_free(tmp_ctx); +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + struct event_context *ev; + struct ctdb_vnn_map *vnnmap=NULL; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(5, 0)); + if (ctdb == NULL) { + exit(1); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(5, 0), "test.tdb", false, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + + if (ctdb_ctrl_getvnnmap(ctdb, timeval_zero(), CTDB_CURRENT_NODE, ctdb, &vnnmap) != 0) { + printf("Unable to get vnnmap from local node\n"); + exit(1); + } + printf("Current Generation %d\n", (int)vnnmap->generation); + + fetch_lock_once(ctdb, ev, vnnmap->generation); + + return 0; +} diff --git a/ctdb/tests/src/ctdb_update_record_persistent.c b/ctdb/tests/src/ctdb_update_record_persistent.c new file mode 100644 index 00000000000..a0bb383ed52 --- /dev/null +++ b/ctdb/tests/src/ctdb_update_record_persistent.c @@ -0,0 +1,138 @@ +/* + simple ctdb test tool + This test just creates/updates a record in a persistent database + + Copyright (C) Ronnie Sahlberg 2012 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "lib/util/db_wrap.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" +#include "ctdb_private.h" + + +static void update_once(struct ctdb_context *ctdb, struct event_context *ev, struct ctdb_db_context *ctdb_db, char *record, char *value) +{ + TDB_DATA key, data, olddata; + struct ctdb_ltdb_header header; + + memset(&header, 0, sizeof(header)); + + key.dptr = (uint8_t *)record; + key.dsize = strlen(record); + + data.dptr = (uint8_t *)value; + data.dsize = strlen(value); + + olddata = tdb_fetch(ctdb_db->ltdb->tdb, key); + if (olddata.dsize != 0) { + memcpy(&header, olddata.dptr, sizeof(header)); + } + header.rsn++; + + if (ctdb_ctrl_updaterecord(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, ctdb_db, key, &header, data) != 0) { + printf("Failed to update record\n"); + exit(1); + } +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct ctdb_context *ctdb; + char *test_db = NULL; + char *record = NULL; + char *value = NULL; + struct ctdb_db_context *ctdb_db; + struct event_context *ev; + + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "database", 'D', POPT_ARG_STRING, &test_db, 0, "database", "string" }, + { "record", 'R', POPT_ARG_STRING, &record, 0, "record", "string" }, + { "value", 'V', POPT_ARG_STRING, &value, 0, "value", "string" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(5, 0)); + if (ctdb == NULL) { + exit(1); + } + + if (test_db == NULL) { + fprintf(stderr, "You must specify the database\n"); + exit(10); + } + + if (record == NULL) { + fprintf(stderr, "You must specify the record\n"); + exit(10); + } + + if (value == NULL) { + fprintf(stderr, "You must specify the value\n"); + exit(10); + } + + /* attach to a specific database */ + ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(5, 0), test_db, true, 0); + if (!ctdb_db) { + printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb)); + exit(1); + } + + printf("Waiting for cluster\n"); + while (1) { + uint32_t recmode=1; + ctdb_ctrl_getrecmode(ctdb, ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode); + if (recmode == 0) break; + event_loop_once(ev); + } + + update_once(ctdb, ev, ctdb_db, record, value); + + return 0; +} diff --git a/ctdb/tests/src/ctdbd_test.c b/ctdb/tests/src/ctdbd_test.c new file mode 100644 index 00000000000..fb29ba8f1b3 --- /dev/null +++ b/ctdb/tests/src/ctdbd_test.c @@ -0,0 +1,90 @@ +/* + ctdbd test include file + + Copyright (C) Martin Schwenke 2011 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#ifndef _CTDBD_TEST_C +#define _CTDBD_TEST_C + +#include "includes.h" +#include "tdb.h" +#include "ctdb_private.h" + +/* + * Need these, since they're defined in ctdbd.c but we can't include + * that. + */ +int script_log_level; +bool fast_start; + +/* UTIL_OBJ */ +#include "lib/util/idtree.c" +#include "lib/util/db_wrap.c" +#include "lib/util/strlist.c" +#include "lib/util/util.c" +#include "lib/util/util_time.c" +#include "lib/util/util_file.c" +#include "lib/util/fault.c" +#include "lib/util/substitute.c" +#include "lib/util/signal.c" + +/* CTDB_COMMON_OBJ */ +#include "common/ctdb_io.c" +#include "common/ctdb_util.c" +#include "common/ctdb_ltdb.c" +#include "common/ctdb_message.c" +#include "common/cmdline.c" +#include "lib/util/debug.c" +#include "common/rb_tree.c" +#include "common/system_common.c" +#include "common/ctdb_logging.c" +#include "common/ctdb_fork.c" + +/* CTDB_SERVER_OBJ */ +#include "server/ctdb_daemon.c" +#include "server/ctdb_recoverd.c" +#include "server/ctdb_recover.c" +#include "server/ctdb_freeze.c" +#include "server/ctdb_tunables.c" +#include "server/ctdb_monitor.c" +#include "server/ctdb_server.c" +#include "server/ctdb_control.c" +#include "server/ctdb_call.c" +#include "server/ctdb_ltdb_server.c" +#include "server/ctdb_traverse.c" +#include "server/eventscript.c" +#include "server/ctdb_takeover.c" +#include "server/ctdb_serverids.c" +#include "server/ctdb_persistent.c" +#include "server/ctdb_keepalive.c" +#include "server/ctdb_logging.c" +#include "server/ctdb_uptime.c" +#include "server/ctdb_vacuum.c" +#include "server/ctdb_banning.c" +#include "server/ctdb_statistics.c" +#include "server/ctdb_update_record.c" +#include "server/ctdb_lock.c" + +/* CTDB_CLIENT_OBJ */ +#include "client/ctdb_client.c" + +/* CTDB_TCP_OBJ */ +#include "tcp/tcp_connect.c" +#include "tcp/tcp_io.c" +#include "tcp/tcp_init.c" + +#endif /* _CTDBD_TEST_C */ diff --git a/ctdb/tests/src/rb_perftest.c b/ctdb/tests/src/rb_perftest.c new file mode 100644 index 00000000000..1760cd1149f --- /dev/null +++ b/ctdb/tests/src/rb_perftest.c @@ -0,0 +1,123 @@ +/* + simple rb vs dlist benchmark + + Copyright (C) Ronnie Sahlberg 2007 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "lib/events/events.h" +#include "lib/util/dlinklist.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> +#include "common/rb_tree.h" + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + + +static int num_records = 1000; + + +struct list_node { + struct list_node *prev, *next; +}; + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + POPT_TABLEEND + }; + int opt; + const char **extra_argv; + int extra_argc = 0; + int ret; + poptContext pc; + struct event_context *ev; + double elapsed; + int i; + trbt_tree_t *tree; + struct list_node *list, *list_new, *list_head=NULL; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + ev = event_context_init(NULL); + + + printf("testing tree insert for %d records\n", num_records); + tree = trbt_create(NULL); + start_timer(); + for (i=0;i<num_records;i++) { + trbt_insert32(tree, i, NULL); + } + elapsed=end_timer(); + printf("%f seconds\n",(float)elapsed); + + + printf("testing dlist (worst case) add to tail for %d records\n", num_records); + list_new=talloc(NULL, struct list_node); + DLIST_ADD(list_head, list_new); + start_timer(); + for (i=0;i<num_records;i++) { + for(list=list_head;list->next;list=list->next) { + /* the events code does a timeval_compare */ + timeval_compare(&tp1, &tp2); + } + + list_new=talloc(NULL, struct list_node); + DLIST_ADD_AFTER(list_head, list_new, list); + } + elapsed=end_timer(); + printf("%f seconds\n",(float)elapsed); + + return 0; +} diff --git a/ctdb/tests/src/rb_test.c b/ctdb/tests/src/rb_test.c new file mode 100644 index 00000000000..092732bcf9b --- /dev/null +++ b/ctdb/tests/src/rb_test.c @@ -0,0 +1,347 @@ +/* + simple rb test tool + + Copyright (C) Ronnie Sahlberg 2007 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "includes.h" +#include "lib/util/dlinklist.h" +#include "system/filesys.h" +#include "popt.h" +#include "cmdline.h" + +#include <sys/time.h> +#include <time.h> +#include "common/rb_tree.h" + +static struct timeval tp1,tp2; + +static void start_timer(void) +{ + gettimeofday(&tp1,NULL); +} + +static double end_timer(void) +{ + gettimeofday(&tp2,NULL); + return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - + (tp1.tv_sec + (tp1.tv_usec*1.0e-6)); +} + +int num_records=5; + +void *callback(void *p, void *d) +{ + uint32_t *data = (uint32_t *)d; + + if (d==NULL) { + data = (uint32_t *)p; + } + + (*data)++; + + return data; +} + +void *random_add(void *p, void *d) +{ + return p; +} + +int traverse(void *p, void *d) +{ + uint32_t *data = (uint32_t *)d; + + printf("traverse data:%d\n",*data); + return 0; +} + +int random_traverse(void *p, void *d) +{ + printf("%s ",(char *)d); + return 0; +} + +static uint32_t calc_checksum = 0; +int traverse_checksum(void *p, void *d) +{ + int i,j,k; + + sscanf(d, "%d.%d.%d", &i, &j, &k); + calc_checksum += i*100+j*10+k; + return 0; +} + +int count_traverse(void *p, void *d) +{ + int *count = p; + (*count)++; + return 0; +} + +int count_traverse_abort(void *p, void *d) +{ + int *count = p; + (*count)++; + return -1; +} + +/* + main program +*/ +int main(int argc, const char *argv[]) +{ + struct poptOption popt_options[] = { + POPT_AUTOHELP + POPT_CTDB_CMDLINE + { "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" }, + POPT_TABLEEND + }; + int opt, traverse_count; + const char **extra_argv; + int extra_argc = 0; + poptContext pc; + int i,j,k; + trbt_tree_t *tree; + uint32_t *data; + uint32_t key[3]; + uint32_t key1[3] = {0,10,20}; + uint32_t key2[3] = {0,10,21}; + uint32_t key3[3] = {0,11,20}; + uint32_t key4[3] = {2,10,20}; + TALLOC_CTX *memctx; + uint32_t **u32array; + uint32_t checksum; + + pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST); + + while ((opt = poptGetNextOpt(pc)) != -1) { + switch (opt) { + default: + fprintf(stderr, "Invalid option %s: %s\n", + poptBadOption(pc, 0), poptStrerror(opt)); + exit(1); + } + } + + /* setup the remaining options for the main program to use */ + extra_argv = poptGetArgs(pc); + if (extra_argv) { + extra_argv++; + while (extra_argv[extra_argc]) extra_argc++; + } + + printf("testing trbt_insert32_callback for %d records\n", num_records); + memctx = talloc_new(NULL); + u32array = talloc_array(memctx, uint32_t *, num_records); + tree = trbt_create(memctx, 0); + for (i=0; i<num_records; i++) { + u32array[i] = talloc(u32array, uint32_t); + *u32array[i] = 0; + trbt_insert32_callback(tree, i, callback, u32array[i]); + } + for (i=3; i<num_records; i++) { + trbt_insert32_callback(tree, i, callback, NULL); + } + + printf("first 3 keys should have data==1\n"); + printf("the rest of the keys should have data==2\n"); + for (i=0; i<num_records; i++) { + data = trbt_lookup32(tree, i); + printf("key:%d data:%d\n", i, *data); + } +// talloc_report_full(tree, stdout); +// talloc_report_full(memctx, stdout); +// print_tree(tree); + + printf("deleting key 2\n"); + talloc_free(u32array[2]); +// talloc_report_full(tree, stdout); +// talloc_report_full(memctx, stdout); +// print_tree(tree); + + printf("deleting key 1\n"); + talloc_free(u32array[1]); +// talloc_report_full(tree, stdout); +// talloc_report_full(memctx, stdout); +// print_tree(tree); + + printf("freeing tree\n"); + talloc_report_full(memctx, stdout); + talloc_free(memctx); + + + printf("testing trbt_insertarray32_callback\n"); + memctx = talloc_new(NULL); + tree = trbt_create(memctx, 0); + u32array = talloc_array(memctx, uint32_t *, 4); + for (i=0;i<4;i++) { + u32array[i] = talloc(u32array, uint32_t); + *u32array[i] = 0; + } + trbt_insertarray32_callback(tree, 3, key1, callback, u32array[0]); + trbt_insertarray32_callback(tree, 3, key1, callback, u32array[0]); + trbt_insertarray32_callback(tree, 3, key2, callback, u32array[1]); + trbt_insertarray32_callback(tree, 3, key3, callback, u32array[2]); + trbt_insertarray32_callback(tree, 3, key2, callback, u32array[1]); + trbt_insertarray32_callback(tree, 3, key1, callback, u32array[0]); + + data = trbt_lookuparray32(tree, 3, key1); + printf("key1 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key2); + printf("key2 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key3); + printf("key3 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key4); + printf("key4 dataptr:%p == %d\n",data,data?*data:-1); + trbt_traversearray32(tree, 3, traverse, NULL); + + printf("\ndeleting key4\n"); + talloc_free(trbt_lookuparray32(tree, 3, key4)); + data = trbt_lookuparray32(tree, 3, key1); + printf("key1 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key2); + printf("key2 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key3); + printf("key3 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key4); + printf("key4 dataptr:%p == %d\n",data,data?*data:-1); + trbt_traversearray32(tree, 3, traverse, NULL); + + printf("\ndeleting key2\n"); + talloc_free(trbt_lookuparray32(tree, 3, key2)); + data = trbt_lookuparray32(tree, 3, key1); + printf("key1 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key2); + printf("key2 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key3); + printf("key3 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key4); + printf("key4 dataptr:%p == %d\n",data,data?*data:-1); + trbt_traversearray32(tree, 3, traverse, NULL); + + printf("\ndeleting key3\n"); + talloc_free(trbt_lookuparray32(tree, 3, key3)); + data = trbt_lookuparray32(tree, 3, key1); + printf("key1 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key2); + printf("key2 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key3); + printf("key3 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key4); + printf("key4 dataptr:%p == %d\n",data,data?*data:-1); + trbt_traversearray32(tree, 3, traverse, NULL); + + printf("\ndeleting key1\n"); + talloc_free(trbt_lookuparray32(tree, 3, key1)); + data = trbt_lookuparray32(tree, 3, key1); + printf("key1 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key2); + printf("key2 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key3); + printf("key3 dataptr:%p == %d\n",data,data?*data:-1); + data = trbt_lookuparray32(tree, 3, key4); + printf("key4 dataptr:%p == %d\n",data,data?*data:-1); + trbt_traversearray32(tree, 3, traverse, NULL); + + talloc_free(tree); + talloc_free(memctx); + + + printf("\nrun random insert and delete for 60 seconds\n"); + memctx = talloc_new(NULL); + tree = trbt_create(memctx, 0); + i=0; + start_timer(); + checksum = 0; + /* add and delete nodes from a 3 level tree fro 60 seconds. + each time a node is added or deleted, traverse the tree and + compute a checksum over the data stored in the tree and compare this + with a checksum we keep which contains what the checksum should be + */ + while(end_timer() < 60.0){ + char *str; + + i++; + key[0]=random()%10; + key[1]=random()%10; + key[2]=random()%10; + if (random()%2) { + if (trbt_lookuparray32(tree, 3, key) == NULL) { + /* this node does not yet exist, add it to the + tree and update the checksum + */ + str=talloc_asprintf(memctx, "%d.%d.%d", key[0],key[1],key[2]); + trbt_insertarray32_callback(tree, 3, key, random_add, str); + checksum += key[0]*100+key[1]*10+key[2]; + } + } else { + if ((str=trbt_lookuparray32(tree, 3, key)) != NULL) { + /* this node does exist in the tree, delete + it and update the checksum accordingly + */ + talloc_free(str); + checksum -= key[0]*100+key[1]*10+key[2]; + } + } + /* traverse all nodes in the tree and calculate the checksum + it better match the one we keep track of in + 'checksum' + */ + calc_checksum = 0; + trbt_traversearray32(tree, 3, traverse_checksum, NULL); + if(checksum != calc_checksum) { + printf("Wrong checksum %d!=%d\n",checksum, calc_checksum); + exit(10); + } + + if(i%1000==999)printf(".");fflush(stdout); + } + printf("\niterations passed:%d\n", i); + trbt_traversearray32(tree, 3, random_traverse, NULL); + printf("\n"); + printf("first node: %s\n", (char *)trbt_findfirstarray32(tree, 3)); + + traverse_count = 0; + trbt_traversearray32(tree, 3, count_traverse, &traverse_count); + printf("\n"); + printf("number of entries in traverse %d\n", traverse_count); + + traverse_count = 0; + trbt_traversearray32(tree, 3, count_traverse_abort, &traverse_count); + printf("\n"); + printf("number of entries in aborted traverse %d\n", traverse_count); + if (traverse_count != 1) { + printf("Failed to abort the traverse. Should have been aborted after 1 element but did iterate over %d elements\n", traverse_count); + exit(10); + } + printf("\ndeleting all entries\n"); + for(i=0;i<10;i++){ + for(j=0;j<10;j++){ + for(k=0;k<10;k++){ + key[0]=i; + key[1]=j; + key[2]=k; + talloc_free(trbt_lookuparray32(tree, 3, key)); + } + } + } + trbt_traversearray32(tree, 3, random_traverse, NULL); + printf("\n"); + talloc_report_full(tree, stdout); + + return 0; +} diff --git a/ctdb/tests/takeover/README b/ctdb/tests/takeover/README new file mode 100644 index 00000000000..764f389a04a --- /dev/null +++ b/ctdb/tests/takeover/README @@ -0,0 +1,5 @@ +Unit tests for the CTDB IP allocation algorithm(s). + +Test case filenames look like <algorithm>.NNN.sh, where <algorithm> +indicates the IP allocation algorithm to use. These use the +ctdb_takeover_test test program. diff --git a/ctdb/tests/takeover/det.001.sh b/ctdb/tests/takeover/det.001.sh new file mode 100755 index 00000000000..2387f12fb93 --- /dev/null +++ b/ctdb/tests/takeover/det.001.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 healthy" + +required_result <<EOF +DATE TIME [PID]: Deterministic IPs enabled. Resetting all ip allocations +DATE TIME [PID]: Unassign IP: 192.168.21.254 from 0 +DATE TIME [PID]: Unassign IP: 192.168.21.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.254 from 0 +DATE TIME [PID]: Unassign IP: 192.168.20.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.251 from 0 +DATE TIME [PID]: Unassign IP: 192.168.20.250 from 1 +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF + +simple_test 2,2,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/det.002.sh b/ctdb/tests/takeover/det.002.sh new file mode 100755 index 00000000000..21fbaec316a --- /dev/null +++ b/ctdb/tests/takeover/det.002.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 2 healthy" + +required_result <<EOF +DATE TIME [PID]: Deterministic IPs enabled. Resetting all ip allocations +DATE TIME [PID]: Unassign IP: 192.168.21.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.250 from 1 +192.168.21.254 0 +192.168.21.253 0 +192.168.21.252 2 +192.168.20.254 0 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 0 +192.168.20.250 0 +192.168.20.249 2 +EOF + +simple_test 0,2,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/det.003.sh b/ctdb/tests/takeover/det.003.sh new file mode 100755 index 00000000000..3666047217a --- /dev/null +++ b/ctdb/tests/takeover/det.003.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 -> all healthy" + +required_result <<EOF +DATE TIME [PID]: Deterministic IPs enabled. Resetting all ip allocations +192.168.21.254 0 +192.168.21.253 1 +192.168.21.252 2 +192.168.20.254 0 +192.168.20.253 1 +192.168.20.252 2 +192.168.20.251 0 +192.168.20.250 1 +192.168.20.249 2 +EOF + +simple_test 0,0,0 <<EOF +192.168.20.249 1 +192.168.20.250 1 +192.168.20.251 1 +192.168.20.252 1 +192.168.20.253 1 +192.168.20.254 1 +192.168.21.252 1 +192.168.21.253 1 +192.168.21.254 1 +EOF diff --git a/ctdb/tests/takeover/lcp2.001.sh b/ctdb/tests/takeover/lcp2.001.sh new file mode 100755 index 00000000000..8772318d6e3 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.001.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 3 -> 1 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF + +simple_test 2,2,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.002.sh b/ctdb/tests/takeover/lcp2.002.sh new file mode 100755 index 00000000000..f3f6f0a6b1a --- /dev/null +++ b/ctdb/tests/takeover/lcp2.002.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 3 -> 2 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 0 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 0 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 0 +EOF + +simple_test 0,2,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.003.sh b/ctdb/tests/takeover/lcp2.003.sh new file mode 100755 index 00000000000..f6cfe57b2d9 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.003.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 -> all healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 1 +192.168.20.254 2 +192.168.20.253 0 +192.168.20.252 1 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 1 +EOF + +simple_test 0,0,0 <<EOF +192.168.20.249 1 +192.168.20.250 1 +192.168.20.251 1 +192.168.20.252 1 +192.168.20.253 1 +192.168.20.254 1 +192.168.21.252 1 +192.168.21.253 1 +192.168.21.254 1 +EOF diff --git a/ctdb/tests/takeover/lcp2.004.sh b/ctdb/tests/takeover/lcp2.004.sh new file mode 100755 index 00000000000..c067184d383 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.004.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 -> all healthy, info logging" + +export CTDB_TEST_LOGLEVEL=3 + +required_result <<EOF +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 1 +192.168.20.254 2 +192.168.20.253 0 +192.168.20.252 1 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 1 +EOF + +simple_test 0,0,0 <<EOF +192.168.20.249 1 +192.168.20.250 1 +192.168.20.251 1 +192.168.20.252 1 +192.168.20.253 1 +192.168.20.254 1 +192.168.21.252 1 +192.168.21.253 1 +192.168.21.254 1 +EOF diff --git a/ctdb/tests/takeover/lcp2.005.sh b/ctdb/tests/takeover/lcp2.005.sh new file mode 100755 index 00000000000..113e52f4acc --- /dev/null +++ b/ctdb/tests/takeover/lcp2.005.sh @@ -0,0 +1,163 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 -> all healthy, debug logging" + +export CTDB_TEST_LOGLEVEL=4 + +required_result <<EOF +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [539166] +DATE TIME [PID]: 1 [-116718] -> 192.168.21.254 -> 0 [+0] +DATE TIME [PID]: 1 [-116718] -> 192.168.21.254 -> 2 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.253 -> 0 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.253 -> 2 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.252 -> 0 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.252 -> 2 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.254 -> 0 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.254 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.252 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.252 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.251 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.250 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.250 -> 2 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.249 -> 0 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.249 -> 2 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [418056] +DATE TIME [PID]: 1 [-102557] -> 192.168.21.254 -> 0 [+14161] +DATE TIME [PID]: 1 [-102557] -> 192.168.21.254 -> 2 [+0] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.253 -> 2 [+0] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.252 -> 0 [+14161] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.252 -> 2 [+0] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.254 -> 0 [+15876] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.254 -> 2 [+0] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.252 -> 0 [+16129] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.252 -> 2 [+0] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 0 [+15625] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.250 -> 0 [+15625] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.250 -> 2 [+0] +DATE TIME [PID]: 1 [-105485] -> 192.168.20.249 -> 0 [+15625] +DATE TIME [PID]: 1 [-105485] -> 192.168.20.249 -> 2 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [312571] +DATE TIME [PID]: 1 [-88396] -> 192.168.21.254 -> 0 [+14161] +DATE TIME [PID]: 1 [-88396] -> 192.168.21.254 -> 2 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 2 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.252 -> 0 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.252 -> 2 [+14161] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.254 -> 0 [+15876] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.252 -> 0 [+16129] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.252 -> 2 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.250 -> 0 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.250 -> 2 [+16129] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.249 -> 0 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.249 -> 2 [+15876] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [222962] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.254 -> 0 [+30037] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.254 -> 2 [+14161] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.252 -> 2 [+14161] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 0 [+30037] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.252 -> 2 [+15625] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.250 -> 2 [+16129] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.249 -> 0 [+29786] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.249 -> 2 [+15876] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [147514] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.254 -> 0 [+30037] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.252 -> 2 [+28322] +DATE TIME [PID]: 1 [-59572] -> 192.168.20.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-59572] -> 192.168.20.252 -> 2 [+31501] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 2 [+31754] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.249 -> 0 [+29786] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.249 -> 2 [+31501] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [87691] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 0 [+44198] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.252 -> 0 [+44451] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.252 -> 2 [+28322] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.252 -> 0 [+45915] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.252 -> 2 [+31501] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.249 -> 0 [+45662] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.249 -> 2 [+31501] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 0 [43947] +DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 0 [+28322] +DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 2 [+44198] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.253 -> 0 [+29786] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.253 -> 2 [+45662] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 2 [+45915] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 2 [43947] +DATE TIME [PID]: 2 [-28322] -> 192.168.21.254 -> 0 [+44198] +DATE TIME [PID]: 2 [-28322] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.254 -> 0 [+45662] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.254 -> 2 [+29786] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.251 -> 0 [+45915] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.251 -> 2 [+29786] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [43744] +DATE TIME [PID]: 1 [-28322] -> 192.168.21.252 -> 0 [+44451] +DATE TIME [PID]: 1 [-28322] -> 192.168.21.252 -> 2 [+44198] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.252 -> 0 [+45915] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.252 -> 2 [+45662] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.249 -> 0 [+45662] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.249 -> 2 [+45662] +DATE TIME [PID]: ---------------------------------------- +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 1 +192.168.20.254 2 +192.168.20.253 0 +192.168.20.252 1 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 1 +EOF + +simple_test 0,0,0 <<EOF +192.168.20.249 1 +192.168.20.250 1 +192.168.20.251 1 +192.168.20.252 1 +192.168.20.253 1 +192.168.20.254 1 +192.168.21.252 1 +192.168.21.253 1 +192.168.21.254 1 +EOF diff --git a/ctdb/tests/takeover/lcp2.006.sh b/ctdb/tests/takeover/lcp2.006.sh new file mode 100755 index 00000000000..13bb40fd69c --- /dev/null +++ b/ctdb/tests/takeover/lcp2.006.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 0 -> 1 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 1 +192.168.21.253 1 +192.168.21.252 1 +192.168.20.254 1 +192.168.20.253 1 +192.168.20.252 1 +192.168.20.251 1 +192.168.20.250 1 +192.168.20.249 1 +EOF + +simple_test 2,0,2 <<EOF +192.168.20.249 -1 +192.168.20.250 -1 +192.168.20.251 -1 +192.168.20.252 -1 +192.168.20.253 -1 +192.168.20.254 -1 +192.168.21.252 -1 +192.168.21.253 -1 +192.168.21.254 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.007.sh b/ctdb/tests/takeover/lcp2.007.sh new file mode 100755 index 00000000000..76fa06e212a --- /dev/null +++ b/ctdb/tests/takeover/lcp2.007.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 0 -> 2 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 1 +192.168.21.253 2 +192.168.21.252 1 +192.168.20.254 1 +192.168.20.253 2 +192.168.20.252 1 +192.168.20.251 1 +192.168.20.250 2 +192.168.20.249 2 +EOF + +simple_test 2,0,0 <<EOF +192.168.20.249 -1 +192.168.20.250 -1 +192.168.20.251 -1 +192.168.20.252 -1 +192.168.20.253 -1 +192.168.20.254 -1 +192.168.21.252 -1 +192.168.21.253 -1 +192.168.21.254 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.008.sh b/ctdb/tests/takeover/lcp2.008.sh new file mode 100755 index 00000000000..f5c0af3a7c6 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.008.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 0 -> all healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 0 +192.168.21.253 1 +192.168.21.252 2 +192.168.20.254 0 +192.168.20.253 1 +192.168.20.252 2 +192.168.20.251 0 +192.168.20.250 1 +192.168.20.249 2 +EOF + +simple_test 0,0,0 <<EOF +192.168.20.249 -1 +192.168.20.250 -1 +192.168.20.251 -1 +192.168.20.252 -1 +192.168.20.253 -1 +192.168.20.254 -1 +192.168.21.252 -1 +192.168.21.253 -1 +192.168.21.254 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.009.sh b/ctdb/tests/takeover/lcp2.009.sh new file mode 100755 index 00000000000..e862c928070 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.009.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 3 healthy -> all disconnected" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF + +simple_test 1,1,1 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.010.sh b/ctdb/tests/takeover/lcp2.010.sh new file mode 100755 index 00000000000..20b1c98e65e --- /dev/null +++ b/ctdb/tests/takeover/lcp2.010.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "2 disjoint groups of nodes/addresses, a node becomes healthy" + +# This illustrates a bug in LCP2 when the the only candidate for a +# source node is chosen to be the "most imbalanced" node. This means +# that nodes in the smaller group aren't necessarily (depends on sort +# order and addresses used) considered as candidates. If the larger +# group has 6 addresses then the "necessarily" goes away and the +# smaller group won't be rebalanced. + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.209.102 3 +192.168.209.101 2 +192.168.140.4 1 +192.168.140.3 1 +192.168.140.2 0 +192.168.140.1 0 +EOF + +simple_test 0,0,0,0 <<EOF +192.168.140.1 0 0,1 +192.168.140.2 0 0,1 +192.168.140.3 1 0,1 +192.168.140.4 1 0,1 +192.168.209.101 2 2,3 +192.168.209.102 2 2,3 +EOF diff --git a/ctdb/tests/takeover/lcp2.011.sh b/ctdb/tests/takeover/lcp2.011.sh new file mode 100755 index 00000000000..f752aa345e3 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.011.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "2 disjoint groups of nodes/addresses, continue a stopped node" + +# Another LCP2 1.0 bug + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +10.11.19.46 3 +10.11.19.45 3 +10.11.19.44 1 +10.11.18.46 1 +10.11.18.45 3 +10.11.18.44 1 +10.11.17.46 3 +10.11.17.45 3 +10.11.17.44 1 +10.11.16.46 1 +10.11.16.45 3 +10.11.16.44 1 +9.11.136.46 2 +9.11.136.45 0 +9.11.136.44 2 +EOF + +simple_test 0,0,0,0 <<EOF +9.11.136.44 2 0,2 +9.11.136.45 2 0,2 +9.11.136.46 2 0,2 +10.11.16.44 1 1,3 +10.11.16.45 3 1,3 +10.11.16.46 1 1,3 +10.11.17.44 1 1,3 +10.11.17.45 3 1,3 +10.11.17.46 3 1,3 +10.11.18.44 1 1,3 +10.11.18.45 3 1,3 +10.11.18.46 1 1,3 +10.11.19.44 1 1,3 +10.11.19.45 3 1,3 +10.11.19.46 3 1,3 +EOF diff --git a/ctdb/tests/takeover/lcp2.012.sh b/ctdb/tests/takeover/lcp2.012.sh new file mode 100755 index 00000000000..8f5c537e967 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.012.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Node with NODE_FLAGS_NOIPTAKEOVER doesn't gain IPs" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 1 +192.168.21.253 2 +192.168.21.252 1 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 1 +192.168.20.251 2 +192.168.20.250 1 +192.168.20.249 1 +EOF + +export CTDB_SET_NoIPTakeover="1,0,0" + +simple_test 0,0,0 <<EOF +192.168.20.249 1 +192.168.20.250 1 +192.168.20.251 1 +192.168.20.252 1 +192.168.20.253 1 +192.168.20.254 1 +192.168.21.252 1 +192.168.21.253 1 +192.168.21.254 1 +EOF diff --git a/ctdb/tests/takeover/lcp2.013.sh b/ctdb/tests/takeover/lcp2.013.sh new file mode 100755 index 00000000000..fb9d724e35f --- /dev/null +++ b/ctdb/tests/takeover/lcp2.013.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "Node with NODE_FLAGS_NOIPTAKEOVER doesn't lose IPs" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 2 +192.168.21.253 1 +192.168.21.252 0 +192.168.20.254 2 +192.168.20.253 1 +192.168.20.252 0 +192.168.20.251 2 +192.168.20.250 1 +192.168.20.249 0 +EOF + +export CTDB_SET_NoIPTakeover="1,0,0" + +simple_test 0,0,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.014.sh b/ctdb/tests/takeover/lcp2.014.sh new file mode 100755 index 00000000000..36eda9284b4 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.014.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, all unhealthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 0 +192.168.21.253 1 +192.168.21.252 2 +192.168.20.254 0 +192.168.20.253 1 +192.168.20.252 2 +192.168.20.251 0 +192.168.20.250 1 +192.168.20.249 2 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.015.sh b/ctdb/tests/takeover/lcp2.015.sh new file mode 100755 index 00000000000..a2569e05b23 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.015.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all IPs assigned, all unhealthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 1 +192.168.20.253 1 +192.168.20.252 1 +192.168.20.251 0 +192.168.20.250 0 +192.168.20.249 0 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 1 +192.168.20.253 1 +192.168.20.252 1 +192.168.20.251 0 +192.168.20.250 0 +192.168.20.249 0 +EOF diff --git a/ctdb/tests/takeover/lcp2.016.sh b/ctdb/tests/takeover/lcp2.016.sh new file mode 100755 index 00000000000..2e2df1b9d0e --- /dev/null +++ b/ctdb/tests/takeover/lcp2.016.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all IPs assigned, 2->3 unhealthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 1 +192.168.21.253 0 +192.168.21.252 2 +192.168.20.254 1 +192.168.20.253 0 +192.168.20.252 2 +192.168.20.251 1 +192.168.20.250 0 +192.168.20.249 2 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.017.sh b/ctdb/tests/takeover/lcp2.017.sh new file mode 100755 index 00000000000..07b22fb9f99 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.017.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, all unhealthy, NoIPHostOnAllDisabled" + +export CTDB_TEST_LOGLEVEL=0 +export CTDB_SET_NoIPHostOnAllDisabled=1 + +required_result <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.018.sh b/ctdb/tests/takeover/lcp2.018.sh new file mode 100755 index 00000000000..4a797f78c62 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.018.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all IPs assigned, all unhealthy, NoIPHostOnAllDisabled" + +export CTDB_TEST_LOGLEVEL=0 +export CTDB_SET_NoIPHostOnAllDisabled=1 + +required_result <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 1 +192.168.20.253 1 +192.168.20.252 1 +192.168.20.251 0 +192.168.20.250 0 +192.168.20.249 0 +EOF diff --git a/ctdb/tests/takeover/lcp2.019.sh b/ctdb/tests/takeover/lcp2.019.sh new file mode 100755 index 00000000000..0d8937cdc54 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.019.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all IPs assigned, 2->3 unhealthy, NoIPHostOnAllDisabled" + +export CTDB_TEST_LOGLEVEL=0 +export CTDB_SET_NoIPHostOnAllDisabled=1 + +required_result <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.020.sh b/ctdb/tests/takeover/lcp2.020.sh new file mode 100755 index 00000000000..e3fe3c4e405 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.020.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all IPs assigned, 2->3 unhealthy, NoIPHostOnAllDisabled on 2" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF + +export CTDB_SET_NoIPHostOnAllDisabled=1,1,0 + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.021.sh b/ctdb/tests/takeover/lcp2.021.sh new file mode 100755 index 00000000000..7dcddb14ee8 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.021.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, 3->2 unhealthy, NoIPHostOnAllDisabled on 2 others" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.21.254 0 +192.168.21.253 0 +192.168.21.252 0 +192.168.20.254 0 +192.168.20.253 0 +192.168.20.252 0 +192.168.20.251 0 +192.168.20.250 0 +192.168.20.249 0 +EOF + +export CTDB_SET_NoIPHostOnAllDisabled=0,1,1 + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.022.sh b/ctdb/tests/takeover/lcp2.022.sh new file mode 100755 index 00000000000..7eb4d8a4330 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.022.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, 3->2 unhealthy, NoIPTakeover on 2 others" + +export CTDB_TEST_LOGLEVEL=0 + +# We expect 1/2 the IPs to move, but the rest to stay (as opposed to +# NoIPHostOnAllDisabled) +required_result <<EOF +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 2 +192.168.20.254 0 +192.168.20.253 0 +192.168.20.252 2 +192.168.20.251 0 +192.168.20.250 2 +192.168.20.249 2 +EOF + +export CTDB_SET_NoIPTakeover=0,1,1 + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.023.sh b/ctdb/tests/takeover/lcp2.023.sh new file mode 100755 index 00000000000..9bffc58c6ba --- /dev/null +++ b/ctdb/tests/takeover/lcp2.023.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all IPs assigned, 1->3 unhealthy" + +export CTDB_TEST_LOGLEVEL=4 + +required_result <<EOF +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 2 [147968] +DATE TIME [PID]: 2 [-58359] -> 192.168.21.254 -> 1 [+0] +DATE TIME [PID]: 2 [-58359] -> 192.168.21.252 -> 1 [+0] +DATE TIME [PID]: 2 [-59572] -> 192.168.20.253 -> 1 [+0] +DATE TIME [PID]: 2 [-59823] -> 192.168.20.251 -> 1 [+0] +DATE TIME [PID]: 2 [-59823] -> 192.168.20.249 -> 1 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 2 [-59823] -> 192.168.20.251 -> 1 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 0 [89609] +DATE TIME [PID]: 0 [-42483] -> 192.168.21.253 -> 1 [+14161] +DATE TIME [PID]: 0 [-45662] -> 192.168.20.254 -> 1 [+15625] +DATE TIME [PID]: 0 [-45662] -> 192.168.20.252 -> 1 [+15625] +DATE TIME [PID]: 0 [-45411] -> 192.168.20.250 -> 1 [+16129] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 0 [-45662] -> 192.168.20.254 -> 1 [+15625] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 2 [88145] +DATE TIME [PID]: 2 [-44198] -> 192.168.21.254 -> 1 [+28322] +DATE TIME [PID]: 2 [-44198] -> 192.168.21.252 -> 1 [+28322] +DATE TIME [PID]: 2 [-43947] -> 192.168.20.253 -> 1 [+31501] +DATE TIME [PID]: 2 [-43947] -> 192.168.20.249 -> 1 [+31501] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 2 [-44198] -> 192.168.21.254 -> 1 [+28322] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 0 [44198] +DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 1 [+44198] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.252 -> 1 [+45662] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 1 [+45915] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 2 [44198] +DATE TIME [PID]: 2 [-28322] -> 192.168.21.252 -> 1 [+44198] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.253 -> 1 [+45662] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.249 -> 1 [+45662] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [43947] +DATE TIME [PID]: 1 [-28322] -> 192.168.21.254 -> 1 [+28322] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.254 -> 1 [+29786] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.251 -> 1 [+29786] +DATE TIME [PID]: ---------------------------------------- +192.168.21.254 1 +192.168.21.253 0 +192.168.21.252 2 +192.168.20.254 1 +192.168.20.253 2 +192.168.20.252 0 +192.168.20.251 1 +192.168.20.250 0 +192.168.20.249 2 +EOF + +simple_test 2,2,2 <<EOF +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 2 +192.168.20.254 0 +192.168.20.253 2 +192.168.20.252 0 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 2 +EOF diff --git a/ctdb/tests/takeover/lcp2.024.sh b/ctdb/tests/takeover/lcp2.024.sh new file mode 100755 index 00000000000..05095523ed5 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.024.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, all healthy, all in STARTUP runstate" + +export CTDB_TEST_LOGLEVEL=2 + +required_result <<EOF +DATE TIME [PID]: Failed to find node to cover ip 192.168.21.254 +DATE TIME [PID]: Failed to find node to cover ip 192.168.21.253 +DATE TIME [PID]: Failed to find node to cover ip 192.168.21.252 +DATE TIME [PID]: Failed to find node to cover ip 192.168.20.254 +DATE TIME [PID]: Failed to find node to cover ip 192.168.20.253 +DATE TIME [PID]: Failed to find node to cover ip 192.168.20.252 +DATE TIME [PID]: Failed to find node to cover ip 192.168.20.251 +DATE TIME [PID]: Failed to find node to cover ip 192.168.20.250 +DATE TIME [PID]: Failed to find node to cover ip 192.168.20.249 +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF + +export CTDB_TEST_RUNSTATE=4,4,4 + +simple_test 0,0,0 <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.025.sh b/ctdb/tests/takeover/lcp2.025.sh new file mode 100755 index 00000000000..44b8583edc0 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.025.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, all healthy, 1 in STARTUP runstate" + +export CTDB_TEST_LOGLEVEL=2 + +required_result <<EOF +192.168.21.254 1 +192.168.21.253 2 +192.168.21.252 1 +192.168.20.254 1 +192.168.20.253 2 +192.168.20.252 1 +192.168.20.251 1 +192.168.20.250 2 +192.168.20.249 2 +EOF + +export CTDB_TEST_RUNSTATE=4,5,5 + +simple_test 0,0,0 <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.026.sh b/ctdb/tests/takeover/lcp2.026.sh new file mode 100755 index 00000000000..4c22ba56074 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.026.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, no IPs assigned, all unhealthy, 1 in STARTUP runstate" + +export CTDB_TEST_LOGLEVEL=2 + +required_result <<EOF +192.168.21.254 1 +192.168.21.253 2 +192.168.21.252 1 +192.168.20.254 1 +192.168.20.253 2 +192.168.20.252 1 +192.168.20.251 1 +192.168.20.250 2 +192.168.20.249 2 +EOF + +export CTDB_TEST_RUNSTATE=4,5,5 + +simple_test 2,2,2 <<EOF +192.168.21.254 -1 +192.168.21.253 -1 +192.168.21.252 -1 +192.168.20.254 -1 +192.168.20.253 -1 +192.168.20.252 -1 +192.168.20.251 -1 +192.168.20.250 -1 +192.168.20.249 -1 +EOF diff --git a/ctdb/tests/takeover/lcp2.027.sh b/ctdb/tests/takeover/lcp2.027.sh new file mode 100755 index 00000000000..20e0f28f950 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.027.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "4 nodes, all IPs assigned, 3->4 unhealthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 3 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 0 +130.216.30.172 3 +130.216.30.171 1 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 0 +10.19.99.250 3 +EOF + +simple_test 0,0,2,0 <<EOF +130.216.30.170 3 +130.216.30.171 2 +130.216.30.172 3 +130.216.30.173 2 +130.216.30.174 1 +130.216.30.175 0 +130.216.30.176 1 +130.216.30.177 0 +130.216.30.178 3 +130.216.30.179 2 +130.216.30.180 1 +130.216.30.181 0 +10.19.99.250 3 +10.19.99.251 2 +10.19.99.252 1 +10.19.99.253 0 +EOF diff --git a/ctdb/tests/takeover/lcp2.028.sh b/ctdb/tests/takeover/lcp2.028.sh new file mode 100755 index 00000000000..60d22d94174 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.028.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "4 nodes, all healthy/assigned, stays unbalanced" + +export CTDB_TEST_LOGLEVEL=3 + +required_result <<EOF +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 2 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 0 +130.216.30.172 3 +130.216.30.171 1 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 0 +10.19.99.250 3 +EOF + +simple_test 0,0,0,0 <<EOF +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 2 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 0 +130.216.30.172 3 +130.216.30.171 1 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 0 +10.19.99.250 3 +EOF diff --git a/ctdb/tests/takeover/lcp2.029.sh b/ctdb/tests/takeover/lcp2.029.sh new file mode 100755 index 00000000000..d3c817f0c53 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.029.sh @@ -0,0 +1,111 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "4 nodes, some IPs unassigned on target nodes" + +export CTDB_TEST_LOGLEVEL=3 + +required_result <<EOF +DATE TIME [PID]: 10.19.99.251 -> 2 [+9216] +DATE TIME [PID]: 130.216.30.173 -> 2 [+24345] +DATE TIME [PID]: 130.216.30.171 -> 2 [+39970] +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 2 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 2 +130.216.30.172 3 +130.216.30.171 2 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 2 +10.19.99.250 3 +EOF + +# In this example were 4 releases from node 2 in a previous iteration +# +# Release of IP 130.216.30.179/27 on interface ethX1 node:3 +# Release of IP 130.216.30.173/27 on interface ethX1 node:0 +# Release of IP 130.216.30.171/27 on interface ethX1 node:1 +# Release of IP 10.19.99.251/22 on interface ethX2 node:0 +# +# However, one release failed so no takeovers were done. This means +# that the target node for each IP still thinks that the IPs are held +# by node 2. The release of 130.216.30.179 was so late that node 2 +# still thought that it held that address. + +simple_test 0,0,0,0 multi <<EOF +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 3 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 2 +130.216.30.172 3 +130.216.30.171 1 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 2 +10.19.99.250 3 + +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 3 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 0 +130.216.30.172 3 +130.216.30.171 2 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 0 +10.19.99.250 3 + +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 2 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 0 +130.216.30.172 3 +130.216.30.171 1 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 0 +10.19.99.250 3 + +130.216.30.181 0 +130.216.30.180 1 +130.216.30.179 2 +130.216.30.178 3 +130.216.30.177 0 +130.216.30.176 1 +130.216.30.175 0 +130.216.30.174 1 +130.216.30.173 0 +130.216.30.172 3 +130.216.30.171 1 +130.216.30.170 3 +10.19.99.253 0 +10.19.99.252 1 +10.19.99.251 0 +10.19.99.250 3 +EOF diff --git a/ctdb/tests/takeover/lcp2.030.sh b/ctdb/tests/takeover/lcp2.030.sh new file mode 100755 index 00000000000..739757b0b17 --- /dev/null +++ b/ctdb/tests/takeover/lcp2.030.sh @@ -0,0 +1,1813 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "900 IPs, 5 nodes, 0 -> 5 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result <<EOF +192.168.10.90 0 +192.168.10.89 1 +192.168.10.88 2 +192.168.10.87 3 +192.168.10.86 4 +192.168.10.85 0 +192.168.10.84 1 +192.168.10.83 2 +192.168.10.82 3 +192.168.10.81 4 +192.168.10.80 0 +192.168.10.79 0 +192.168.10.78 1 +192.168.10.77 2 +192.168.10.76 3 +192.168.10.75 4 +192.168.10.74 1 +192.168.10.73 2 +192.168.10.72 3 +192.168.10.71 3 +192.168.10.70 4 +192.168.10.69 0 +192.168.10.68 1 +192.168.10.67 2 +192.168.10.66 4 +192.168.10.65 0 +192.168.10.64 1 +192.168.10.63 0 +192.168.10.62 1 +192.168.10.61 2 +192.168.10.60 3 +192.168.10.59 4 +192.168.10.58 2 +192.168.10.57 3 +192.168.10.56 0 +192.168.10.55 0 +192.168.10.54 1 +192.168.10.53 2 +192.168.10.52 3 +192.168.10.51 4 +192.168.10.50 1 +192.168.10.49 4 +192.168.10.48 2 +192.168.10.47 0 +192.168.10.46 1 +192.168.10.45 2 +192.168.10.44 3 +192.168.10.43 4 +192.168.10.42 2 +192.168.10.41 3 +192.168.10.40 1 +192.168.10.39 3 +192.168.10.38 4 +192.168.10.37 0 +192.168.10.36 1 +192.168.10.35 2 +192.168.10.34 4 +192.168.10.33 0 +192.168.10.32 3 +192.168.10.31 0 +192.168.10.30 1 +192.168.10.29 2 +192.168.10.28 3 +192.168.10.27 4 +192.168.10.26 3 +192.168.10.25 2 +192.168.10.24 0 +192.168.10.23 3 +192.168.10.22 4 +192.168.10.21 0 +192.168.10.20 1 +192.168.10.19 2 +192.168.10.18 4 +192.168.10.17 1 +192.168.10.16 4 +192.168.10.15 0 +192.168.10.14 1 +192.168.10.13 2 +192.168.10.12 3 +192.168.10.11 4 +192.168.10.10 2 +192.168.10.9 3 +192.168.10.8 4 +192.168.10.7 0 +192.168.10.6 1 +192.168.10.5 2 +192.168.10.4 3 +192.168.10.3 4 +192.168.10.2 0 +192.168.10.1 1 +192.168.9.90 0 +192.168.9.89 1 +192.168.9.88 2 +192.168.9.87 3 +192.168.9.86 4 +192.168.9.85 0 +192.168.9.84 1 +192.168.9.83 2 +192.168.9.82 3 +192.168.9.81 4 +192.168.9.80 0 +192.168.9.79 0 +192.168.9.78 1 +192.168.9.77 2 +192.168.9.76 3 +192.168.9.75 4 +192.168.9.74 1 +192.168.9.73 2 +192.168.9.72 3 +192.168.9.71 3 +192.168.9.70 4 +192.168.9.69 0 +192.168.9.68 1 +192.168.9.67 2 +192.168.9.66 4 +192.168.9.65 0 +192.168.9.64 1 +192.168.9.63 0 +192.168.9.62 1 +192.168.9.61 2 +192.168.9.60 3 +192.168.9.59 4 +192.168.9.58 2 +192.168.9.57 3 +192.168.9.56 4 +192.168.9.55 0 +192.168.9.54 1 +192.168.9.53 2 +192.168.9.52 3 +192.168.9.51 4 +192.168.9.50 0 +192.168.9.49 1 +192.168.9.48 2 +192.168.9.47 0 +192.168.9.46 1 +192.168.9.45 2 +192.168.9.44 3 +192.168.9.43 4 +192.168.9.42 2 +192.168.9.41 4 +192.168.9.40 3 +192.168.9.39 0 +192.168.9.38 1 +192.168.9.37 2 +192.168.9.36 3 +192.168.9.35 4 +192.168.9.34 0 +192.168.9.33 1 +192.168.9.32 4 +192.168.9.31 0 +192.168.9.30 1 +192.168.9.29 2 +192.168.9.28 3 +192.168.9.27 4 +192.168.9.26 2 +192.168.9.25 3 +192.168.9.24 0 +192.168.9.23 3 +192.168.9.22 4 +192.168.9.21 0 +192.168.9.20 1 +192.168.9.19 2 +192.168.9.18 4 +192.168.9.17 1 +192.168.9.16 3 +192.168.9.15 0 +192.168.9.14 1 +192.168.9.13 2 +192.168.9.12 3 +192.168.9.11 4 +192.168.9.10 2 +192.168.9.9 4 +192.168.9.8 3 +192.168.9.7 0 +192.168.9.6 1 +192.168.9.5 2 +192.168.9.4 3 +192.168.9.3 4 +192.168.9.2 0 +192.168.9.1 1 +192.168.8.90 0 +192.168.8.89 1 +192.168.8.88 2 +192.168.8.87 3 +192.168.8.86 4 +192.168.8.85 0 +192.168.8.84 1 +192.168.8.83 2 +192.168.8.82 3 +192.168.8.81 4 +192.168.8.80 0 +192.168.8.79 0 +192.168.8.78 1 +192.168.8.77 2 +192.168.8.76 3 +192.168.8.75 4 +192.168.8.74 1 +192.168.8.73 2 +192.168.8.72 3 +192.168.8.71 3 +192.168.8.70 4 +192.168.8.69 0 +192.168.8.68 1 +192.168.8.67 2 +192.168.8.66 4 +192.168.8.65 3 +192.168.8.64 0 +192.168.8.63 0 +192.168.8.62 1 +192.168.8.61 2 +192.168.8.60 3 +192.168.8.59 4 +192.168.8.58 1 +192.168.8.57 2 +192.168.8.56 3 +192.168.8.55 0 +192.168.8.54 1 +192.168.8.53 2 +192.168.8.52 3 +192.168.8.51 4 +192.168.8.50 0 +192.168.8.49 4 +192.168.8.48 1 +192.168.8.47 0 +192.168.8.46 1 +192.168.8.45 2 +192.168.8.44 3 +192.168.8.43 4 +192.168.8.42 2 +192.168.8.41 1 +192.168.8.40 4 +192.168.8.39 0 +192.168.8.38 1 +192.168.8.37 2 +192.168.8.36 3 +192.168.8.35 4 +192.168.8.34 3 +192.168.8.33 0 +192.168.8.32 2 +192.168.8.31 0 +192.168.8.30 1 +192.168.8.29 2 +192.168.8.28 3 +192.168.8.27 4 +192.168.8.26 2 +192.168.8.25 1 +192.168.8.24 3 +192.168.8.23 3 +192.168.8.22 4 +192.168.8.21 0 +192.168.8.20 1 +192.168.8.19 2 +192.168.8.18 4 +192.168.8.17 0 +192.168.8.16 4 +192.168.8.15 0 +192.168.8.14 1 +192.168.8.13 2 +192.168.8.12 3 +192.168.8.11 4 +192.168.8.10 1 +192.168.8.9 2 +192.168.8.8 4 +192.168.8.7 0 +192.168.8.6 1 +192.168.8.5 2 +192.168.8.4 3 +192.168.8.3 4 +192.168.8.2 3 +192.168.8.1 0 +192.168.7.90 0 +192.168.7.89 1 +192.168.7.88 2 +192.168.7.87 3 +192.168.7.86 4 +192.168.7.85 0 +192.168.7.84 1 +192.168.7.83 2 +192.168.7.82 3 +192.168.7.81 4 +192.168.7.80 1 +192.168.7.79 0 +192.168.7.78 1 +192.168.7.77 2 +192.168.7.76 3 +192.168.7.75 4 +192.168.7.74 2 +192.168.7.73 3 +192.168.7.72 0 +192.168.7.71 3 +192.168.7.70 4 +192.168.7.69 0 +192.168.7.68 1 +192.168.7.67 2 +192.168.7.66 4 +192.168.7.65 1 +192.168.7.64 3 +192.168.7.63 0 +192.168.7.62 1 +192.168.7.61 2 +192.168.7.60 3 +192.168.7.59 4 +192.168.7.58 2 +192.168.7.57 0 +192.168.7.56 1 +192.168.7.55 0 +192.168.7.54 1 +192.168.7.53 2 +192.168.7.52 3 +192.168.7.51 4 +192.168.7.50 3 +192.168.7.49 4 +192.168.7.48 2 +192.168.7.47 0 +192.168.7.46 1 +192.168.7.45 2 +192.168.7.44 3 +192.168.7.43 4 +192.168.7.42 2 +192.168.7.41 0 +192.168.7.40 1 +192.168.7.39 4 +192.168.7.38 0 +192.168.7.37 1 +192.168.7.36 2 +192.168.7.35 3 +192.168.7.34 4 +192.168.7.33 3 +192.168.7.32 0 +192.168.7.31 0 +192.168.7.30 1 +192.168.7.29 2 +192.168.7.28 3 +192.168.7.27 4 +192.168.7.26 2 +192.168.7.25 0 +192.168.7.24 1 +192.168.7.23 3 +192.168.7.22 4 +192.168.7.21 0 +192.168.7.20 1 +192.168.7.19 2 +192.168.7.18 4 +192.168.7.17 3 +192.168.7.16 4 +192.168.7.15 0 +192.168.7.14 1 +192.168.7.13 2 +192.168.7.12 3 +192.168.7.11 4 +192.168.7.10 3 +192.168.7.9 2 +192.168.7.8 0 +192.168.7.7 2 +192.168.7.6 4 +192.168.7.5 0 +192.168.7.4 1 +192.168.7.3 3 +192.168.7.2 4 +192.168.7.1 1 +192.168.6.90 0 +192.168.6.89 1 +192.168.6.88 2 +192.168.6.87 3 +192.168.6.86 4 +192.168.6.85 0 +192.168.6.84 1 +192.168.6.83 2 +192.168.6.82 4 +192.168.6.81 3 +192.168.6.80 0 +192.168.6.79 0 +192.168.6.78 1 +192.168.6.77 2 +192.168.6.76 3 +192.168.6.75 4 +192.168.6.74 2 +192.168.6.73 3 +192.168.6.72 1 +192.168.6.71 3 +192.168.6.70 4 +192.168.6.69 0 +192.168.6.68 1 +192.168.6.67 2 +192.168.6.66 4 +192.168.6.65 0 +192.168.6.64 1 +192.168.6.63 0 +192.168.6.62 1 +192.168.6.61 2 +192.168.6.60 3 +192.168.6.59 4 +192.168.6.58 2 +192.168.6.57 3 +192.168.6.56 0 +192.168.6.55 3 +192.168.6.54 4 +192.168.6.53 1 +192.168.6.52 2 +192.168.6.51 0 +192.168.6.50 4 +192.168.6.49 1 +192.168.6.48 2 +192.168.6.47 0 +192.168.6.46 1 +192.168.6.45 2 +192.168.6.44 3 +192.168.6.43 4 +192.168.6.42 2 +192.168.6.41 4 +192.168.6.40 3 +192.168.6.39 0 +192.168.6.38 1 +192.168.6.37 2 +192.168.6.36 3 +192.168.6.35 4 +192.168.6.34 0 +192.168.6.33 1 +192.168.6.32 4 +192.168.6.31 0 +192.168.6.30 1 +192.168.6.29 2 +192.168.6.28 3 +192.168.6.27 4 +192.168.6.26 2 +192.168.6.25 3 +192.168.6.24 0 +192.168.6.23 3 +192.168.6.22 4 +192.168.6.21 0 +192.168.6.20 1 +192.168.6.19 2 +192.168.6.18 4 +192.168.6.17 1 +192.168.6.16 3 +192.168.6.15 0 +192.168.6.14 1 +192.168.6.13 2 +192.168.6.12 3 +192.168.6.11 4 +192.168.6.10 2 +192.168.6.9 3 +192.168.6.8 4 +192.168.6.7 0 +192.168.6.6 1 +192.168.6.5 2 +192.168.6.4 3 +192.168.6.3 4 +192.168.6.2 0 +192.168.6.1 1 +192.168.5.90 0 +192.168.5.89 1 +192.168.5.88 2 +192.168.5.87 3 +192.168.5.86 4 +192.168.5.85 0 +192.168.5.84 1 +192.168.5.83 2 +192.168.5.82 4 +192.168.5.81 3 +192.168.5.80 0 +192.168.5.79 0 +192.168.5.78 1 +192.168.5.77 2 +192.168.5.76 3 +192.168.5.75 4 +192.168.5.74 2 +192.168.5.73 3 +192.168.5.72 1 +192.168.5.71 3 +192.168.5.70 4 +192.168.5.69 2 +192.168.5.68 0 +192.168.5.67 1 +192.168.5.66 4 +192.168.5.65 2 +192.168.5.64 0 +192.168.5.63 0 +192.168.5.62 1 +192.168.5.61 2 +192.168.5.60 3 +192.168.5.59 4 +192.168.5.58 1 +192.168.5.57 3 +192.168.5.56 2 +192.168.5.55 0 +192.168.5.54 1 +192.168.5.53 2 +192.168.5.52 3 +192.168.5.51 4 +192.168.5.50 0 +192.168.5.49 4 +192.168.5.48 1 +192.168.5.47 0 +192.168.5.46 1 +192.168.5.45 2 +192.168.5.44 3 +192.168.5.43 4 +192.168.5.42 1 +192.168.5.41 3 +192.168.5.40 2 +192.168.5.39 2 +192.168.5.38 3 +192.168.5.37 4 +192.168.5.36 0 +192.168.5.35 1 +192.168.5.34 4 +192.168.5.33 0 +192.168.5.32 4 +192.168.5.31 0 +192.168.5.30 1 +192.168.5.29 2 +192.168.5.28 3 +192.168.5.27 4 +192.168.5.26 1 +192.168.5.25 3 +192.168.5.24 2 +192.168.5.23 3 +192.168.5.22 4 +192.168.5.21 2 +192.168.5.20 0 +192.168.5.19 1 +192.168.5.18 4 +192.168.5.17 0 +192.168.5.16 3 +192.168.5.15 0 +192.168.5.14 1 +192.168.5.13 2 +192.168.5.12 3 +192.168.5.11 4 +192.168.5.10 1 +192.168.5.9 4 +192.168.5.8 3 +192.168.5.7 0 +192.168.5.6 1 +192.168.5.5 2 +192.168.5.4 3 +192.168.5.3 4 +192.168.5.2 2 +192.168.5.1 0 +192.168.4.90 0 +192.168.4.89 1 +192.168.4.88 2 +192.168.4.87 3 +192.168.4.86 4 +192.168.4.85 0 +192.168.4.84 1 +192.168.4.83 2 +192.168.4.82 3 +192.168.4.81 4 +192.168.4.80 0 +192.168.4.79 0 +192.168.4.78 1 +192.168.4.77 2 +192.168.4.76 3 +192.168.4.75 4 +192.168.4.74 1 +192.168.4.73 2 +192.168.4.72 3 +192.168.4.71 3 +192.168.4.70 4 +192.168.4.69 0 +192.168.4.68 1 +192.168.4.67 2 +192.168.4.66 4 +192.168.4.65 1 +192.168.4.64 3 +192.168.4.63 0 +192.168.4.62 1 +192.168.4.61 2 +192.168.4.60 3 +192.168.4.59 4 +192.168.4.58 0 +192.168.4.57 2 +192.168.4.56 1 +192.168.4.55 0 +192.168.4.54 1 +192.168.4.53 2 +192.168.4.52 3 +192.168.4.51 4 +192.168.4.50 3 +192.168.4.49 4 +192.168.4.48 0 +192.168.4.47 0 +192.168.4.46 1 +192.168.4.45 2 +192.168.4.44 3 +192.168.4.43 4 +192.168.4.42 2 +192.168.4.41 0 +192.168.4.40 1 +192.168.4.39 4 +192.168.4.38 0 +192.168.4.37 1 +192.168.4.36 2 +192.168.4.35 3 +192.168.4.34 4 +192.168.4.33 3 +192.168.4.32 2 +192.168.4.31 0 +192.168.4.30 1 +192.168.4.29 2 +192.168.4.28 3 +192.168.4.27 4 +192.168.4.26 0 +192.168.4.25 2 +192.168.4.24 1 +192.168.4.23 3 +192.168.4.22 4 +192.168.4.21 0 +192.168.4.20 1 +192.168.4.19 2 +192.168.4.18 4 +192.168.4.17 3 +192.168.4.16 1 +192.168.4.15 0 +192.168.4.14 1 +192.168.4.13 2 +192.168.4.12 3 +192.168.4.11 4 +192.168.4.10 3 +192.168.4.9 0 +192.168.4.8 2 +192.168.4.7 2 +192.168.4.6 3 +192.168.4.5 4 +192.168.4.4 0 +192.168.4.3 1 +192.168.4.2 4 +192.168.4.1 4 +192.168.3.90 0 +192.168.3.89 1 +192.168.3.88 2 +192.168.3.87 3 +192.168.3.86 4 +192.168.3.85 0 +192.168.3.84 1 +192.168.3.83 2 +192.168.3.82 3 +192.168.3.81 4 +192.168.3.80 0 +192.168.3.79 0 +192.168.3.78 1 +192.168.3.77 2 +192.168.3.76 3 +192.168.3.75 4 +192.168.3.74 1 +192.168.3.73 2 +192.168.3.72 3 +192.168.3.71 3 +192.168.3.70 4 +192.168.3.69 0 +192.168.3.68 1 +192.168.3.67 2 +192.168.3.66 4 +192.168.3.65 0 +192.168.3.64 3 +192.168.3.63 0 +192.168.3.62 1 +192.168.3.61 2 +192.168.3.60 3 +192.168.3.59 4 +192.168.3.58 2 +192.168.3.57 1 +192.168.3.56 3 +192.168.3.55 0 +192.168.3.54 1 +192.168.3.53 2 +192.168.3.52 3 +192.168.3.51 4 +192.168.3.50 0 +192.168.3.49 4 +192.168.3.48 2 +192.168.3.47 0 +192.168.3.46 1 +192.168.3.45 2 +192.168.3.44 3 +192.168.3.43 4 +192.168.3.42 2 +192.168.3.41 1 +192.168.3.40 0 +192.168.3.39 1 +192.168.3.38 2 +192.168.3.37 3 +192.168.3.36 4 +192.168.3.35 0 +192.168.3.34 4 +192.168.3.33 3 +192.168.3.32 4 +192.168.3.31 0 +192.168.3.30 1 +192.168.3.29 2 +192.168.3.28 3 +192.168.3.27 4 +192.168.3.26 2 +192.168.3.25 1 +192.168.3.24 0 +192.168.3.23 3 +192.168.3.22 4 +192.168.3.21 0 +192.168.3.20 1 +192.168.3.19 2 +192.168.3.18 4 +192.168.3.17 3 +192.168.3.16 1 +192.168.3.15 0 +192.168.3.14 1 +192.168.3.13 2 +192.168.3.12 3 +192.168.3.11 4 +192.168.3.10 2 +192.168.3.9 1 +192.168.3.8 0 +192.168.3.7 4 +192.168.3.6 0 +192.168.3.5 1 +192.168.3.4 2 +192.168.3.3 3 +192.168.3.2 4 +192.168.3.1 3 +192.168.2.90 0 +192.168.2.89 1 +192.168.2.88 2 +192.168.2.87 3 +192.168.2.86 4 +192.168.2.85 0 +192.168.2.84 1 +192.168.2.83 2 +192.168.2.82 3 +192.168.2.81 4 +192.168.2.80 1 +192.168.2.79 0 +192.168.2.78 1 +192.168.2.77 2 +192.168.2.76 3 +192.168.2.75 4 +192.168.2.74 2 +192.168.2.73 3 +192.168.2.72 0 +192.168.2.71 3 +192.168.2.70 4 +192.168.2.69 0 +192.168.2.68 1 +192.168.2.67 2 +192.168.2.66 4 +192.168.2.65 1 +192.168.2.64 3 +192.168.2.63 0 +192.168.2.62 1 +192.168.2.61 2 +192.168.2.60 3 +192.168.2.59 4 +192.168.2.58 0 +192.168.2.57 2 +192.168.2.56 1 +192.168.2.55 0 +192.168.2.54 1 +192.168.2.53 2 +192.168.2.52 3 +192.168.2.51 4 +192.168.2.50 3 +192.168.2.49 4 +192.168.2.48 0 +192.168.2.47 0 +192.168.2.46 1 +192.168.2.45 2 +192.168.2.44 3 +192.168.2.43 4 +192.168.2.42 2 +192.168.2.41 0 +192.168.2.40 1 +192.168.2.39 0 +192.168.2.38 1 +192.168.2.37 2 +192.168.2.36 3 +192.168.2.35 4 +192.168.2.34 3 +192.168.2.33 4 +192.168.2.32 2 +192.168.2.31 0 +192.168.2.30 1 +192.168.2.29 2 +192.168.2.28 3 +192.168.2.27 4 +192.168.2.26 2 +192.168.2.25 0 +192.168.2.24 1 +192.168.2.23 3 +192.168.2.22 4 +192.168.2.21 0 +192.168.2.20 1 +192.168.2.19 2 +192.168.2.18 4 +192.168.2.17 3 +192.168.2.16 4 +192.168.2.15 0 +192.168.2.14 1 +192.168.2.13 2 +192.168.2.12 3 +192.168.2.11 4 +192.168.2.10 0 +192.168.2.9 2 +192.168.2.8 3 +192.168.2.7 2 +192.168.2.6 4 +192.168.2.5 0 +192.168.2.4 1 +192.168.2.3 3 +192.168.2.2 4 +192.168.2.1 1 +192.168.1.90 0 +192.168.1.89 1 +192.168.1.88 2 +192.168.1.87 3 +192.168.1.86 4 +192.168.1.85 0 +192.168.1.84 1 +192.168.1.83 2 +192.168.1.82 3 +192.168.1.81 4 +192.168.1.80 0 +192.168.1.79 0 +192.168.1.78 1 +192.168.1.77 2 +192.168.1.76 3 +192.168.1.75 4 +192.168.1.74 1 +192.168.1.73 2 +192.168.1.72 3 +192.168.1.71 3 +192.168.1.70 4 +192.168.1.69 0 +192.168.1.68 1 +192.168.1.67 2 +192.168.1.66 4 +192.168.1.65 0 +192.168.1.64 1 +192.168.1.63 0 +192.168.1.62 1 +192.168.1.61 2 +192.168.1.60 3 +192.168.1.59 4 +192.168.1.58 2 +192.168.1.57 3 +192.168.1.56 1 +192.168.1.55 0 +192.168.1.54 1 +192.168.1.53 2 +192.168.1.52 3 +192.168.1.51 4 +192.168.1.50 0 +192.168.1.49 4 +192.168.1.48 2 +192.168.1.47 0 +192.168.1.46 1 +192.168.1.45 2 +192.168.1.44 3 +192.168.1.43 4 +192.168.1.42 2 +192.168.1.41 3 +192.168.1.40 0 +192.168.1.39 3 +192.168.1.38 4 +192.168.1.37 0 +192.168.1.36 1 +192.168.1.35 2 +192.168.1.34 4 +192.168.1.33 1 +192.168.1.32 3 +192.168.1.31 0 +192.168.1.30 1 +192.168.1.29 2 +192.168.1.28 3 +192.168.1.27 4 +192.168.1.26 2 +192.168.1.25 3 +192.168.1.24 0 +192.168.1.23 3 +192.168.1.22 4 +192.168.1.21 0 +192.168.1.20 1 +192.168.1.19 2 +192.168.1.18 4 +192.168.1.17 1 +192.168.1.16 4 +192.168.1.15 0 +192.168.1.14 1 +192.168.1.13 2 +192.168.1.12 3 +192.168.1.11 4 +192.168.1.10 2 +192.168.1.9 3 +192.168.1.8 0 +192.168.1.7 3 +192.168.1.6 4 +192.168.1.5 0 +192.168.1.4 1 +192.168.1.3 2 +192.168.1.2 4 +192.168.1.1 1 +EOF + +simple_test 0,0,0,0,0 <<EOF +192.168.1.1 -1 +192.168.1.2 -1 +192.168.1.3 -1 +192.168.1.4 -1 +192.168.1.5 -1 +192.168.1.6 -1 +192.168.1.7 -1 +192.168.1.8 -1 +192.168.1.9 -1 +192.168.1.10 -1 +192.168.1.11 -1 +192.168.1.12 -1 +192.168.1.13 -1 +192.168.1.14 -1 +192.168.1.15 -1 +192.168.1.16 -1 +192.168.1.17 -1 +192.168.1.18 -1 +192.168.1.19 -1 +192.168.1.20 -1 +192.168.1.21 -1 +192.168.1.22 -1 +192.168.1.23 -1 +192.168.1.24 -1 +192.168.1.25 -1 +192.168.1.26 -1 +192.168.1.27 -1 +192.168.1.28 -1 +192.168.1.29 -1 +192.168.1.30 -1 +192.168.1.31 -1 +192.168.1.32 -1 +192.168.1.33 -1 +192.168.1.34 -1 +192.168.1.35 -1 +192.168.1.36 -1 +192.168.1.37 -1 +192.168.1.38 -1 +192.168.1.39 -1 +192.168.1.40 -1 +192.168.1.41 -1 +192.168.1.42 -1 +192.168.1.43 -1 +192.168.1.44 -1 +192.168.1.45 -1 +192.168.1.46 -1 +192.168.1.47 -1 +192.168.1.48 -1 +192.168.1.49 -1 +192.168.1.50 -1 +192.168.1.51 -1 +192.168.1.52 -1 +192.168.1.53 -1 +192.168.1.54 -1 +192.168.1.55 -1 +192.168.1.56 -1 +192.168.1.57 -1 +192.168.1.58 -1 +192.168.1.59 -1 +192.168.1.60 -1 +192.168.1.61 -1 +192.168.1.62 -1 +192.168.1.63 -1 +192.168.1.64 -1 +192.168.1.65 -1 +192.168.1.66 -1 +192.168.1.67 -1 +192.168.1.68 -1 +192.168.1.69 -1 +192.168.1.70 -1 +192.168.1.71 -1 +192.168.1.72 -1 +192.168.1.73 -1 +192.168.1.74 -1 +192.168.1.75 -1 +192.168.1.76 -1 +192.168.1.77 -1 +192.168.1.78 -1 +192.168.1.79 -1 +192.168.1.80 -1 +192.168.1.81 -1 +192.168.1.82 -1 +192.168.1.83 -1 +192.168.1.84 -1 +192.168.1.85 -1 +192.168.1.86 -1 +192.168.1.87 -1 +192.168.1.88 -1 +192.168.1.89 -1 +192.168.1.90 -1 +192.168.2.1 -1 +192.168.2.2 -1 +192.168.2.3 -1 +192.168.2.4 -1 +192.168.2.5 -1 +192.168.2.6 -1 +192.168.2.7 -1 +192.168.2.8 -1 +192.168.2.9 -1 +192.168.2.10 -1 +192.168.2.11 -1 +192.168.2.12 -1 +192.168.2.13 -1 +192.168.2.14 -1 +192.168.2.15 -1 +192.168.2.16 -1 +192.168.2.17 -1 +192.168.2.18 -1 +192.168.2.19 -1 +192.168.2.20 -1 +192.168.2.21 -1 +192.168.2.22 -1 +192.168.2.23 -1 +192.168.2.24 -1 +192.168.2.25 -1 +192.168.2.26 -1 +192.168.2.27 -1 +192.168.2.28 -1 +192.168.2.29 -1 +192.168.2.30 -1 +192.168.2.31 -1 +192.168.2.32 -1 +192.168.2.33 -1 +192.168.2.34 -1 +192.168.2.35 -1 +192.168.2.36 -1 +192.168.2.37 -1 +192.168.2.38 -1 +192.168.2.39 -1 +192.168.2.40 -1 +192.168.2.41 -1 +192.168.2.42 -1 +192.168.2.43 -1 +192.168.2.44 -1 +192.168.2.45 -1 +192.168.2.46 -1 +192.168.2.47 -1 +192.168.2.48 -1 +192.168.2.49 -1 +192.168.2.50 -1 +192.168.2.51 -1 +192.168.2.52 -1 +192.168.2.53 -1 +192.168.2.54 -1 +192.168.2.55 -1 +192.168.2.56 -1 +192.168.2.57 -1 +192.168.2.58 -1 +192.168.2.59 -1 +192.168.2.60 -1 +192.168.2.61 -1 +192.168.2.62 -1 +192.168.2.63 -1 +192.168.2.64 -1 +192.168.2.65 -1 +192.168.2.66 -1 +192.168.2.67 -1 +192.168.2.68 -1 +192.168.2.69 -1 +192.168.2.70 -1 +192.168.2.71 -1 +192.168.2.72 -1 +192.168.2.73 -1 +192.168.2.74 -1 +192.168.2.75 -1 +192.168.2.76 -1 +192.168.2.77 -1 +192.168.2.78 -1 +192.168.2.79 -1 +192.168.2.80 -1 +192.168.2.81 -1 +192.168.2.82 -1 +192.168.2.83 -1 +192.168.2.84 -1 +192.168.2.85 -1 +192.168.2.86 -1 +192.168.2.87 -1 +192.168.2.88 -1 +192.168.2.89 -1 +192.168.2.90 -1 +192.168.3.1 -1 +192.168.3.2 -1 +192.168.3.3 -1 +192.168.3.4 -1 +192.168.3.5 -1 +192.168.3.6 -1 +192.168.3.7 -1 +192.168.3.8 -1 +192.168.3.9 -1 +192.168.3.10 -1 +192.168.3.11 -1 +192.168.3.12 -1 +192.168.3.13 -1 +192.168.3.14 -1 +192.168.3.15 -1 +192.168.3.16 -1 +192.168.3.17 -1 +192.168.3.18 -1 +192.168.3.19 -1 +192.168.3.20 -1 +192.168.3.21 -1 +192.168.3.22 -1 +192.168.3.23 -1 +192.168.3.24 -1 +192.168.3.25 -1 +192.168.3.26 -1 +192.168.3.27 -1 +192.168.3.28 -1 +192.168.3.29 -1 +192.168.3.30 -1 +192.168.3.31 -1 +192.168.3.32 -1 +192.168.3.33 -1 +192.168.3.34 -1 +192.168.3.35 -1 +192.168.3.36 -1 +192.168.3.37 -1 +192.168.3.38 -1 +192.168.3.39 -1 +192.168.3.40 -1 +192.168.3.41 -1 +192.168.3.42 -1 +192.168.3.43 -1 +192.168.3.44 -1 +192.168.3.45 -1 +192.168.3.46 -1 +192.168.3.47 -1 +192.168.3.48 -1 +192.168.3.49 -1 +192.168.3.50 -1 +192.168.3.51 -1 +192.168.3.52 -1 +192.168.3.53 -1 +192.168.3.54 -1 +192.168.3.55 -1 +192.168.3.56 -1 +192.168.3.57 -1 +192.168.3.58 -1 +192.168.3.59 -1 +192.168.3.60 -1 +192.168.3.61 -1 +192.168.3.62 -1 +192.168.3.63 -1 +192.168.3.64 -1 +192.168.3.65 -1 +192.168.3.66 -1 +192.168.3.67 -1 +192.168.3.68 -1 +192.168.3.69 -1 +192.168.3.70 -1 +192.168.3.71 -1 +192.168.3.72 -1 +192.168.3.73 -1 +192.168.3.74 -1 +192.168.3.75 -1 +192.168.3.76 -1 +192.168.3.77 -1 +192.168.3.78 -1 +192.168.3.79 -1 +192.168.3.80 -1 +192.168.3.81 -1 +192.168.3.82 -1 +192.168.3.83 -1 +192.168.3.84 -1 +192.168.3.85 -1 +192.168.3.86 -1 +192.168.3.87 -1 +192.168.3.88 -1 +192.168.3.89 -1 +192.168.3.90 -1 +192.168.4.1 -1 +192.168.4.2 -1 +192.168.4.3 -1 +192.168.4.4 -1 +192.168.4.5 -1 +192.168.4.6 -1 +192.168.4.7 -1 +192.168.4.8 -1 +192.168.4.9 -1 +192.168.4.10 -1 +192.168.4.11 -1 +192.168.4.12 -1 +192.168.4.13 -1 +192.168.4.14 -1 +192.168.4.15 -1 +192.168.4.16 -1 +192.168.4.17 -1 +192.168.4.18 -1 +192.168.4.19 -1 +192.168.4.20 -1 +192.168.4.21 -1 +192.168.4.22 -1 +192.168.4.23 -1 +192.168.4.24 -1 +192.168.4.25 -1 +192.168.4.26 -1 +192.168.4.27 -1 +192.168.4.28 -1 +192.168.4.29 -1 +192.168.4.30 -1 +192.168.4.31 -1 +192.168.4.32 -1 +192.168.4.33 -1 +192.168.4.34 -1 +192.168.4.35 -1 +192.168.4.36 -1 +192.168.4.37 -1 +192.168.4.38 -1 +192.168.4.39 -1 +192.168.4.40 -1 +192.168.4.41 -1 +192.168.4.42 -1 +192.168.4.43 -1 +192.168.4.44 -1 +192.168.4.45 -1 +192.168.4.46 -1 +192.168.4.47 -1 +192.168.4.48 -1 +192.168.4.49 -1 +192.168.4.50 -1 +192.168.4.51 -1 +192.168.4.52 -1 +192.168.4.53 -1 +192.168.4.54 -1 +192.168.4.55 -1 +192.168.4.56 -1 +192.168.4.57 -1 +192.168.4.58 -1 +192.168.4.59 -1 +192.168.4.60 -1 +192.168.4.61 -1 +192.168.4.62 -1 +192.168.4.63 -1 +192.168.4.64 -1 +192.168.4.65 -1 +192.168.4.66 -1 +192.168.4.67 -1 +192.168.4.68 -1 +192.168.4.69 -1 +192.168.4.70 -1 +192.168.4.71 -1 +192.168.4.72 -1 +192.168.4.73 -1 +192.168.4.74 -1 +192.168.4.75 -1 +192.168.4.76 -1 +192.168.4.77 -1 +192.168.4.78 -1 +192.168.4.79 -1 +192.168.4.80 -1 +192.168.4.81 -1 +192.168.4.82 -1 +192.168.4.83 -1 +192.168.4.84 -1 +192.168.4.85 -1 +192.168.4.86 -1 +192.168.4.87 -1 +192.168.4.88 -1 +192.168.4.89 -1 +192.168.4.90 -1 +192.168.5.1 -1 +192.168.5.2 -1 +192.168.5.3 -1 +192.168.5.4 -1 +192.168.5.5 -1 +192.168.5.6 -1 +192.168.5.7 -1 +192.168.5.8 -1 +192.168.5.9 -1 +192.168.5.10 -1 +192.168.5.11 -1 +192.168.5.12 -1 +192.168.5.13 -1 +192.168.5.14 -1 +192.168.5.15 -1 +192.168.5.16 -1 +192.168.5.17 -1 +192.168.5.18 -1 +192.168.5.19 -1 +192.168.5.20 -1 +192.168.5.21 -1 +192.168.5.22 -1 +192.168.5.23 -1 +192.168.5.24 -1 +192.168.5.25 -1 +192.168.5.26 -1 +192.168.5.27 -1 +192.168.5.28 -1 +192.168.5.29 -1 +192.168.5.30 -1 +192.168.5.31 -1 +192.168.5.32 -1 +192.168.5.33 -1 +192.168.5.34 -1 +192.168.5.35 -1 +192.168.5.36 -1 +192.168.5.37 -1 +192.168.5.38 -1 +192.168.5.39 -1 +192.168.5.40 -1 +192.168.5.41 -1 +192.168.5.42 -1 +192.168.5.43 -1 +192.168.5.44 -1 +192.168.5.45 -1 +192.168.5.46 -1 +192.168.5.47 -1 +192.168.5.48 -1 +192.168.5.49 -1 +192.168.5.50 -1 +192.168.5.51 -1 +192.168.5.52 -1 +192.168.5.53 -1 +192.168.5.54 -1 +192.168.5.55 -1 +192.168.5.56 -1 +192.168.5.57 -1 +192.168.5.58 -1 +192.168.5.59 -1 +192.168.5.60 -1 +192.168.5.61 -1 +192.168.5.62 -1 +192.168.5.63 -1 +192.168.5.64 -1 +192.168.5.65 -1 +192.168.5.66 -1 +192.168.5.67 -1 +192.168.5.68 -1 +192.168.5.69 -1 +192.168.5.70 -1 +192.168.5.71 -1 +192.168.5.72 -1 +192.168.5.73 -1 +192.168.5.74 -1 +192.168.5.75 -1 +192.168.5.76 -1 +192.168.5.77 -1 +192.168.5.78 -1 +192.168.5.79 -1 +192.168.5.80 -1 +192.168.5.81 -1 +192.168.5.82 -1 +192.168.5.83 -1 +192.168.5.84 -1 +192.168.5.85 -1 +192.168.5.86 -1 +192.168.5.87 -1 +192.168.5.88 -1 +192.168.5.89 -1 +192.168.5.90 -1 +192.168.6.1 -1 +192.168.6.2 -1 +192.168.6.3 -1 +192.168.6.4 -1 +192.168.6.5 -1 +192.168.6.6 -1 +192.168.6.7 -1 +192.168.6.8 -1 +192.168.6.9 -1 +192.168.6.10 -1 +192.168.6.11 -1 +192.168.6.12 -1 +192.168.6.13 -1 +192.168.6.14 -1 +192.168.6.15 -1 +192.168.6.16 -1 +192.168.6.17 -1 +192.168.6.18 -1 +192.168.6.19 -1 +192.168.6.20 -1 +192.168.6.21 -1 +192.168.6.22 -1 +192.168.6.23 -1 +192.168.6.24 -1 +192.168.6.25 -1 +192.168.6.26 -1 +192.168.6.27 -1 +192.168.6.28 -1 +192.168.6.29 -1 +192.168.6.30 -1 +192.168.6.31 -1 +192.168.6.32 -1 +192.168.6.33 -1 +192.168.6.34 -1 +192.168.6.35 -1 +192.168.6.36 -1 +192.168.6.37 -1 +192.168.6.38 -1 +192.168.6.39 -1 +192.168.6.40 -1 +192.168.6.41 -1 +192.168.6.42 -1 +192.168.6.43 -1 +192.168.6.44 -1 +192.168.6.45 -1 +192.168.6.46 -1 +192.168.6.47 -1 +192.168.6.48 -1 +192.168.6.49 -1 +192.168.6.50 -1 +192.168.6.51 -1 +192.168.6.52 -1 +192.168.6.53 -1 +192.168.6.54 -1 +192.168.6.55 -1 +192.168.6.56 -1 +192.168.6.57 -1 +192.168.6.58 -1 +192.168.6.59 -1 +192.168.6.60 -1 +192.168.6.61 -1 +192.168.6.62 -1 +192.168.6.63 -1 +192.168.6.64 -1 +192.168.6.65 -1 +192.168.6.66 -1 +192.168.6.67 -1 +192.168.6.68 -1 +192.168.6.69 -1 +192.168.6.70 -1 +192.168.6.71 -1 +192.168.6.72 -1 +192.168.6.73 -1 +192.168.6.74 -1 +192.168.6.75 -1 +192.168.6.76 -1 +192.168.6.77 -1 +192.168.6.78 -1 +192.168.6.79 -1 +192.168.6.80 -1 +192.168.6.81 -1 +192.168.6.82 -1 +192.168.6.83 -1 +192.168.6.84 -1 +192.168.6.85 -1 +192.168.6.86 -1 +192.168.6.87 -1 +192.168.6.88 -1 +192.168.6.89 -1 +192.168.6.90 -1 +192.168.7.1 -1 +192.168.7.2 -1 +192.168.7.3 -1 +192.168.7.4 -1 +192.168.7.5 -1 +192.168.7.6 -1 +192.168.7.7 -1 +192.168.7.8 -1 +192.168.7.9 -1 +192.168.7.10 -1 +192.168.7.11 -1 +192.168.7.12 -1 +192.168.7.13 -1 +192.168.7.14 -1 +192.168.7.15 -1 +192.168.7.16 -1 +192.168.7.17 -1 +192.168.7.18 -1 +192.168.7.19 -1 +192.168.7.20 -1 +192.168.7.21 -1 +192.168.7.22 -1 +192.168.7.23 -1 +192.168.7.24 -1 +192.168.7.25 -1 +192.168.7.26 -1 +192.168.7.27 -1 +192.168.7.28 -1 +192.168.7.29 -1 +192.168.7.30 -1 +192.168.7.31 -1 +192.168.7.32 -1 +192.168.7.33 -1 +192.168.7.34 -1 +192.168.7.35 -1 +192.168.7.36 -1 +192.168.7.37 -1 +192.168.7.38 -1 +192.168.7.39 -1 +192.168.7.40 -1 +192.168.7.41 -1 +192.168.7.42 -1 +192.168.7.43 -1 +192.168.7.44 -1 +192.168.7.45 -1 +192.168.7.46 -1 +192.168.7.47 -1 +192.168.7.48 -1 +192.168.7.49 -1 +192.168.7.50 -1 +192.168.7.51 -1 +192.168.7.52 -1 +192.168.7.53 -1 +192.168.7.54 -1 +192.168.7.55 -1 +192.168.7.56 -1 +192.168.7.57 -1 +192.168.7.58 -1 +192.168.7.59 -1 +192.168.7.60 -1 +192.168.7.61 -1 +192.168.7.62 -1 +192.168.7.63 -1 +192.168.7.64 -1 +192.168.7.65 -1 +192.168.7.66 -1 +192.168.7.67 -1 +192.168.7.68 -1 +192.168.7.69 -1 +192.168.7.70 -1 +192.168.7.71 -1 +192.168.7.72 -1 +192.168.7.73 -1 +192.168.7.74 -1 +192.168.7.75 -1 +192.168.7.76 -1 +192.168.7.77 -1 +192.168.7.78 -1 +192.168.7.79 -1 +192.168.7.80 -1 +192.168.7.81 -1 +192.168.7.82 -1 +192.168.7.83 -1 +192.168.7.84 -1 +192.168.7.85 -1 +192.168.7.86 -1 +192.168.7.87 -1 +192.168.7.88 -1 +192.168.7.89 -1 +192.168.7.90 -1 +192.168.8.1 -1 +192.168.8.2 -1 +192.168.8.3 -1 +192.168.8.4 -1 +192.168.8.5 -1 +192.168.8.6 -1 +192.168.8.7 -1 +192.168.8.8 -1 +192.168.8.9 -1 +192.168.8.10 -1 +192.168.8.11 -1 +192.168.8.12 -1 +192.168.8.13 -1 +192.168.8.14 -1 +192.168.8.15 -1 +192.168.8.16 -1 +192.168.8.17 -1 +192.168.8.18 -1 +192.168.8.19 -1 +192.168.8.20 -1 +192.168.8.21 -1 +192.168.8.22 -1 +192.168.8.23 -1 +192.168.8.24 -1 +192.168.8.25 -1 +192.168.8.26 -1 +192.168.8.27 -1 +192.168.8.28 -1 +192.168.8.29 -1 +192.168.8.30 -1 +192.168.8.31 -1 +192.168.8.32 -1 +192.168.8.33 -1 +192.168.8.34 -1 +192.168.8.35 -1 +192.168.8.36 -1 +192.168.8.37 -1 +192.168.8.38 -1 +192.168.8.39 -1 +192.168.8.40 -1 +192.168.8.41 -1 +192.168.8.42 -1 +192.168.8.43 -1 +192.168.8.44 -1 +192.168.8.45 -1 +192.168.8.46 -1 +192.168.8.47 -1 +192.168.8.48 -1 +192.168.8.49 -1 +192.168.8.50 -1 +192.168.8.51 -1 +192.168.8.52 -1 +192.168.8.53 -1 +192.168.8.54 -1 +192.168.8.55 -1 +192.168.8.56 -1 +192.168.8.57 -1 +192.168.8.58 -1 +192.168.8.59 -1 +192.168.8.60 -1 +192.168.8.61 -1 +192.168.8.62 -1 +192.168.8.63 -1 +192.168.8.64 -1 +192.168.8.65 -1 +192.168.8.66 -1 +192.168.8.67 -1 +192.168.8.68 -1 +192.168.8.69 -1 +192.168.8.70 -1 +192.168.8.71 -1 +192.168.8.72 -1 +192.168.8.73 -1 +192.168.8.74 -1 +192.168.8.75 -1 +192.168.8.76 -1 +192.168.8.77 -1 +192.168.8.78 -1 +192.168.8.79 -1 +192.168.8.80 -1 +192.168.8.81 -1 +192.168.8.82 -1 +192.168.8.83 -1 +192.168.8.84 -1 +192.168.8.85 -1 +192.168.8.86 -1 +192.168.8.87 -1 +192.168.8.88 -1 +192.168.8.89 -1 +192.168.8.90 -1 +192.168.9.1 -1 +192.168.9.2 -1 +192.168.9.3 -1 +192.168.9.4 -1 +192.168.9.5 -1 +192.168.9.6 -1 +192.168.9.7 -1 +192.168.9.8 -1 +192.168.9.9 -1 +192.168.9.10 -1 +192.168.9.11 -1 +192.168.9.12 -1 +192.168.9.13 -1 +192.168.9.14 -1 +192.168.9.15 -1 +192.168.9.16 -1 +192.168.9.17 -1 +192.168.9.18 -1 +192.168.9.19 -1 +192.168.9.20 -1 +192.168.9.21 -1 +192.168.9.22 -1 +192.168.9.23 -1 +192.168.9.24 -1 +192.168.9.25 -1 +192.168.9.26 -1 +192.168.9.27 -1 +192.168.9.28 -1 +192.168.9.29 -1 +192.168.9.30 -1 +192.168.9.31 -1 +192.168.9.32 -1 +192.168.9.33 -1 +192.168.9.34 -1 +192.168.9.35 -1 +192.168.9.36 -1 +192.168.9.37 -1 +192.168.9.38 -1 +192.168.9.39 -1 +192.168.9.40 -1 +192.168.9.41 -1 +192.168.9.42 -1 +192.168.9.43 -1 +192.168.9.44 -1 +192.168.9.45 -1 +192.168.9.46 -1 +192.168.9.47 -1 +192.168.9.48 -1 +192.168.9.49 -1 +192.168.9.50 -1 +192.168.9.51 -1 +192.168.9.52 -1 +192.168.9.53 -1 +192.168.9.54 -1 +192.168.9.55 -1 +192.168.9.56 -1 +192.168.9.57 -1 +192.168.9.58 -1 +192.168.9.59 -1 +192.168.9.60 -1 +192.168.9.61 -1 +192.168.9.62 -1 +192.168.9.63 -1 +192.168.9.64 -1 +192.168.9.65 -1 +192.168.9.66 -1 +192.168.9.67 -1 +192.168.9.68 -1 +192.168.9.69 -1 +192.168.9.70 -1 +192.168.9.71 -1 +192.168.9.72 -1 +192.168.9.73 -1 +192.168.9.74 -1 +192.168.9.75 -1 +192.168.9.76 -1 +192.168.9.77 -1 +192.168.9.78 -1 +192.168.9.79 -1 +192.168.9.80 -1 +192.168.9.81 -1 +192.168.9.82 -1 +192.168.9.83 -1 +192.168.9.84 -1 +192.168.9.85 -1 +192.168.9.86 -1 +192.168.9.87 -1 +192.168.9.88 -1 +192.168.9.89 -1 +192.168.9.90 -1 +192.168.10.1 -1 +192.168.10.2 -1 +192.168.10.3 -1 +192.168.10.4 -1 +192.168.10.5 -1 +192.168.10.6 -1 +192.168.10.7 -1 +192.168.10.8 -1 +192.168.10.9 -1 +192.168.10.10 -1 +192.168.10.11 -1 +192.168.10.12 -1 +192.168.10.13 -1 +192.168.10.14 -1 +192.168.10.15 -1 +192.168.10.16 -1 +192.168.10.17 -1 +192.168.10.18 -1 +192.168.10.19 -1 +192.168.10.20 -1 +192.168.10.21 -1 +192.168.10.22 -1 +192.168.10.23 -1 +192.168.10.24 -1 +192.168.10.25 -1 +192.168.10.26 -1 +192.168.10.27 -1 +192.168.10.28 -1 +192.168.10.29 -1 +192.168.10.30 -1 +192.168.10.31 -1 +192.168.10.32 -1 +192.168.10.33 -1 +192.168.10.34 -1 +192.168.10.35 -1 +192.168.10.36 -1 +192.168.10.37 -1 +192.168.10.38 -1 +192.168.10.39 -1 +192.168.10.40 -1 +192.168.10.41 -1 +192.168.10.42 -1 +192.168.10.43 -1 +192.168.10.44 -1 +192.168.10.45 -1 +192.168.10.46 -1 +192.168.10.47 -1 +192.168.10.48 -1 +192.168.10.49 -1 +192.168.10.50 -1 +192.168.10.51 -1 +192.168.10.52 -1 +192.168.10.53 -1 +192.168.10.54 -1 +192.168.10.55 -1 +192.168.10.56 -1 +192.168.10.57 -1 +192.168.10.58 -1 +192.168.10.59 -1 +192.168.10.60 -1 +192.168.10.61 -1 +192.168.10.62 -1 +192.168.10.63 -1 +192.168.10.64 -1 +192.168.10.65 -1 +192.168.10.66 -1 +192.168.10.67 -1 +192.168.10.68 -1 +192.168.10.69 -1 +192.168.10.70 -1 +192.168.10.71 -1 +192.168.10.72 -1 +192.168.10.73 -1 +192.168.10.74 -1 +192.168.10.75 -1 +192.168.10.76 -1 +192.168.10.77 -1 +192.168.10.78 -1 +192.168.10.79 -1 +192.168.10.80 -1 +192.168.10.81 -1 +192.168.10.82 -1 +192.168.10.83 -1 +192.168.10.84 -1 +192.168.10.85 -1 +192.168.10.86 -1 +192.168.10.87 -1 +192.168.10.88 -1 +192.168.10.89 -1 +192.168.10.90 -1 +EOF diff --git a/ctdb/tests/takeover/nondet.001.sh b/ctdb/tests/takeover/nondet.001.sh new file mode 100755 index 00000000000..6f79c3460d2 --- /dev/null +++ b/ctdb/tests/takeover/nondet.001.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 healthy" + +required_result <<EOF +DATE TIME [PID]: Unassign IP: 192.168.21.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.21.252 from 0 +DATE TIME [PID]: Unassign IP: 192.168.20.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.252 from 0 +DATE TIME [PID]: Unassign IP: 192.168.20.250 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.249 from 0 +192.168.21.254 2 +192.168.21.253 2 +192.168.21.252 2 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 2 +192.168.20.251 2 +192.168.20.250 2 +192.168.20.249 2 +EOF + +simple_test 2,2,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/nondet.002.sh b/ctdb/tests/takeover/nondet.002.sh new file mode 100755 index 00000000000..c46f6a237cd --- /dev/null +++ b/ctdb/tests/takeover/nondet.002.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 2 healthy" + +required_result <<EOF +DATE TIME [PID]: Unassign IP: 192.168.21.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.253 from 1 +DATE TIME [PID]: Unassign IP: 192.168.20.250 from 1 +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 0 +192.168.20.254 2 +192.168.20.253 2 +192.168.20.252 0 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 0 +EOF + +simple_test 0,2,0 <<EOF +192.168.20.249 0 +192.168.20.250 1 +192.168.20.251 2 +192.168.20.252 0 +192.168.20.253 1 +192.168.20.254 2 +192.168.21.252 0 +192.168.21.253 1 +192.168.21.254 2 +EOF diff --git a/ctdb/tests/takeover/nondet.003.sh b/ctdb/tests/takeover/nondet.003.sh new file mode 100755 index 00000000000..2a9dfb4679b --- /dev/null +++ b/ctdb/tests/takeover/nondet.003.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 -> all healthy" + +required_result <<EOF +192.168.21.254 0 +192.168.21.253 2 +192.168.21.252 0 +192.168.20.254 2 +192.168.20.253 0 +192.168.20.252 2 +192.168.20.251 1 +192.168.20.250 1 +192.168.20.249 1 +EOF + +simple_test 0,0,0 <<EOF +192.168.20.249 1 +192.168.20.250 1 +192.168.20.251 1 +192.168.20.252 1 +192.168.20.253 1 +192.168.20.254 1 +192.168.21.252 1 +192.168.21.253 1 +192.168.21.254 1 +EOF diff --git a/ctdb/tests/takeover/scripts/local.sh b/ctdb/tests/takeover/scripts/local.sh new file mode 100644 index 00000000000..3b69d14fc92 --- /dev/null +++ b/ctdb/tests/takeover/scripts/local.sh @@ -0,0 +1,26 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +test_prog="ctdb_takeover_tests ctdb_takeover_run_core" + +define_test () +{ + _f=$(basename "$0" ".sh") + + export CTDB_IP_ALGORITHM="${_f%%.*}" + case "$CTDB_IP_ALGORITHM" in + lcp2|nondet|det) : ;; + *) die "Unknown algorithm for testcase \"$_f\"" ;; + esac + + printf "%-12s - %s\n" "$_f" "$1" +} + +simple_test () +{ + # Do some filtering of the output to replace date/time. + OUT_FILTER='s@^[^\]]*\]:@DATE\ TIME\ \[PID\]:@' + + _out=$($VALGRIND $test_prog "$@" 2>&1) + + result_check "Algorithm: $CTDB_IP_ALGORITHM" +} diff --git a/ctdb/tests/takeover/simulation/README b/ctdb/tests/takeover/simulation/README new file mode 100644 index 00000000000..4a8267bf6e1 --- /dev/null +++ b/ctdb/tests/takeover/simulation/README @@ -0,0 +1,6 @@ +This contains a Python simulation of CTDB's IP reallocation algorithm. + +It is useful for experimenting with improvements. + +To use this on RHEL5 you'll need python2.6 from EPEL +<http://fedoraproject.org/wiki/EPEL>. diff --git a/ctdb/tests/takeover/simulation/ctdb_takeover.py b/ctdb/tests/takeover/simulation/ctdb_takeover.py new file mode 100755 index 00000000000..4b7ceef4682 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ctdb_takeover.py @@ -0,0 +1,888 @@ +#!/usr/bin/env python + +# ctdb ip takeover code + +# Copyright (C) Martin Schwenke, Ronnie Sahlberg 2010, 2011 + +# Based on original CTDB C code: +# +# Copyright (C) Ronnie Sahlberg 2007 +# Copyright (C) Andrew Tridgell 2007 + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <http://www.gnu.org/licenses/>. + + +import os +import sys +# Use optparse since newer argparse not available in RHEL5/EPEL. +from optparse import OptionParser +import copy +import random +import itertools + +# For parsing IP addresses +import socket +import struct + +# For external algorithm +import subprocess +import re + +options = None + +def process_args(extra_options=[]): + global options + + parser = OptionParser(option_list=extra_options) + + parser.add_option("--nd", + action="store_false", dest="deterministic_public_ips", + default=True, + help="turn off deterministic_public_ips") + parser.add_option("--ni", + action="store_true", dest="no_ip_failback", default=False, + help="turn on no_ip_failback") + parser.add_option("-L", "--lcp2", + action="store_true", dest="lcp2", default=False, + help="use LCP2 IP rebalancing algorithm [default: %default]") + parser.add_option("-e", "--external", + action="store_true", dest="external", default=False, + help="use external test program to implement IP allocation algorithm [default: %default]") + parser.add_option("-b", "--balance", + action="store_true", dest="balance", default=False, + help="show (im)balance information after each event") + parser.add_option("-d", "--diff", + action="store_true", dest="diff", default=False, + help="show IP address movements for each event") + parser.add_option("-n", "--no-print", + action="store_false", dest="show", default=True, + help="don't show IP address layout after each event") + parser.add_option("-v", "--verbose", + action="count", dest="verbose", default=0, + help="print information and actions taken to stdout") + parser.add_option("-r", "--retries", + action="store", type="int", dest="retries", default=5, + help="number of retry loops for rebalancing non-deterministic failback [default: %default]") + parser.add_option("-i", "--iterations", + action="store", type="int", dest="iterations", + default=1000, + help="number of iterations to run in test [default: %default]") + parser.add_option("-o", "--odds", + action="store", type="int", dest="odds", default=4, + help="make the chances of a failover 1 in ODDS [default: %default]") + parser.add_option("-A", "--aggressive", + action="store_true", dest="aggressive", default=False, + help="apply ODDS to try to flip each node [default: %default]") + + def seed_callback(option, opt, value, parser): + random.seed(value) + parser.add_option("-s", "--seed", + action="callback", type="int", callback=seed_callback, + help="initial random number seed for random events") + + parser.add_option("-x", "--exit", + action="store_true", dest="exit", default=False, + help="exit on the 1st gratuitous IP move or IP imbalance") + parser.add_option("-H", "--hard-imbalance-limit", + action="store", type="int", dest="hard_limit", default=1, + help="exceeding this limit causes termination [default: %default]") + parser.add_option("-S", "--soft-imbalance-limit", + action="store", type="int", dest="soft_limit", default=1, + help="exceeding this limit increments a counter [default: %default]") + + (options, args) = parser.parse_args() + + if len(args) != 0: + parser.error("too many arguments") + + # Could use a callback for this or change the default, but + # laziness is sometimes a virtue. ;-) + if options.lcp2: + options.deterministic_public_ips = False + +def print_begin(t, delim='='): + print delim * 40 + print "%s:" % (t) + +def print_end(): + print "-" * 40 + +def verbose_begin(t): + if options.verbose > 0: + print_begin(t) + +def verbose_end(): + if options.verbose > 0: + print_end() + +def verbose_print(t): + if options.verbose > 0: + if not type(t) == list: + t = [t] + if t != []: + print "\n".join([str(i) for i in t]) + +# more than this and we switch to the logging module... :-) +def debug_begin(t): + if options.verbose > 1: + print_begin(t, '-') + +def debug_end(): + if options.verbose > 1: + print_end() + +def debug_print(t): + if options.verbose > 1: + if not type(t) == list: + t = [t] + if t != []: + print "\n".join([str(i) for i in t]) + +def ip_to_list_of_ints(ip): + # Be lazy... but only expose errors in IPv4 addresses, since + # they'll be more commonly used. :-) + try: + l = socket.inet_pton(socket.AF_INET6, ip) + except: + # Pad with leading 0s. This makes IPv4 addresses comparable + # with IPv6 but reduces the overall effectiveness of the + # algorithm. The alternative would be to treat these + # addresses separately while trying to keep all the IPs in + # overall balance. + l = "".join(itertools.repeat("\0", 12)) + \ + socket.inet_pton(socket.AF_INET, ip) + + return map(lambda x: struct.unpack('B', x)[0], l) + +def ip_distance(ip1, ip2): + """Calculate the distance between 2 IPs. + + This is the length of the longtest common prefix between the IPs. + It is calculated by XOR-ing the 2 IPs together and counting the + number of leading zeroes.""" + + distance = 0 + for (o1, o2) in zip(ip_to_list_of_ints(ip1), ip_to_list_of_ints(ip2)): + # XOR this pair of octets + x = o1 ^ o2 + # count number leading zeroes + if x == 0: + distance += 8 + else: + # bin() gives minimal length '0bNNN' string + distance += (8 - (len(bin(x)) - 2)) + break + + return distance + +def ip_distance_2_sum(ip, ips): + """Calculate the IP distance for the given IP relative to IPs. + + This could be made more efficient by insering ip_distance_2 into + the loop in this function. However, that would result in some + loss of clarity and also will not be necessary in a C + implemntation.""" + + sum = 0 + for i in ips: + sum += ip_distance(ip, i) ** 2 + + return sum + +def imbalance_metric(ips): + """Return the imbalance metric for a group of IPs. + + This is the sum of squares of the IP distances between each pair of IPs.""" + if len(ips) > 1: + (h, t) = (ips[0], ips[1:]) + return ip_distance_2_sum(h, t) + imbalance_metric(t) + else: + return 0 + +def mean(l): + return float(sum(l))/len(l) + +class Node(object): + def __init__(self, public_addresses): + # List of list allows groups of IPs to be passed in. They're + # not actually used in the algorithm but are just used by + # calculate_imbalance() for checking the simulation. Note + # that people can pass in garbage and make this code + # fail... but we're all friends here in simulation world... + # :-) + if type(public_addresses[0]) is str: + self.public_addresses = set(public_addresses) + self.ip_groups = [] + else: + # flatten + self.public_addresses = set([i for s in public_addresses for i in s]) + self.ip_groups = public_addresses + + self.current_addresses = set() + self.healthy = True + self.imbalance = -1 + + def __str__(self): + return "%s %s%s" % \ + ("*" if len(self.public_addresses) == 0 else \ + (" " if self.healthy else "#"), + sorted(list(self.current_addresses)), + " %d" % self.imbalance if options.lcp2 else "") + + def can_node_serve_ip(self, ip): + return ip in self.public_addresses + + def node_ip_coverage(self, ips=None): + return len([a for a in self.current_addresses if ips == None or a in ips]) + + def set_imbalance(self, imbalance=-1): + """Set the imbalance metric to the given value. If none given + then calculate it.""" + + if imbalance != -1: + self.imbalance = imbalance + else: + self.imbalance = imbalance_metric(list(self.current_addresses)) + + def get_imbalance(self): + return self.imbalance + +class Cluster(object): + def __init__(self): + self.nodes = [] + self.deterministic_public_ips = options.deterministic_public_ips + self.no_ip_failback = options.no_ip_failback + self.all_public_ips = set() + + # Statistics + self.ip_moves = [] + self.grat_ip_moves = [] + self.imbalance = [] + self.imbalance_groups = [] + self.imbalance_count = 0 + self.imbalance_groups_count = itertools.repeat(0) + self.imbalance_metric = [] + self.events = -1 + self.num_unhealthy = [] + + self.prev = None + + def __str__(self): + return "\n".join(["%2d %s" % (i, n) \ + for (i, n) in enumerate(self.nodes)]) + + # This is naive. It assumes that IP groups are indicated by the + # 1st node having IP groups. + def have_ip_groups(self): + return (len(self.nodes[0].ip_groups) > 0) + + def print_statistics(self): + print_begin("STATISTICS") + print "Events: %6d" % self.events + print "Total IP moves: %6d" % sum(self.ip_moves) + print "Gratuitous IP moves: %6d" % sum(self.grat_ip_moves) + print "Max imbalance: %6d" % max(self.imbalance) + if self.have_ip_groups(): + print "Max group imbalance counts: ", map(max, zip(*self.imbalance_groups)) + print "Mean imbalance: %f" % mean(self.imbalance) + if self.have_ip_groups(): + print "Mean group imbalances counts: ", map(mean, zip(*self.imbalance_groups)) + print "Final imbalance: %6d" % self.imbalance[-1] + if self.have_ip_groups(): + print "Final group imbalances: ", self.imbalance_groups[-1] + if options.lcp2: + print "Max LCP2 imbalance : %6d" % max(self.imbalance_metric) + print "Soft imbalance count: %6d" % self.imbalance_count + if self.have_ip_groups(): + print "Soft imbalance group counts: ", self.imbalance_groups_count + if options.lcp2: + print "Final LCP2 imbalance : %6d" % self.imbalance_metric[-1] + print "Maximum unhealthy: %6d" % max(self.num_unhealthy) + print_end() + + def find_pnn_with_ip(self, ip): + for (i, n) in enumerate(self.nodes): + if ip in n.current_addresses: + return i + return -1 + + def quietly_remove_ip(self, ip): + # Remove address from old node. + old = self.find_pnn_with_ip(ip) + if old != -1: + self.nodes[old].current_addresses.remove(ip) + + def add_node(self, node): + self.nodes.append(node) + self.all_public_ips |= node.public_addresses + + def healthy(self, *pnns): + verbose_begin("HEALTHY") + + for pnn in pnns: + self.nodes[pnn].healthy = True + verbose_print(pnn) + + verbose_end() + + def unhealthy(self, *pnns): + + verbose_begin("UNHEALTHY") + + for pnn in pnns: + self.nodes[pnn].healthy = False + verbose_print(pnn) + + verbose_end() + + def do_something_random(self): + + """Make random node(s) healthy or unhealthy. + + If options.aggressive is False then: If all nodes are healthy + or unhealthy, then invert one of them; otherwise, there's a 1 + in options.odds chance of making another node unhealthy. + + If options.aggressive is True then: For each node there is a 1 + in options.odds chance of flipping the state of that node + between healthy and unhealthy.""" + + if not options.aggressive: + num_nodes = len(self.nodes) + healthy_pnns = [i for (i,n) in enumerate(self.nodes) if n.healthy] + num_healthy = len(healthy_pnns) + + if num_nodes == num_healthy: + self.unhealthy(random.randint(0, num_nodes-1)) + elif num_healthy == 0: + self.healthy(random.randint(0, num_nodes-1)) + elif random.randint(1, options.odds) == 1: + self.unhealthy(random.choice(healthy_pnns)) + else: + all_pnns = range(num_nodes) + unhealthy_pnns = sorted(list(set(all_pnns) - set(healthy_pnns))) + self.healthy(random.choice(unhealthy_pnns)) + else: + # We need to make at least one change or we retry...x + changed = False + while not changed: + for (pnn, n) in enumerate(self.nodes): + if random.randint(1, options.odds) == 1: + changed = True + if n.healthy: + self.unhealthy(pnn) + else: + self.healthy(pnn) + + def random_iterations(self): + i = 1 + while i <= options.iterations: + verbose_begin("EVENT %d" % i) + verbose_end() + self.do_something_random() + if self.recover() and options.exit: + break + i += 1 + + self.print_statistics() + + def imbalance_for_ips(self, ips): + + imbalance = 0 + + maxnode = -1 + minnode = -1 + + for ip in ips: + for (i, n) in enumerate(self.nodes): + + if not n.healthy or not n.can_node_serve_ip(ip): + continue + + num = n.node_ip_coverage(ips) + + if maxnode == -1 or num > maxnum: + maxnode = i + maxnum = num + + if minnode == -1 or num < minnum: + minnode = i + minnum = num + + if maxnode == -1 or minnode == -1: + continue + + i = maxnum - minnum + #if i < 2: + # i = 0 + imbalance = max([imbalance, i]) + + return imbalance + + + def calculate_imbalance(self): + + # First, do all the assigned IPs. + assigned = sorted([ip + for n in self.nodes + for ip in n.current_addresses]) + + i = self.imbalance_for_ips(assigned) + + ig = [] + # FIXME? If dealing with IP groups, assume the nodes are all + # the same. + for ips in self.nodes[0].ip_groups: + gi = self.imbalance_for_ips(ips) + ig.append(gi) + + return (i, ig) + + + def diff(self): + """Calculate differences in IP assignments between self and prev. + + Gratuitous IP moves (from a healthy node to a healthy node) + are prefixed by !!.""" + + ip_moves = 0 + grat_ip_moves = 0 + details = [] + + for (new, n) in enumerate(self.nodes): + for ip in n.current_addresses: + old = self.prev.find_pnn_with_ip(ip) + if old != new: + ip_moves += 1 + if old != -1 and \ + self.prev.nodes[new].healthy and \ + self.nodes[new].healthy and \ + self.nodes[old].healthy and \ + self.prev.nodes[old].healthy: + prefix = "!!" + grat_ip_moves += 1 + else: + prefix = " " + details.append("%s %s: %d -> %d" % + (prefix, ip, old, new)) + + return (ip_moves, grat_ip_moves, details) + + def find_takeover_node(self, ip): + + pnn = -1 + min = 0 + for (i, n) in enumerate(self.nodes): + if not n.healthy: + continue + + if not n.can_node_serve_ip(ip): + continue + + num = n.node_ip_coverage() + + if (pnn == -1): + pnn = i + min = num + else: + if num < min: + pnn = i + min = num + + if pnn == -1: + verbose_print("Could not find node to take over public address %s" % ip) + return False + + self.nodes[pnn].current_addresses.add(ip) + + verbose_print("%s -> %d" % (ip, pnn)) + return True + + def basic_allocate_unassigned(self): + + assigned = set([ip for n in self.nodes for ip in n.current_addresses]) + unassigned = sorted(list(self.all_public_ips - assigned)) + + for ip in unassigned: + self.find_takeover_node(ip) + + def basic_failback(self, retries_l): + + assigned = sorted([ip + for n in self.nodes + for ip in n.current_addresses]) + for ip in assigned: + + maxnode = -1 + minnode = -1 + for (i, n) in enumerate(self.nodes): + if not n.healthy: + continue + + if not n.can_node_serve_ip(ip): + continue + + num = n.node_ip_coverage() + + if maxnode == -1: + maxnode = i + maxnum = num + else: + if num > maxnum: + maxnode = i + maxnum = num + if minnode == -1: + minnode = i + minnum = num + else: + if num < minnum: + minnode = i + minnum = num + + if maxnode == -1: + print "Could not find maxnode. May not be able to serve ip", ip + continue + + #if self.deterministic_public_ips: + # continue + + if maxnum > minnum + 1 and retries_l[0] < options.retries: + # Remove the 1st ip from maxnode + t = sorted(list(self.nodes[maxnode].current_addresses)) + realloc = t[0] + verbose_print("%s <- %d" % (realloc, maxnode)) + self.nodes[maxnode].current_addresses.remove(realloc) + # Redo the outer loop. + retries_l[0] += 1 + return True + + return False + + + def lcp2_allocate_unassigned(self): + + # Assign as many unassigned addresses as possible. Keep + # selecting the optimal assignment until we don't manage to + # assign anything. + assigned = set([ip for n in self.nodes for ip in n.current_addresses]) + unassigned = sorted(list(self.all_public_ips - assigned)) + + should_loop = True + while len(unassigned) > 0 and should_loop: + should_loop = False + + debug_begin(" CONSIDERING MOVES (UNASSIGNED)") + + minnode = -1 + mindsum = 0 + minip = None + + for ip in unassigned: + for dstnode in range(len(self.nodes)): + if self.nodes[dstnode].can_node_serve_ip(ip) and \ + self.nodes[dstnode].healthy: + dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses) + dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum + debug_print(" %s -> %d [+%d]" % \ + (ip, + dstnode, + dstimbl - self.nodes[dstnode].get_imbalance())) + + if (minnode == -1) or (dstdsum < mindsum): + minnode = dstnode + minimbl = dstimbl + mindsum = dstdsum + minip = ip + should_loop = True + debug_end() + + if minnode != -1: + self.nodes[minnode].current_addresses.add(minip) + self.nodes[minnode].set_imbalance(self.nodes[minnode].get_imbalance() + mindsum) + verbose_print("%s -> %d [+%d]" % (minip, minnode, mindsum)) + unassigned.remove(minip) + + for ip in unassigned: + verbose_print("Could not find node to take over public address %s" % ip) + + def lcp2_failback(self, targets): + + # Get the node with the highest imbalance metric. + srcnode = -1 + maximbl = 0 + for (pnn, n) in enumerate(self.nodes): + b = n.get_imbalance() + if (srcnode == -1) or (b > maximbl): + srcnode = pnn + maximbl = b + + # This means that all nodes had 0 or 1 addresses, so can't + # be imbalanced. + if maximbl == 0: + return False + + # We'll need this a few times... + ips = self.nodes[srcnode].current_addresses + + # Find an IP and destination node that best reduces imbalance. + optimum = None + debug_begin(" CONSIDERING MOVES FROM %d [%d]" % (srcnode, maximbl)) + for ip in ips: + # What is this IP address costing the source node? + srcdsum = ip_distance_2_sum(ip, ips - set([ip])) + srcimbl = maximbl - srcdsum + + # Consider this IP address would cost each potential + # destination node. Destination nodes are limited to + # those that are newly healthy, since we don't want to + # do gratuitous failover of IPs just to make minor + # balance improvements. + for dstnode in targets: + if self.nodes[dstnode].can_node_serve_ip(ip) and \ + self.nodes[dstnode].healthy: + dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses) + dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum + debug_print(" %d [%d] -> %s -> %d [+%d]" % \ + (srcnode, + srcimbl - self.nodes[srcnode].get_imbalance(), + ip, + dstnode, + dstimbl - self.nodes[dstnode].get_imbalance())) + + if (dstimbl < maximbl) and (dstdsum < srcdsum): + if optimum is None: + optimum = (ip, srcnode, srcimbl, dstnode, dstimbl) + else: + (x, sn, si, dn, di) = optimum + if (srcimbl + dstimbl) < (si + di): + optimum = (ip, srcnode, srcimbl, dstnode, dstimbl) + debug_end() + + if optimum is not None: + # We found a move that makes things better... + (ip, srcnode, srcimbl, dstnode, dstimbl) = optimum + ini_srcimbl = self.nodes[srcnode].get_imbalance() + ini_dstimbl = self.nodes[dstnode].get_imbalance() + + self.nodes[srcnode].current_addresses.remove(ip) + self.nodes[srcnode].set_imbalance(srcimbl) + + self.nodes[dstnode].current_addresses.add(ip) + self.nodes[dstnode].set_imbalance(dstimbl) + + verbose_print("%d [%d] -> %s -> %d [+%d]" % \ + (srcnode, + srcimbl - ini_srcimbl, + ip, + dstnode, + dstimbl - ini_dstimbl)) + + return True + + return False + + def ctdb_takeover_run_python(self): + + # Don't bother with the num_healthy stuff. It is an + # irrelevant detail. + + # We just keep the allocate IPs in the current_addresses field + # of the node. This needs to readable, not efficient! + + if self.deterministic_public_ips: + # Remap everything. + addr_list = sorted(list(self.all_public_ips)) + for (i, ip) in enumerate(addr_list): + self.quietly_remove_ip(ip) + # Add addresses to new node. + pnn = i % len(self.nodes) + self.nodes[pnn].current_addresses.add(ip) + verbose_print("%s -> %d" % (ip, pnn)) + + # Remove public addresses from unhealthy nodes. + for (pnn, n) in enumerate(self.nodes): + if not n.healthy: + verbose_print(["%s <- %d" % (ip, pnn) + for ip in n.current_addresses]) + n.current_addresses = set() + + # If a node can't serve an assigned address then remove it. + for n in self.nodes: + verbose_print(["%s <- %d" % (ip, pnn) + for ip in n.current_addresses - n.public_addresses]) + n.current_addresses &= n.public_addresses + + if options.lcp2: + newly_healthy = [pnn for (pnn, n) in enumerate(self.nodes) + if len(n.current_addresses) == 0 and n.healthy] + for n in self.nodes: + n.set_imbalance() + + # We'll only retry the balancing act up to options.retries + # times (for the basic non-deterministic algorithm). This + # nonsense gives us a reference on the retries count in + # Python. It will be easier in C. :-) + # For LCP2 we reassignas many IPs from heavily "loaded" nodes + # to nodes that are newly healthy, looping until we fail to + # reassign an IP. + retries_l = [0] + should_loop = True + while should_loop: + should_loop = False + + if options.lcp2: + self.lcp2_allocate_unassigned() + else: + self.basic_allocate_unassigned() + + if self.no_ip_failback or self.deterministic_public_ips: + break + + if options.lcp2: + if len(newly_healthy) == 0: + break + should_loop = self.lcp2_failback(newly_healthy) + else: + should_loop = self.basic_failback(retries_l) + + def ctdb_takeover_run_external(self): + + # Written while asleep... + + # Convert the cluster state to something that be fed to + # ctdb_takeover_tests ctdb_takeover_run_core ... + + in_lines = [] + for ip in sorted(list(self.all_public_ips)): + allowed = [] + assigned = -1 + for (i, n) in enumerate(self.nodes): + if n.can_node_serve_ip(ip): + allowed.append("%s" % i) + if ip in n.current_addresses: + assigned = i + line = "%s\t%d\t%s" % (ip, assigned, ",".join(allowed)) + in_lines.append(line) + + nodestates = ",".join(["0" if n.healthy else "1" for n in self.nodes]) + + if options.lcp2: + os.environ["CTDB_LCP2"] = "yes" + if options.verbose > 1: + os.environ["CTDB_TEST_LOGLEVEL"] = "4" + elif options.verbose == 1: + os.environ["CTDB_TEST_LOGLEVEL"] = "3" + else: + os.environ["CTDB_TEST_LOGLEVEL"] = "0" + + p = subprocess.Popen("../../bin/ctdb_takeover_tests ctdb_takeover_run_core %s 2>&1" % nodestates, + shell=True, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.write("\n".join(in_lines)) + p.stdin.close() + + # Flush all of the assigned IPs. + for n in self.nodes: + n.current_addresses = set() + + # Uses the results to populate the current_addresses for each + # node. + for line in p.stdout.read().split("\n"): + # Some lines are debug, some are the final IP + # configuration. Let's use a gross hack that assumes any + # line with 2 words is IP configuration. That will do for + # now. + words = re.split("\s+", line) + if len(words) == 2: + # Add the IP as current for the specified node. + self.nodes[int(words[1])].current_addresses.add(words[0]) + else: + # First 3 words are log date/time, remove them... + print " ".join(words[3:]) + + # Now fake up the LCP calculations. + for n in self.nodes: + n.set_imbalance() + + def ctdb_takeover_run(self): + + self.events += 1 + + if options.external: + return self.ctdb_takeover_run_external() + else: + return self.ctdb_takeover_run_python() + + def recover(self): + verbose_begin("TAKEOVER") + + self.ctdb_takeover_run() + + verbose_end() + + grat_ip_moves = 0 + + if self.prev is not None: + (ip_moves, grat_ip_moves, details) = self.diff() + self.ip_moves.append(ip_moves) + self.grat_ip_moves.append(grat_ip_moves) + + if options.diff: + print_begin("DIFF") + print "\n".join(details) + print_end() + + (imbalance, imbalance_groups) = self.calculate_imbalance() + self.imbalance.append(imbalance) + self.imbalance_groups.append(imbalance_groups) + + if imbalance > options.soft_limit: + self.imbalance_count += 1 + + # There must be a cleaner way... + t = [] + for (c, i) in zip(self.imbalance_groups_count, imbalance_groups): + if i > options.soft_limit: + t.append(c + i) + else: + t.append(c) + self.imbalance_groups_count = t + + imbalance_metric = max([n.get_imbalance() for n in self.nodes]) + self.imbalance_metric.append(imbalance_metric) + if options.balance: + print_begin("IMBALANCE") + print "ALL IPS:", imbalance + if self.have_ip_groups(): + print "IP GROUPS:", imbalance_groups + if options.lcp2: + print "LCP2 IMBALANCE:", imbalance_metric + print_end() + + num_unhealthy = len(self.nodes) - \ + len([n for n in self.nodes if n.healthy]) + self.num_unhealthy.append(num_unhealthy) + + if options.show: + print_begin("STATE") + print self + print_end() + + self.prev = None + self.prev = copy.deepcopy(self) + + # True is bad! + return (grat_ip_moves > 0) or \ + (not self.have_ip_groups() and imbalance > options.hard_limit) or \ + (self.have_ip_groups() and (max(imbalance_groups) > options.hard_limit)) diff --git a/ctdb/tests/takeover/simulation/hey_jude.py b/ctdb/tests/takeover/simulation/hey_jude.py new file mode 100755 index 00000000000..a6b14c5c9b4 --- /dev/null +++ b/ctdb/tests/takeover/simulation/hey_jude.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses10 = ['10.4.20.%d' % n for n in range(154, 168)] +addresses172a = ['172.20.106.%d' % n for n in range(110, 124)] +addresses172b = ['172.20.107.%d' % n for n in range(110, 117)] + +c = Cluster() + +#for i in range(7): +# c.add_node(Node([addresses10, addresses172])) + + +for i in range(4): + c.add_node(Node([addresses172a, addresses172b])) +for i in range(3): + c.add_node(Node(addresses10)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups1.py b/ctdb/tests/takeover/simulation/ip_groups1.py new file mode 100755 index 00000000000..0808f466cf1 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups1.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +# 2 IP groups, both on the same 5 nodes, with each group on different +# interfaces/VLANs. One group has many more addresses to test how +# well an "imbalanced" configuration will balance... + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses20 = ['192.168.20.%d' % n for n in range(1, 13)] +addresses128 = ['192.168.128.%d' % n for n in range(1, 5)] + +c = Cluster() + +for i in range(5): + c.add_node(Node([addresses20, addresses128])) + +#for i in range(3): +# c.add_node(Node([addresses20])) + + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups2.py b/ctdb/tests/takeover/simulation/ip_groups2.py new file mode 100755 index 00000000000..c6c10266461 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups2.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +# 2 groups of addresses, combined into 1 pool so the checking +# algorithm doesn't know about the groups, across 2 nodes. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses20 = ['192.168.20.%d' % n for n in range(1, 13)] +addresses21 = ['192.168.21.%d' % n for n in range(1, 5)] + +c = Cluster() + +for i in range(2): + c.add_node(Node(addresses20 + addresses21)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups3.py b/ctdb/tests/takeover/simulation/ip_groups3.py new file mode 100755 index 00000000000..149946d72b4 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups3.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +# 4 IP groups, across 10 nodes, with each group on different +# interfaces/VLANs. 80 addresses in total but not evenly balanced, to +# help check some of the more extreme behaviour. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['192.168.1.%d' % n for n in range(1, 41)] +addresses2 = ['192.168.2.%d' % n for n in range(1, 21)] +addresses3 = ['192.168.3.%d' % n for n in range(1, 11)] +addresses4 = ['192.168.4.%d' % n for n in range(1, 11)] + +# Try detecting imbalance with square root of number of nodes? Or +# just with a parameter indicating how unbalanced you're willing to +# accept... + +c = Cluster() + +for i in range(10): + c.add_node(Node([addresses1, addresses2, addresses3, addresses4])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups4.py b/ctdb/tests/takeover/simulation/ip_groups4.py new file mode 100755 index 00000000000..fdcef7f0a69 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups4.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +# 2 IP groups, across 2 nodes, with each group on different +# interfaces. 4 addresses per group. A nice little canonical 2 node +# configuration. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['192.168.1.%d' % n for n in range(1, 5)] +addresses2 = ['192.168.2.%d' % n for n in range(1, 5)] + +# Try detecting imbalance with square root of number of nodes? Or +# just with a parameter indicating how unbalanced you're willing to +# accept... + +c = Cluster() + +for i in range(2): + c.add_node(Node([addresses1, addresses2])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups5.py b/ctdb/tests/takeover/simulation/ip_groups5.py new file mode 100755 index 00000000000..8c461506389 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups5.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# 1 IP group, to test backward compatibility of LCP2 algorithm. 16 +# addresses across 4 nodes. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['192.168.1.%d' % n for n in range(1, 17)] + +# Try detecting imbalance with square root of number of nodes? Or +# just with a parameter indicating how unbalanced you're willing to +# accept... + +c = Cluster() + +for i in range(4): + c.add_node(Node(addresses1)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/mgmt_simple.py b/ctdb/tests/takeover/simulation/mgmt_simple.py new file mode 100755 index 00000000000..f891199655a --- /dev/null +++ b/ctdb/tests/takeover/simulation/mgmt_simple.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +# This is an example showing a current SONAS configuration with 3 +# interface node and a management node. When run with deterministic +# IPs there are gratuitous IP reassignments. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] + +c = Cluster() + +for i in range(3): + c.add_node(Node(addresses)) + +c.add_node(Node([])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/node_group.py b/ctdb/tests/takeover/simulation/node_group.py new file mode 100755 index 00000000000..bf7de58aa97 --- /dev/null +++ b/ctdb/tests/takeover/simulation/node_group.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# This demonstrates a node group configurations. +# +# Node groups can be defined with the syntax "-g N@IP0,IP1-IP2,IP3". +# This says to create a group of N nodes with IPs IP0, IP1, ..., IP2, +# IP3. Run it with deterministic IPs causes lots of gratuitous IP +# reassignments. Running with --nd fixes this. + +import ctdb_takeover +import sys +from optparse import make_option +import string + +ctdb_takeover.process_args([ + make_option("-g", "--group", + action="append", type="string", dest="groups", + help="define a node group using N@IPs syntax"), + ]) + +def expand_range(r): + sr = r.split("-", 1) + if len(sr) == 2: + all = string.ascii_uppercase + string.ascii_lowercase + sr = list(all[all.index(sr[0]):all.index(sr[1])+1]) + return sr + +def add_node_group(s): + (count, ips_str) = s.split("@", 1) + ips = [i for r in ips_str.split(",") \ + for i in expand_range(r) if r != ""] + for i in range(int(count)): + c.add_node(ctdb_takeover.Node(ips)) + +c = ctdb_takeover.Cluster() + +if ctdb_takeover.options.groups is None: + print "Error: no node groups defined." + sys.exit(1) + +for g in ctdb_takeover.options.groups: + add_node_group(g) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/node_group_extra.py b/ctdb/tests/takeover/simulation/node_group_extra.py new file mode 100755 index 00000000000..7e9e518bddd --- /dev/null +++ b/ctdb/tests/takeover/simulation/node_group_extra.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# This example demonstrates a node group configuration. Is it meant +# to be the same as node_group_simple.py, but with a couple of nodes +# added later, so they are listed after the management node. + +# When run with deterministic IPs (use "-d" to show the problem) it +# does many gratuitous IP reassignments. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] + ['P', 'Q', 'R', 'S', 'T', 'U'] +addresses2 = ['I', 'J', 'K', 'L'] + +c = Cluster() + +for i in range(4): + c.add_node(Node(addresses1)) + +for i in range(3): + c.add_node(Node(addresses2)) + +c.add_node(Node([])) +c.add_node(Node(addresses1)) +c.add_node(Node(addresses2)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/node_group_simple.py b/ctdb/tests/takeover/simulation/node_group_simple.py new file mode 100755 index 00000000000..3c58ef7314a --- /dev/null +++ b/ctdb/tests/takeover/simulation/node_group_simple.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# This example demonstrates a simple, sensible node group +# configuration. When run with deterministic IPs (use "-d" to show +# the problem) it does many gratuitous IP reassignments. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] +addresses2 = ['I', 'J', 'K'] + +c = Cluster() + +for i in range(4): + c.add_node(Node(addresses1)) + +for i in range(3): + c.add_node(Node(addresses2)) + +c.add_node(Node([])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/nondet_path_01.py b/ctdb/tests/takeover/simulation/nondet_path_01.py new file mode 100755 index 00000000000..a62847a2163 --- /dev/null +++ b/ctdb/tests/takeover/simulation/nondet_path_01.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +# This is a contrived example that makes the balancing algorithm fail +# for nondeterministic IPs (run with "-dv --nd" to see the failure). + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['A', 'B', 'C', 'D'] +addresses2 = ['B', 'E', 'F'] + +c = Cluster() + +for i in range(2): + c.add_node(Node(addresses1)) + +c.add_node(Node(addresses2)) + +c.recover() + +c.unhealthy(1) +c.recover() +c.healthy(1) +c.recover() diff --git a/ctdb/tests/test_check_tcp_ports.sh b/ctdb/tests/test_check_tcp_ports.sh new file mode 100755 index 00000000000..e439b6d3744 --- /dev/null +++ b/ctdb/tests/test_check_tcp_ports.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +DIRNAME=$(dirname $0) + +. ${DIRNAME}/../config/functions + +SERVICE="test-service" + +PORTS="$@" + +if [ "x${PORTS}" = "x" ] ; then + PORTS=139 +fi + +ctdb_check_tcp_ports ${SERVICE} ${PORTS} + +echo "Test for service '${SERVICE}' on tcp ports ${PORTS} succeeded!" diff --git a/ctdb/tests/tool/README b/ctdb/tests/tool/README new file mode 100644 index 00000000000..816052862fd --- /dev/null +++ b/ctdb/tests/tool/README @@ -0,0 +1,17 @@ +Unit tests for the ctdb tool (i.e. tools/ctdb). + +Test case filenames can take 2 forms: + +* func.<some_function>.NNN.sh + + Run <some_function> in the ctdb tool code using the + ctdb_tool_functest test program. This test program uses test stubs + for CTDB client functions. + +* stubby.<command>.NNN.sh + + Run the ctdb_tool_stubby test program with <command> as the 1st + argument - subsequent are passed to simple_test(). ctdb_tool_stubby + is linked against the test stubs for CTDB client functions. + +To add tests here you may need to add appropriate test stubs. diff --git a/ctdb/tests/tool/func.parse_nodestring.001.sh b/ctdb/tests/tool/func.parse_nodestring.001.sh new file mode 100755 index 00000000000..d7caf89da98 --- /dev/null +++ b/ctdb/tests/tool/func.parse_nodestring.001.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, dd_ok, 3 healthy" + +required_result <<EOF +NODES: 0 1 2 +PNN MODE: BROADCAST_ALL (4026531842) +EOF + +simple_test all true <<EOF +0 192.168.20.41 0x0 +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 CURRENT RECMASTER +EOF diff --git a/ctdb/tests/tool/func.parse_nodestring.002.sh b/ctdb/tests/tool/func.parse_nodestring.002.sh new file mode 100755 index 00000000000..c89e444eaf0 --- /dev/null +++ b/ctdb/tests/tool/func.parse_nodestring.002.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, dd_ok, 2 ok/1 disconnected" + +required_result <<EOF +NODES: 0 1 2 +PNN MODE: BROADCAST_ALL (4026531842) +EOF + +simple_test all true <<EOF +0 192.168.20.41 0x0 +1 192.168.20.42 0x1 +2 192.168.20.43 0x0 CURRENT RECMASTER +EOF diff --git a/ctdb/tests/tool/func.parse_nodestring.003.sh b/ctdb/tests/tool/func.parse_nodestring.003.sh new file mode 100755 index 00000000000..3e03ac40419 --- /dev/null +++ b/ctdb/tests/tool/func.parse_nodestring.003.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, current disconnected" + +required_result 10 <<EOF +DATE TIME [PID]: Unable to get nodemap from local node +EOF + +simple_test all true <<EOF +0 192.168.20.41 0x0 +1 192.168.20.42 0x0 +2 192.168.20.43 0x1 CURRENT RECMASTER +EOF diff --git a/ctdb/tests/tool/scripts/local.sh b/ctdb/tests/tool/scripts/local.sh new file mode 100644 index 00000000000..385e2ad64b4 --- /dev/null +++ b/ctdb/tests/tool/scripts/local.sh @@ -0,0 +1,56 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +if "$TEST_VERBOSE" ; then + debug () { echo "$@" ; } +else + debug () { : ; } +fi + +define_test () +{ + _f=$(basename "$0" ".sh") + + case "$_f" in + func.*) + _func="${_f#func.}" + _func="${_func%.*}" # Strip test number + test_prog="ctdb_functest ${_func}" + ;; + stubby.*) + _cmd="${_f#stubby.}" + _cmd="${_cmd%.*}" # Strip test number + test_prog="ctdb_stubtest ${_cmd}" + ;; + *) + die "Unknown pattern for testcase \"$_f\"" + esac + + printf "%-28s - %s\n" "$_f" "$1" +} + +setup_natgw () +{ + debug "Setting up NAT gateway" + + natgw_config_dir="${TEST_VAR_DIR}/natgw_config" + mkdir -p "$natgw_config_dir" + + # These will accumulate, 1 per test... but will be cleaned up at + # the end. + export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir") + + cat >"$CTDB_NATGW_NODES" +} + +simple_test () +{ + # Most of the tests when the tool fails will have a date/time/pid + # prefix. Strip that because it isn't possible to match it. + if [ $required_rc -ne 0 ] ; then + OUT_FILTER='s@^[0-9/]+\ [0-9:\.]+\ \[[\ 0-9]+\]:@DATE\ TIME\ \[PID\]:@' + fi + + _out=$($VALGRIND $test_prog "$@" 2>&1) + + result_check +} diff --git a/ctdb/tests/tool/stubby.getcapabilities.001.sh b/ctdb/tests/tool/stubby.getcapabilities.001.sh new file mode 100755 index 00000000000..df4a659008f --- /dev/null +++ b/ctdb/tests/tool/stubby.getcapabilities.001.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all ok" + +required_result 0 <<EOF +RECMASTER: YES +LMASTER: YES +LVS: NO +NATGW: YES +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x0 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.getcapabilities.002.sh b/ctdb/tests/tool/stubby.getcapabilities.002.sh new file mode 100755 index 00000000000..9a37c4a93ec --- /dev/null +++ b/ctdb/tests/tool/stubby.getcapabilities.002.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 1 disconnected" + +required_result 0 <<EOF +RECMASTER: YES +LMASTER: YES +LVS: NO +NATGW: YES +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x0 CURRENT RECMASTER +1 192.168.20.42 0x1 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.getcapabilities.003.sh b/ctdb/tests/tool/stubby.getcapabilities.003.sh new file mode 100755 index 00000000000..33b1b74a203 --- /dev/null +++ b/ctdb/tests/tool/stubby.getcapabilities.003.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, current disconnected" + +required_result 10 <<EOF +DATE TIME [PID]: Unable to get nodemap from local node +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x1 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.lvs.001.sh b/ctdb/tests/tool/stubby.lvs.001.sh new file mode 100755 index 00000000000..29e9ce0cdc3 --- /dev/null +++ b/ctdb/tests/tool/stubby.lvs.001.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all ok" + +# This isn't very useful, since the stub for capabilities does set LVS :-) +required_result 0 <<EOF +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x0 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.lvsmaster.001.sh b/ctdb/tests/tool/stubby.lvsmaster.001.sh new file mode 100755 index 00000000000..38de280e348 --- /dev/null +++ b/ctdb/tests/tool/stubby.lvsmaster.001.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all ok" + +# This isn't very useful, since the stub for capabilities doesn't set LVS :-) +required_result 255 <<EOF +There is no LVS master +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x0 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.lvsmaster.002.sh b/ctdb/tests/tool/stubby.lvsmaster.002.sh new file mode 100755 index 00000000000..ea6e441496d --- /dev/null +++ b/ctdb/tests/tool/stubby.lvsmaster.002.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, current disconnected" + +# This isn't very useful, since the stub for capabilities doesn't set LVS :-) +required_result 10 <<EOF +DATE TIME [PID]: Unable to get nodemap from local node +EOF + +simple_test -Y <<EOF +NODEMAP +0 192.168.20.41 0x1 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.natgwlist.001.sh b/ctdb/tests/tool/stubby.natgwlist.001.sh new file mode 100755 index 00000000000..f1d2d37f834 --- /dev/null +++ b/ctdb/tests/tool/stubby.natgwlist.001.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all in natgw group, all ok" + +setup_natgw <<EOF +192.168.20.41 +192.168.20.42 +192.168.20.43 +EOF + +required_result 0 <<EOF +0 192.168.20.41 +Number of nodes:3 +pnn:0 192.168.20.41 OK (THIS NODE) +pnn:1 192.168.20.42 OK +pnn:2 192.168.20.43 OK +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x0 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.natgwlist.002.sh b/ctdb/tests/tool/stubby.natgwlist.002.sh new file mode 100755 index 00000000000..37f172298ca --- /dev/null +++ b/ctdb/tests/tool/stubby.natgwlist.002.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all in natgw group, 1 unhealthy" + +setup_natgw <<EOF +192.168.20.41 +192.168.20.42 +192.168.20.43 +EOF + +required_result 0 <<EOF +1 192.168.20.42 +Number of nodes:3 +pnn:0 192.168.20.41 UNHEALTHY +pnn:1 192.168.20.42 OK (THIS NODE) +pnn:2 192.168.20.43 OK +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x2 +1 192.168.20.42 0x0 CURRENT RECMASTER +2 192.168.20.43 0x0 + +VNNMAP +654321 +0 +1 +2 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: +EOF diff --git a/ctdb/tests/tool/stubby.natgwlist.003.sh b/ctdb/tests/tool/stubby.natgwlist.003.sh new file mode 100755 index 00000000000..19b1797e255 --- /dev/null +++ b/ctdb/tests/tool/stubby.natgwlist.003.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 2 in natgw group, 1 unhealthy" + +setup_natgw <<EOF +192.168.20.41 +192.168.20.43 +EOF + +required_result 0 <<EOF +2 192.168.20.43 +Number of nodes:2 +pnn:0 192.168.20.41 UNHEALTHY +pnn:2 192.168.20.43 OK +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x2 +1 192.168.20.42 0x0 CURRENT RECMASTER +2 192.168.20.43 0x0 + +VNNMAP +654321 +0 +1 +2 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: +EOF diff --git a/ctdb/tests/tool/stubby.natgwlist.004.sh b/ctdb/tests/tool/stubby.natgwlist.004.sh new file mode 100755 index 00000000000..2abec5e18a1 --- /dev/null +++ b/ctdb/tests/tool/stubby.natgwlist.004.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all unhealthy, all but 1 stopped" + +setup_natgw <<EOF +192.168.20.41 +192.168.20.42 +192.168.20.43 +EOF + +required_result 0 <<EOF +2 192.168.20.43 +Number of nodes:3 +pnn:0 192.168.20.41 UNHEALTHY|STOPPED|INACTIVE +pnn:1 192.168.20.42 UNHEALTHY|STOPPED|INACTIVE (THIS NODE) +pnn:2 192.168.20.43 UNHEALTHY +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x22 +1 192.168.20.42 0x22 CURRENT RECMASTER +2 192.168.20.43 0x2 + +VNNMAP +654321 +0 +1 +2 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: +EOF diff --git a/ctdb/tests/tool/stubby.natgwlist.005.sh b/ctdb/tests/tool/stubby.natgwlist.005.sh new file mode 100755 index 00000000000..42c7dbbab58 --- /dev/null +++ b/ctdb/tests/tool/stubby.natgwlist.005.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, all stopped" + +setup_natgw <<EOF +192.168.20.41 +192.168.20.42 +192.168.20.43 +EOF + +required_result 0 <<EOF +0 192.168.20.41 +Number of nodes:3 +pnn:0 192.168.20.41 STOPPED|INACTIVE +pnn:1 192.168.20.42 STOPPED|INACTIVE (THIS NODE) +pnn:2 192.168.20.43 STOPPED|INACTIVE +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x20 +1 192.168.20.42 0x20 CURRENT RECMASTER +2 192.168.20.43 0x20 + +VNNMAP +654321 +0 +1 +2 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: +EOF diff --git a/ctdb/tests/tool/stubby.nodestatus.001.sh b/ctdb/tests/tool/stubby.nodestatus.001.sh new file mode 100755 index 00000000000..6392b8d1ecc --- /dev/null +++ b/ctdb/tests/tool/stubby.nodestatus.001.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, 3 nodes, all OK" + +required_result 0 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 OK +pnn:1 192.168.20.42 OK +pnn:2 192.168.20.43 OK (THIS NODE) +EOF + +simple_test all <<EOF +NODEMAP +0 192.168.20.41 0x0 +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 CURRENT RECMASTER + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.nodestatus.002.sh b/ctdb/tests/tool/stubby.nodestatus.002.sh new file mode 100755 index 00000000000..f5b1909a07c --- /dev/null +++ b/ctdb/tests/tool/stubby.nodestatus.002.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "-n all, 3 nodes, all OK" + +required_result 0 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 OK +pnn:1 192.168.20.42 OK +pnn:2 192.168.20.43 OK (THIS NODE) +EOF + +simple_test all <<EOF +NODEMAP +0 192.168.20.41 0x0 +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 CURRENT RECMASTER + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.nodestatus.003.sh b/ctdb/tests/tool/stubby.nodestatus.003.sh new file mode 100755 index 00000000000..a3a7a42e434 --- /dev/null +++ b/ctdb/tests/tool/stubby.nodestatus.003.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, 3 nodes, 1 disconnected" + +required_result 1 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 OK +pnn:1 192.168.20.42 DISCONNECTED|INACTIVE +pnn:2 192.168.20.43 OK (THIS NODE) +EOF + +simple_test all <<EOF +NODEMAP +0 192.168.20.41 0x0 +1 192.168.20.42 0x1 +2 192.168.20.43 0x0 CURRENT RECMASTER + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.nodestatus.004.sh b/ctdb/tests/tool/stubby.nodestatus.004.sh new file mode 100755 index 00000000000..bc9890590c9 --- /dev/null +++ b/ctdb/tests/tool/stubby.nodestatus.004.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "-n all, 3 nodes, 1 disconnected" + +# -n all asks each node for the node status and +# thus reports THIS NODE for each node + +required_result 0 <<EOF +pnn:0 192.168.20.41 OK (THIS NODE) +pnn:2 192.168.20.43 OK (THIS NODE) +EOF + +simple_test -n all <<EOF +NODEMAP +0 192.168.20.41 0x0 +1 192.168.20.42 0x1 +2 192.168.20.43 0x0 CURRENT RECMASTER + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.nodestatus.005.sh b/ctdb/tests/tool/stubby.nodestatus.005.sh new file mode 100755 index 00000000000..cb532e7c827 --- /dev/null +++ b/ctdb/tests/tool/stubby.nodestatus.005.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "-n all all, 3 nodes, 1 disconnected" + +required_result 1 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 OK (THIS NODE) +pnn:1 192.168.20.42 DISCONNECTED|INACTIVE +pnn:2 192.168.20.43 OK +Number of nodes:3 +pnn:0 192.168.20.41 OK +pnn:1 192.168.20.42 DISCONNECTED|INACTIVE +pnn:2 192.168.20.43 OK (THIS NODE) +EOF + +simple_test -n all all <<EOF +NODEMAP +0 192.168.20.41 0x0 +1 192.168.20.42 0x1 +2 192.168.20.43 0x0 CURRENT RECMASTER + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.status.001.sh b/ctdb/tests/tool/stubby.status.001.sh new file mode 100755 index 00000000000..48b5bac24a9 --- /dev/null +++ b/ctdb/tests/tool/stubby.status.001.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, 3 nodes, all ok" + +required_result 0 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 OK (THIS NODE) +pnn:1 192.168.20.42 OK +pnn:2 192.168.20.43 OK +Generation:654321 +Size:3 +hash:0 lmaster:0 +hash:1 lmaster:1 +hash:2 lmaster:2 +Recovery mode:NORMAL (0) +Recovery master:0 +EOF + +simple_test all <<EOF +NODEMAP +0 192.168.20.41 0x0 CURRENT RECMASTER +1 192.168.20.42 0x0 +2 192.168.20.43 0x0 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF diff --git a/ctdb/tests/tool/stubby.status.002.sh b/ctdb/tests/tool/stubby.status.002.sh new file mode 100755 index 00000000000..fceceb30e0d --- /dev/null +++ b/ctdb/tests/tool/stubby.status.002.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all, 3 nodes, 1 unhealthy" + +required_result 0 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 UNHEALTHY +pnn:1 192.168.20.42 OK (THIS NODE) +pnn:2 192.168.20.43 OK +Generation:654321 +Size:3 +hash:0 lmaster:0 +hash:1 lmaster:1 +hash:2 lmaster:2 +Recovery mode:NORMAL (0) +Recovery master:1 +EOF + +simple_test <<EOF +NODEMAP +0 192.168.20.41 0x2 +1 192.168.20.42 0x0 CURRENT RECMASTER +2 192.168.20.43 0x0 + +VNNMAP +654321 +0 +1 +2 + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: +EOF diff --git a/ctdb/tests/tool/testcases/stubby.nodestatus.005.sh b/ctdb/tests/tool/testcases/stubby.nodestatus.005.sh new file mode 100755 index 00000000000..a18608dd4b8 --- /dev/null +++ b/ctdb/tests/tool/testcases/stubby.nodestatus.005.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +. "${TESTS_SUBDIR}/common.sh" + +define_test "-n all all, 3 nodes, 1 disconnected" + +required_result 1 <<EOF +Number of nodes:3 +pnn:0 192.168.20.41 OK (THIS NODE) +pnn:1 192.168.20.42 DISCONNECTED|INACTIVE +pnn:2 192.168.20.43 OK +Number of nodes:3 +pnn:0 192.168.20.41 OK +pnn:1 192.168.20.42 DISCONNECTED|INACTIVE +pnn:2 192.168.20.43 OK (THIS NODE) +EOF + +simple_test -n all all <<EOF +NODEMAP +0 192.168.20.41 0x0 +1 192.168.20.42 0x1 +2 192.168.20.43 0x0 CURRENT RECMASTER + +IFACES +:Name:LinkStatus:References: +:eth2:1:2: +:eth1:1:4: + +VNNMAP +654321 +0 +1 +2 +EOF |