From d4785e813aa5c2abf651c105edb1a8874afd1137 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Thu, 29 Mar 2012 14:54:22 +1100 Subject: Tests - turn of time logging by default We haven't seen problems related to time jumps for a long time. Turn this off by default. To switch it back on set $CTDB_TEST_TIME_LOGGING to any non-null value. Signed-off-by: Martin Schwenke (This used to be ctdb commit 2aa9bbf3a52dde0707eb06acd91e57c8da5c717f) --- ctdb/tests/scripts/ctdb_test_functions.bash | 3 ++- ctdb/tests/simple/00_ctdb_onnode.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 256ad99223..95a9cd1f87 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -116,7 +116,8 @@ ctdb_test_exit () echo "*** TEST COMPLETED (RC=$status) AT $(date '+%F %T'), CLEANING UP..." - if [ -n "$CTDB_TEST_REAL_CLUSTER" -a $status -ne 0 ] ; then + if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" -a \ + $status -ne 0 ] ; then ctdb_check_time_logs fi diff --git a/ctdb/tests/simple/00_ctdb_onnode.sh b/ctdb/tests/simple/00_ctdb_onnode.sh index fa5e25affc..3d8221a58c 100755 --- a/ctdb/tests/simple/00_ctdb_onnode.sh +++ b/ctdb/tests/simple/00_ctdb_onnode.sh @@ -31,7 +31,7 @@ onnode all onnode all true # We're seeing some weirdness with CTDB controls timing out. We're # wondering if time is jumping forward, so this creates a time log on # each node that we can examine later if tests fail weirdly. -if [ -n "$CTDB_TEST_REAL_CLUSTER" ] ; then +if [ -n "$CTDB_TEST_REAL_CLUSTER" -a -n "$CTDB_TEST_TIME_LOGGING" ] ; then echo "Starting time logging on each node..." f="/var/log/ctdb.test.time.log" onnode -p all "[ -f $f ] || while : ; do date '+%s %N' ; sleep 1 ; done >$f 2>&1 Date: Thu, 12 Apr 2012 14:17:46 +1000 Subject: Tests - IP allocation - add another LCP2 test based on a test failure Signed-off-by: Martin Schwenke (This used to be ctdb commit 89e322562f25dcb8d84357a19e71b2272b2f29a8) --- ctdb/tests/takeover/testcases/lcp2.011.sh | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 ctdb/tests/takeover/testcases/lcp2.011.sh diff --git a/ctdb/tests/takeover/testcases/lcp2.011.sh b/ctdb/tests/takeover/testcases/lcp2.011.sh new file mode 100755 index 0000000000..127a7a3491 --- /dev/null +++ b/ctdb/tests/takeover/testcases/lcp2.011.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +. "${TAKEOVER_TESTS_DIR}/common.sh" + +define_test "2 disjoint groups of nodes/addresses, continue a stopped node" + +# Another LCP2 1.0 bug + +export CTDB_TEST_LOGLEVEL=0 + +required_result < Date: Thu, 12 Apr 2012 15:18:17 +1000 Subject: Tests - IP allocation - 2 tests for NODE_FLAGS_NOIPTAKEOVER Signed-off-by: Martin Schwenke (This used to be ctdb commit fa54a1095a7504863ed30bff8e0828a8be6dde55) --- ctdb/tests/takeover/testcases/lcp2.012.sh | 31 +++++++++++++++++++++++++++++++ ctdb/tests/takeover/testcases/lcp2.013.sh | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100755 ctdb/tests/takeover/testcases/lcp2.012.sh create mode 100755 ctdb/tests/takeover/testcases/lcp2.013.sh diff --git a/ctdb/tests/takeover/testcases/lcp2.012.sh b/ctdb/tests/takeover/testcases/lcp2.012.sh new file mode 100755 index 0000000000..d53b188134 --- /dev/null +++ b/ctdb/tests/takeover/testcases/lcp2.012.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TAKEOVER_TESTS_DIR}/common.sh" + +define_test "Node with NODE_FLAGS_NOIPTAKEOVER doesn't gain IPs" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < Date: Mon, 16 Apr 2012 14:27:22 +1000 Subject: tests - export new variable TEST_SCRIPTS_DIR This replaces previous script-local variable ctdb_test_scripts_dir. Signed-off-by: Martin Schwenke (This used to be ctdb commit 107b465172205cb304549fcffaf36b9416696c15) --- ctdb/tests/scripts/ctdb_test_env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ctdb/tests/scripts/ctdb_test_env b/ctdb/tests/scripts/ctdb_test_env index f550d7945f..18d100f3d8 100755 --- a/ctdb/tests/scripts/ctdb_test_env +++ b/ctdb/tests/scripts/ctdb_test_env @@ -1,14 +1,14 @@ #!/bin/bash -ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd) -export CTDB_DIR=$(dirname $(dirname $ctdb_test_scripts_dir)) +export TEST_SCRIPTS_DIR=$(cd $(dirname $0) ; pwd) +export CTDB_DIR=$(dirname $(dirname $TEST_SCRIPTS_DIR)) var_dir=$CTDB_DIR/tests/var ###################################################################### ctdb_tools_dir=$CTDB_DIR/tools -PATH="${ctdb_test_scripts_dir}:${ctdb_tools_dir}:${PATH}" +PATH="${TEST_SCRIPTS_DIR}:${ctdb_tools_dir}:${PATH}" export CTDB_TIMEOUT=60 @@ -17,7 +17,7 @@ export CTDB_TIMEOUT=60 if [ -n "$CTDB_TEST_REMOTE_DIR" ] ; then CTDB_TEST_WRAPPER="${CTDB_TEST_REMOTE_DIR}/test_wrap" else - CTDB_TEST_WRAPPER="${ctdb_test_scripts_dir}/test_wrap" + CTDB_TEST_WRAPPER="${TEST_SCRIPTS_DIR}/test_wrap" fi export CTDB_TEST_WRAPPER -- cgit From 2c141e6c05b7d512dd690cba5dd9a2c9c04ba472 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:32:55 +1000 Subject: tests - add scripts/common.sh Signed-off-by: Martin Schwenke (This used to be ctdb commit a7ad94fe9f2e773567dbb6500469dd2dd2f2f04b) --- ctdb/tests/scripts/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 ctdb/tests/scripts/common.sh diff --git a/ctdb/tests/scripts/common.sh b/ctdb/tests/scripts/common.sh new file mode 100644 index 0000000000..581663c438 --- /dev/null +++ b/ctdb/tests/scripts/common.sh @@ -0,0 +1,9 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Common variables and functions for all CTDB tests. + +# Print a message and exit. +die () +{ + echo "$1" >&2 ; exit ${2:-1} +} -- cgit From f6178fcc9db4e05bddba1af16fd5ef6c56e9a515 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:33:37 +1000 Subject: tests - add scripts/unit.sh This will be sourced by all unit tests. Signed-off-by: Martin Schwenke (This used to be ctdb commit afdaa5f032938d56ff315d9553cb285ebc413c4c) --- ctdb/tests/scripts/unit.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ctdb/tests/scripts/unit.sh diff --git a/ctdb/tests/scripts/unit.sh b/ctdb/tests/scripts/unit.sh new file mode 100644 index 0000000000..977cc9ae9c --- /dev/null +++ b/ctdb/tests/scripts/unit.sh @@ -0,0 +1,16 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +. "${TEST_SCRIPTS_DIR}/common.sh" + +# Common variables and functions for CTDB unit tests. + +required_result () +{ + required_rc="${1:-0}" + required_output=$(cat) +} + +local="${TEST_SUBDIR}/scripts/local.sh" +if [ -r "$local" ] ; then + . "$local" +fi -- cgit From fec6fe628736c2b20ceeb53a43faa1ebff0a7ff0 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:48:49 +1000 Subject: tests - move functions only used by scripts/run_tests into that script Along with minor logic tweaks and removal of test_exit(). Signed-off-by: Martin Schwenke (This used to be ctdb commit 00713eb46cce638339845799bba2da041b3d02fb) --- ctdb/tests/scripts/ctdb_test_functions.bash | 61 ---------------------------- ctdb/tests/scripts/run_tests | 63 ++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash index 95a9cd1f87..e729486f55 100644 --- a/ctdb/tests/scripts/ctdb_test_functions.bash +++ b/ctdb/tests/scripts/ctdb_test_functions.bash @@ -8,51 +8,6 @@ fail () ###################################################################### -ctdb_test_begin () -{ - local name="$1" - - teststarttime=$(date '+%s') - testduration=0 - - echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" - echo "Running test $name ($(date '+%T'))" - echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" -} - -ctdb_test_end () -{ - local name="$1" ; shift - local status="$1" ; shift - # "$@" is command-line - - local interp="SKIPPED" - local statstr=" (reason $*)" - if [ -n "$status" ] ; then - if [ $status -eq 0 ] ; then - interp="PASSED" - statstr="" - echo "ALL OK: $*" - else - interp="FAILED" - statstr=" (status $status)" - testfailures=$(($testfailures+1)) - fi - fi - - testduration=$(($(date +%s)-$teststarttime)) - - echo "==========================================================================" - echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)" - echo "==========================================================================" - -} - -test_exit () -{ - exit $(($testfailures+0)) -} - ctdb_check_time_logs () { local threshold=20 @@ -144,22 +99,6 @@ ctdb_test_exit_hook_add () ctdb_test_exit_hook="${ctdb_test_exit_hook}${ctdb_test_exit_hook:+ ; }$*" } -ctdb_test_run () -{ - local name="$1" ; shift - - [ -n "$1" ] || set -- "$name" - - ctdb_test_begin "$name" - - local status=0 - "$@" || status=$? - - ctdb_test_end "$name" "$status" "$*" - - return $status -} - ctdb_test_usage() { local status=${1:-2} diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 1ce089d852..394b4150ac 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -46,8 +46,66 @@ fi ###################################################################### +ctdb_test_begin () +{ + local name="$1" + + teststarttime=$(date '+%s') + testduration=0 + + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" + echo "Running test $name ($(date '+%T'))" + echo "--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--==--" +} + +ctdb_test_end () +{ + local name="$1" ; shift + local status="$1" ; shift + # "$@" is command-line + + local interp="SKIPPED" + local statstr=" (reason $*)" + if [ -n "$status" ] ; then + if [ $status -eq 0 ] ; then + interp="PASSED" + statstr="" + echo "ALL OK: $*" + else + interp="FAILED" + statstr=" (status $status)" + fi + fi + + testduration=$(($(date +%s)-$teststarttime)) + + echo "==========================================================================" + echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)" + echo "==========================================================================" + +} + +ctdb_test_run () +{ + local name="$1" ; shift + + [ -n "$1" ] || set -- "$name" + + ctdb_test_begin "$name" + + local status=0 + "$@" || status=$? + + ctdb_test_end "$name" "$status" "$*" + + return $status +} + +###################################################################### + tests_total=0 tests_passed=0 +tests_failed=0 summary="" rows=$(if tty -s ; then stty size ; else echo x 80 ; fi | sed -e 's@.* @@' -e 's@^0$@80@') @@ -69,6 +127,7 @@ for f; do t=" PASSED " else t="*FAILED*" + tests_failed=$(($tests_failed + 1)) fi if $with_desc ; then desc=$(tail -n +4 $tf | head -n 1) @@ -89,4 +148,6 @@ fi rm -f "$sf" -test_exit +if [ $tests_failed -gt 0 ] ; then + exit 1 +fi -- cgit From 6eb8bf776ec0dc240a773ded0a7aaf99312dd271 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:51:22 +1000 Subject: tests - add -v option to set TEST_VERBOSE=true Signed-off-by: Martin Schwenke (This used to be ctdb commit 43badc5418b9f533398cd579607d9f1fc0f8f417) --- ctdb/tests/scripts/run_tests | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 394b4150ac..51a89bb3a9 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -21,7 +21,9 @@ with_summary=false with_desc=false quiet=false -temp=$(getopt -n "$prog" -o "xdhqs" -l help -- "$@") +export TEST_VERBOSE=false + +temp=$(getopt -n "$prog" -o "xdhqsv" -l help -- "$@") [ $? != 0 ] && usage @@ -33,6 +35,7 @@ while true ; do -d) with_desc=true ; shift ;; # 4th line of output is description -q) quiet=true ; shift ;; -s) with_summary=true ; shift ;; + -v) TEST_VERBOSE=true ; shift ;; --) shift ; break ;; *) usage ;; esac -- cgit From 9267727edb3c55665578a5a868127806ea39403e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:52:17 +1000 Subject: tests - run_tests can take a directory as an argument This makes it run all tests in the specified directory. Signed-off-by: Martin Schwenke (This used to be ctdb commit 89719384a74161ffa0c03602ecdd9e758d521d75) --- ctdb/tests/scripts/run_tests | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 51a89bb3a9..daa4b9c412 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -119,10 +119,16 @@ sf=$(mktemp) set -o pipefail -for f; do - [ -x $f ] || fail "test \"$f\" is not executable" +run_one_test () +{ + _f="$1" + + [ -x "$_f" ] || fail "test \"$_f\" is not executable" tests_total=$(($tests_total + 1)) - ctdb_test_run "$f" | tee "$tf" | show_progress + + export TEST_SUBDIR=$(dirname "$_f") + + ctdb_test_run "$_f" | tee "$tf" | show_progress status=$? if $with_summary ; then if [ $status -eq 0 ] ; then @@ -134,9 +140,21 @@ for f; do fi if $with_desc ; then desc=$(tail -n +4 $tf | head -n 1) - f="$desc" + _f="$desc" fi - echo "$t $f" >>"$sf" + echo "$t $_f" >>"$sf" + fi +} + +for f ; do + if [ -d "$f" ] ; then + for i in $(ls "$f/"*".sh" 2>/dev/null) ; do + run_one_test "$i" + done + elif [ -f "$f" ] ; then + run_one_test "$f" + else + fail "test \"$f\" is not recognised" fi done -- cgit From 8e71c5ec902544e4c1941098cd2864a56a03dc94 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 14:54:51 +1000 Subject: tests - run_tests includes common.sh, uses die() Signed-off-by: Martin Schwenke (This used to be ctdb commit 21df43c74bfcff420fdaf9df5440c25529c543d2) --- ctdb/tests/scripts/run_tests | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index daa4b9c412..c2ff15a69b 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -5,7 +5,7 @@ # the arguments that it sees. . $(dirname $0)/ctdb_test_env : -. ctdb_test_functions.bash +. "${TEST_SCRIPTS_DIR}/common.sh" usage() { cat < Date: Fri, 13 Apr 2012 12:05:59 +1000 Subject: tests/tool: Fix the nodestatus test Although not obvious, when asking each node for nodestatus, it will report each node is the current node. Signed-off-by: Amitay Isaacs (This used to be ctdb commit a65ba7211ec7f102b32060b173606a61fb5c20a9) --- ctdb/tests/tool/testcases/stubby.nodestatus.004.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ctdb/tests/tool/testcases/stubby.nodestatus.004.sh b/ctdb/tests/tool/testcases/stubby.nodestatus.004.sh index 6f5193f643..0679f62aa6 100755 --- a/ctdb/tests/tool/testcases/stubby.nodestatus.004.sh +++ b/ctdb/tests/tool/testcases/stubby.nodestatus.004.sh @@ -4,8 +4,11 @@ define_test "-n all, 3 nodes, 1 disconnected" +# -n all asks each node for the node status and +# thus reports THIS NODE for each node + required_result 0 < Date: Fri, 13 Apr 2012 12:06:53 +1000 Subject: tests/tool: New nodestatus test Signed-off-by: Amitay Isaacs (This used to be ctdb commit ca9e75eaea9c9b02bb44ca338e28fdae0c8d582c) --- ctdb/tests/tool/testcases/stubby.nodestatus.005.sh | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 ctdb/tests/tool/testcases/stubby.nodestatus.005.sh diff --git a/ctdb/tests/tool/testcases/stubby.nodestatus.005.sh b/ctdb/tests/tool/testcases/stubby.nodestatus.005.sh new file mode 100755 index 0000000000..a18608dd4b --- /dev/null +++ b/ctdb/tests/tool/testcases/stubby.nodestatus.005.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +. "${TESTS_SUBDIR}/common.sh" + +define_test "-n all all, 3 nodes, 1 disconnected" + +required_result 1 < Date: Mon, 16 Apr 2012 14:25:50 +1000 Subject: tests/tool - Restructure according to new convention Signed-off-by: Martin Schwenke (This used to be ctdb commit 89571585d87b391ff79647cd1f0f6ac193079e72) --- ctdb/tests/tool/common.sh | 79 ---------------------- ctdb/tests/tool/func.parse_nodestring.001.sh | 16 +++++ ctdb/tests/tool/func.parse_nodestring.002.sh | 16 +++++ ctdb/tests/tool/func.parse_nodestring.003.sh | 15 ++++ ctdb/tests/tool/run_tests.sh | 35 ---------- ctdb/tests/tool/scripts/local.sh | 70 +++++++++++++++++++ ctdb/tests/tool/stubby.nodestatus.001.sh | 30 ++++++++ ctdb/tests/tool/stubby.nodestatus.002.sh | 30 ++++++++ ctdb/tests/tool/stubby.nodestatus.003.sh | 30 ++++++++ ctdb/tests/tool/stubby.nodestatus.004.sh | 31 +++++++++ ctdb/tests/tool/stubby.nodestatus.005.sh | 34 ++++++++++ ctdb/tests/tool/stubby.status.001.sh | 37 ++++++++++ ctdb/tests/tool/stubby.status.002.sh | 37 ++++++++++ .../tool/testcases/func.parse_nodestring.001.sh | 16 ----- .../tool/testcases/func.parse_nodestring.002.sh | 16 ----- .../tool/testcases/func.parse_nodestring.003.sh | 15 ---- ctdb/tests/tool/testcases/stubby.nodestatus.001.sh | 30 -------- ctdb/tests/tool/testcases/stubby.nodestatus.002.sh | 30 -------- ctdb/tests/tool/testcases/stubby.nodestatus.003.sh | 30 -------- ctdb/tests/tool/testcases/stubby.nodestatus.004.sh | 31 --------- ctdb/tests/tool/testcases/stubby.nodestatus.005.sh | 34 ---------- ctdb/tests/tool/testcases/stubby.status.001.sh | 37 ---------- ctdb/tests/tool/testcases/stubby.status.002.sh | 37 ---------- 23 files changed, 346 insertions(+), 390 deletions(-) delete mode 100644 ctdb/tests/tool/common.sh create mode 100755 ctdb/tests/tool/func.parse_nodestring.001.sh create mode 100755 ctdb/tests/tool/func.parse_nodestring.002.sh create mode 100755 ctdb/tests/tool/func.parse_nodestring.003.sh delete mode 100755 ctdb/tests/tool/run_tests.sh create mode 100644 ctdb/tests/tool/scripts/local.sh create mode 100755 ctdb/tests/tool/stubby.nodestatus.001.sh create mode 100755 ctdb/tests/tool/stubby.nodestatus.002.sh create mode 100755 ctdb/tests/tool/stubby.nodestatus.003.sh create mode 100755 ctdb/tests/tool/stubby.nodestatus.004.sh create mode 100755 ctdb/tests/tool/stubby.nodestatus.005.sh create mode 100755 ctdb/tests/tool/stubby.status.001.sh create mode 100755 ctdb/tests/tool/stubby.status.002.sh delete mode 100755 ctdb/tests/tool/testcases/func.parse_nodestring.001.sh delete mode 100755 ctdb/tests/tool/testcases/func.parse_nodestring.002.sh delete mode 100755 ctdb/tests/tool/testcases/func.parse_nodestring.003.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.nodestatus.001.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.nodestatus.002.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.nodestatus.003.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.nodestatus.004.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.nodestatus.005.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.status.001.sh delete mode 100755 ctdb/tests/tool/testcases/stubby.status.002.sh diff --git a/ctdb/tests/tool/common.sh b/ctdb/tests/tool/common.sh deleted file mode 100644 index 506c7d0c4c..0000000000 --- a/ctdb/tests/tool/common.sh +++ /dev/null @@ -1,79 +0,0 @@ -# Hey Emacs, this is a -*- shell-script -*- !!! :-) - -# Print a message and exit. -die () { echo "$@" >&2 ; exit 1 ; } - -test_bin="$(dirname ${TESTS_SUBDIR})/bin" - -define_test () -{ - _f="$0" - _f="${_f#./}" # strip leading ./ - _f="${_f#testcases/}" # strip leading testcases/ - _f="${_f%.sh}" # strip off .sh suffix if any - - case "$_f" in - func.*) - _func="${_f#func.}" - _func="${_func%.*}" # Strip test number - test_prog="${test_bin}/ctdb_tool_libctdb ${_func}" - ;; - stubby.*) - _cmd="${_f#stubby.}" - _cmd="${_cmd%.*}" # Strip test number - test_prog="${test_bin}/ctdb_tool_stubby ${_cmd}" - ;; - *) - die "Unknown pattern for testcase \"$_f\"" - esac - - printf "%-28s - %s\n" "$_f" "$1" -} - -required_result () -{ - required_rc="${1:-0}" - required_output=$(cat) -} - -simple_test () -{ - _out=$($test_prog "$@" 2>&1) - _rc=$? - - # Most of the tests when the tool fails will have a date/time/pid - # prefix. Strip that because it isn't possible to match it. - if [ $required_rc -ne 0 ] ; then - OUT_FILTER='s@^[0-9/]+ [0-9:\.]+ \[[ 0-9]+\]:@DATE TIME \[PID\]:@' - fi - - if [ -n "$OUT_FILTER" ] ; then - _fout=$(echo "$_out" | sed -r "$OUT_FILTER") - else - _fout="$_out" - fi - - if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then - if [ "$TESTS_VERBOSE" = "yes" ] ; then - cat </dev/null) -fi - -"$test_dir/scripts/run_tests" $opts "$@" $tests || exit 1 - -echo "All OK" -exit 0 diff --git a/ctdb/tests/tool/scripts/local.sh b/ctdb/tests/tool/scripts/local.sh new file mode 100644 index 0000000000..fe8416ee98 --- /dev/null +++ b/ctdb/tests/tool/scripts/local.sh @@ -0,0 +1,70 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Print a message and exit. +die () { echo "$@" >&2 ; exit 1 ; } + +test_bin="$(dirname ${TEST_SUBDIR})/bin" + +define_test () +{ + _f=$(basename "$0" ".sh") + + case "$_f" in + func.*) + _func="${_f#func.}" + _func="${_func%.*}" # Strip test number + test_prog="${test_bin}/ctdb_tool_libctdb ${_func}" + ;; + stubby.*) + _cmd="${_f#stubby.}" + _cmd="${_cmd%.*}" # Strip test number + test_prog="${test_bin}/ctdb_tool_stubby ${_cmd}" + ;; + *) + die "Unknown pattern for testcase \"$_f\"" + esac + + printf "%-28s - %s\n" "$_f" "$1" +} + +simple_test () +{ + _out=$($test_prog "$@" 2>&1) + _rc=$? + + # Most of the tests when the tool fails will have a date/time/pid + # prefix. Strip that because it isn't possible to match it. + if [ $required_rc -ne 0 ] ; then + OUT_FILTER='s@^[0-9/]+ [0-9:\.]+ \[[ 0-9]+\]:@DATE TIME \[PID\]:@' + fi + + if [ -n "$OUT_FILTER" ] ; then + _fout=$(echo "$_out" | sed -r "$OUT_FILTER") + else + _fout="$_out" + fi + + if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then + if [ "$TEST_VERBOSE" = "yes" ] ; then + cat < Date: Mon, 16 Apr 2012 15:29:44 +1000 Subject: tests/onnode - Restructure according to new convention Signed-off-by: Martin Schwenke (This used to be ctdb commit 4c87888b2a559c0d1f285919a42de450a537a02f) --- ctdb/tests/onnode/0001.sh | 2 +- ctdb/tests/onnode/0002.sh | 2 +- ctdb/tests/onnode/0003.sh | 2 +- ctdb/tests/onnode/0004.sh | 2 +- ctdb/tests/onnode/0005.sh | 2 +- ctdb/tests/onnode/0006.sh | 2 +- ctdb/tests/onnode/0070.sh | 2 +- ctdb/tests/onnode/0071.sh | 2 +- ctdb/tests/onnode/0072.sh | 2 +- ctdb/tests/onnode/0075.sh | 2 +- ctdb/tests/onnode/0080.sh | 2 +- ctdb/tests/onnode/0081.sh | 2 +- ctdb/tests/onnode/0090.sh | 2 +- ctdb/tests/onnode/0091.sh | 2 +- ctdb/tests/onnode/common.sh | 103 ------------------------------------- ctdb/tests/onnode/run_tests.sh | 31 ----------- ctdb/tests/onnode/scripts/local.sh | 88 +++++++++++++++++++++++++++++++ 17 files changed, 102 insertions(+), 148 deletions(-) delete mode 100644 ctdb/tests/onnode/common.sh delete mode 100755 ctdb/tests/onnode/run_tests.sh create mode 100644 ctdb/tests/onnode/scripts/local.sh diff --git a/ctdb/tests/onnode/0001.sh b/ctdb/tests/onnode/0001.sh index 3c2d541ef4..28533748f2 100755 --- a/ctdb/tests/onnode/0001.sh +++ b/ctdb/tests/onnode/0001.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE all hostname" diff --git a/ctdb/tests/onnode/0002.sh b/ctdb/tests/onnode/0002.sh index b18d91efa2..c3c8c77a0e 100755 --- a/ctdb/tests/onnode/0002.sh +++ b/ctdb/tests/onnode/0002.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE -q all hostname" diff --git a/ctdb/tests/onnode/0003.sh b/ctdb/tests/onnode/0003.sh index d925799261..d79bca2806 100755 --- a/ctdb/tests/onnode/0003.sh +++ b/ctdb/tests/onnode/0003.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE -p all hostname" diff --git a/ctdb/tests/onnode/0004.sh b/ctdb/tests/onnode/0004.sh index a6e6424ace..d0986b2ffd 100755 --- a/ctdb/tests/onnode/0004.sh +++ b/ctdb/tests/onnode/0004.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE -pq all hostname" diff --git a/ctdb/tests/onnode/0005.sh b/ctdb/tests/onnode/0005.sh index af237da9a3..0eccbb04a3 100755 --- a/ctdb/tests/onnode/0005.sh +++ b/ctdb/tests/onnode/0005.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE 3 hostname" diff --git a/ctdb/tests/onnode/0006.sh b/ctdb/tests/onnode/0006.sh index aa6eeba89e..b027850240 100755 --- a/ctdb/tests/onnode/0006.sh +++ b/ctdb/tests/onnode/0006.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE -v 3 hostname" diff --git a/ctdb/tests/onnode/0070.sh b/ctdb/tests/onnode/0070.sh index f38c95f5fd..902d78b231 100755 --- a/ctdb/tests/onnode/0070.sh +++ b/ctdb/tests/onnode/0070.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE ok hostname" diff --git a/ctdb/tests/onnode/0071.sh b/ctdb/tests/onnode/0071.sh index 8a1cb59dd7..ebf2f61fcd 100755 --- a/ctdb/tests/onnode/0071.sh +++ b/ctdb/tests/onnode/0071.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE ok hostname" diff --git a/ctdb/tests/onnode/0072.sh b/ctdb/tests/onnode/0072.sh index d80361a435..cb29e3b9a2 100755 --- a/ctdb/tests/onnode/0072.sh +++ b/ctdb/tests/onnode/0072.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE ok hostname" diff --git a/ctdb/tests/onnode/0075.sh b/ctdb/tests/onnode/0075.sh index aa2e8be0ba..6ce9b9b190 100755 --- a/ctdb/tests/onnode/0075.sh +++ b/ctdb/tests/onnode/0075.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE con hostname" diff --git a/ctdb/tests/onnode/0080.sh b/ctdb/tests/onnode/0080.sh index 095f65b224..bca478ada5 100755 --- a/ctdb/tests/onnode/0080.sh +++ b/ctdb/tests/onnode/0080.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE recmaster hostname" diff --git a/ctdb/tests/onnode/0081.sh b/ctdb/tests/onnode/0081.sh index e593934f66..412db87e4c 100755 --- a/ctdb/tests/onnode/0081.sh +++ b/ctdb/tests/onnode/0081.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE lvsmaster hostname" diff --git a/ctdb/tests/onnode/0090.sh b/ctdb/tests/onnode/0090.sh index 1147e34c10..dd50c51b70 100755 --- a/ctdb/tests/onnode/0090.sh +++ b/ctdb/tests/onnode/0090.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE natgw hostname" diff --git a/ctdb/tests/onnode/0091.sh b/ctdb/tests/onnode/0091.sh index 5239eef1b4..528eec16df 100755 --- a/ctdb/tests/onnode/0091.sh +++ b/ctdb/tests/onnode/0091.sh @@ -1,6 +1,6 @@ #!/bin/sh -. "${ONNODE_TESTS_DIR}/common.sh" +. "${TEST_SCRIPTS_DIR}/unit.sh" cmd="$ONNODE natgw hostname" diff --git a/ctdb/tests/onnode/common.sh b/ctdb/tests/onnode/common.sh deleted file mode 100644 index f920bc398a..0000000000 --- a/ctdb/tests/onnode/common.sh +++ /dev/null @@ -1,103 +0,0 @@ -# Hey Emacs, this is a -*- shell-script -*- !!! :-) - -# Set indirectly by run_tests at top level. -unset CTDB_NODES_SOCKETS - -# Default to just "onnode". -: ${ONNODE:=onnode} - -# Augment PATH with relevant stubs/ directories. - -if [ -d "${ONNODE_TESTS_DIR}/stubs" ] ; then - PATH="${ONNODE_TESTS_DIR}/stubs:$PATH" -fi - -export ONNODE_TESTCASE_DIR=$(dirname "$0") -if [ $(basename "$ONNODE_TESTCASE_DIR") = "onnode" ] ; then - # Just a test script, no testcase subdirectory. - ONNODE_TESTCASE_DIR="$ONNODE_TESTS_DIR" -else - if [ -d "${ONNODE_TESTCASE_DIR}/stubs" ] ; then - PATH="${ONNODE_TESTCASE_DIR}/stubs:$PATH" - fi -fi - -# Find CTDB nodes file. -if [ -z "$CTDB_NODES_FILE" ] ; then - if [ -r "${ONNODE_TESTCASE_DIR}/nodes" ] ; then - CTDB_NODES_FILE="${ONNODE_TESTCASE_DIR}/nodes" - elif [ -r "${ONNODE_TESTS_DIR}/nodes" ] ; then - CTDB_NODES_FILE="${ONNODE_TESTS_DIR}/nodes" - else - CTDB_NODES_FILE="${CTDB_BASE:-/etc/ctdb}/nodes" - fi -fi - -export CTDB_NODES_FILE - -export ONNODE_TESTS_VAR_DIR="${ONNODE_TESTS_DIR}/var" -mkdir -p "$ONNODE_TESTS_VAR_DIR" - -if [ -z "$CTDB_BASE" ] ; then - export CTDB_BASE=$(dirname "$CTDB_NODES_FILE") -fi - -define_test () -{ - _f="$0" - _f="${_f#./}" # strip leading ./ - _f="${_f%%/*}" # if subdir, strip off file - _f="${_f%.sh}" # strip off .sh suffix if any - - echo "$_f $1 - $2" -} - -# Set output for ctdb command. Option 1st argument is return code. -ctdb_set_output () -{ - _out="$ONNODE_TESTS_VAR_DIR/ctdb.out" - cat >"$_out" - - _rc="$ONNODE_TESTS_VAR_DIR/ctdb.rc" - echo "${1:-0}" >"$_rc" - - trap "rm -f $_out $_rc" 0 -} - -required_result () -{ - required_rc="${1:-0}" - required_output=$(cat) -} - -simple_test () -{ - _sort="cat" - if [ "$1" = "-s" ] ; then - shift - _sort="sort" - fi - _out=$("$@" 2>&1) - _rc=$? - _out=$(echo "$_out" | $_sort ) - - if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then - echo "PASSED" - else - cat </dev/null) -fi - -"$test_dir/scripts/run_tests" $opts "$@" $tests || exit 1 - -echo "All OK" -exit 0 diff --git a/ctdb/tests/onnode/scripts/local.sh b/ctdb/tests/onnode/scripts/local.sh new file mode 100644 index 0000000000..a990c695f8 --- /dev/null +++ b/ctdb/tests/onnode/scripts/local.sh @@ -0,0 +1,88 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Set indirectly by run_tests at top level. +unset CTDB_NODES_SOCKETS + +# Default to just "onnode". +: ${ONNODE:=onnode} + +# Augment PATH with relevant stubs/ directories. + +if [ -d "${TEST_SUBDIR}/stubs" ] ; then + PATH="${TEST_SUBDIR}/stubs:$PATH" +fi + +# Find CTDB nodes file. +if [ -z "$CTDB_NODES_FILE" ] ; then + if [ -r "${TEST_SUBDIR}/nodes" ] ; then + CTDB_NODES_FILE="${TEST_SUBDIR}/nodes" + else + CTDB_NODES_FILE="${CTDB_BASE:-/etc/ctdb}/nodes" + fi +fi + +export CTDB_NODES_FILE + +export ONNODE_TESTS_VAR_DIR="${TEST_SUBDIR}/var" +mkdir -p "$ONNODE_TESTS_VAR_DIR" + +if [ -z "$CTDB_BASE" ] ; then + export CTDB_BASE=$(dirname "$CTDB_NODES_FILE") +fi + +define_test () +{ + _f=$(basename "$0") + + echo "$_f $1 - $2" +} + +# Set output for ctdb command. Option 1st argument is return code. +ctdb_set_output () +{ + _out="$ONNODE_TESTS_VAR_DIR/ctdb.out" + cat >"$_out" + + _rc="$ONNODE_TESTS_VAR_DIR/ctdb.rc" + echo "${1:-0}" >"$_rc" + + trap "rm -f $_out $_rc" 0 +} + +required_result () +{ + required_rc="${1:-0}" + required_output=$(cat) +} + +simple_test () +{ + _sort="cat" + if [ "$1" = "-s" ] ; then + shift + _sort="sort" + fi + _out=$("$@" 2>&1) + _rc=$? + _out=$(echo "$_out" | $_sort ) + + if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then + echo "PASSED" + else + cat < Date: Mon, 16 Apr 2012 15:53:24 +1000 Subject: tests/takeover - Restructure according to new convention Signed-off-by: Martin Schwenke (This used to be ctdb commit f94897ef9504c144937409688fa7a646e0b0ceae) --- ctdb/tests/takeover/README | 6 - ctdb/tests/takeover/common.sh | 70 -- ctdb/tests/takeover/ctdb_takeover.py | 888 --------------------- ctdb/tests/takeover/ip_groups1.py | 25 - ctdb/tests/takeover/ip_groups2.py | 20 - ctdb/tests/takeover/ip_groups3.py | 27 - ctdb/tests/takeover/ip_groups4.py | 25 - ctdb/tests/takeover/ip_groups5.py | 23 - ctdb/tests/takeover/lcp2.001.sh | 31 + ctdb/tests/takeover/lcp2.002.sh | 31 + ctdb/tests/takeover/lcp2.003.sh | 31 + ctdb/tests/takeover/lcp2.004.sh | 37 + ctdb/tests/takeover/lcp2.005.sh | 181 +++++ ctdb/tests/takeover/lcp2.006.sh | 31 + ctdb/tests/takeover/lcp2.007.sh | 31 + ctdb/tests/takeover/lcp2.008.sh | 31 + ctdb/tests/takeover/lcp2.009.sh | 31 + ctdb/tests/takeover/lcp2.010.sh | 32 + ctdb/tests/takeover/lcp2.011.sh | 45 ++ ctdb/tests/takeover/lcp2.012.sh | 31 + ctdb/tests/takeover/lcp2.013.sh | 31 + ctdb/tests/takeover/mgmt_simple.py | 22 - ctdb/tests/takeover/node_group.py | 46 -- ctdb/tests/takeover/node_group_extra.py | 31 - ctdb/tests/takeover/node_group_simple.py | 26 - ctdb/tests/takeover/nondet.001.sh | 29 + ctdb/tests/takeover/nondet.002.sh | 29 + ctdb/tests/takeover/nondet.003.sh | 29 + ctdb/tests/takeover/nondet_path_01.py | 25 - ctdb/tests/takeover/run_tests.sh | 31 - ctdb/tests/takeover/scripts/local.sh | 58 ++ ctdb/tests/takeover/simulation/README | 6 + ctdb/tests/takeover/simulation/ctdb_takeover.py | 888 +++++++++++++++++++++ ctdb/tests/takeover/simulation/hey_jude.py | 24 + ctdb/tests/takeover/simulation/ip_groups1.py | 25 + ctdb/tests/takeover/simulation/ip_groups2.py | 20 + ctdb/tests/takeover/simulation/ip_groups3.py | 27 + ctdb/tests/takeover/simulation/ip_groups4.py | 25 + ctdb/tests/takeover/simulation/ip_groups5.py | 23 + ctdb/tests/takeover/simulation/mgmt_simple.py | 22 + ctdb/tests/takeover/simulation/node_group.py | 46 ++ ctdb/tests/takeover/simulation/node_group_extra.py | 31 + .../tests/takeover/simulation/node_group_simple.py | 26 + ctdb/tests/takeover/simulation/nondet_path_01.py | 25 + ctdb/tests/takeover/testcases/lcp2.001.sh | 31 - ctdb/tests/takeover/testcases/lcp2.002.sh | 31 - ctdb/tests/takeover/testcases/lcp2.003.sh | 31 - ctdb/tests/takeover/testcases/lcp2.004.sh | 37 - ctdb/tests/takeover/testcases/lcp2.005.sh | 181 ----- ctdb/tests/takeover/testcases/lcp2.006.sh | 31 - ctdb/tests/takeover/testcases/lcp2.007.sh | 31 - ctdb/tests/takeover/testcases/lcp2.008.sh | 31 - ctdb/tests/takeover/testcases/lcp2.009.sh | 31 - ctdb/tests/takeover/testcases/lcp2.010.sh | 32 - ctdb/tests/takeover/testcases/lcp2.011.sh | 45 -- ctdb/tests/takeover/testcases/lcp2.012.sh | 31 - ctdb/tests/takeover/testcases/lcp2.013.sh | 31 - ctdb/tests/takeover/testcases/nondet.001.sh | 29 - ctdb/tests/takeover/testcases/nondet.002.sh | 29 - ctdb/tests/takeover/testcases/nondet.003.sh | 29 - 60 files changed, 1907 insertions(+), 1926 deletions(-) delete mode 100644 ctdb/tests/takeover/README delete mode 100644 ctdb/tests/takeover/common.sh delete mode 100755 ctdb/tests/takeover/ctdb_takeover.py delete mode 100755 ctdb/tests/takeover/ip_groups1.py delete mode 100755 ctdb/tests/takeover/ip_groups2.py delete mode 100755 ctdb/tests/takeover/ip_groups3.py delete mode 100755 ctdb/tests/takeover/ip_groups4.py delete mode 100755 ctdb/tests/takeover/ip_groups5.py create mode 100755 ctdb/tests/takeover/lcp2.001.sh create mode 100755 ctdb/tests/takeover/lcp2.002.sh create mode 100755 ctdb/tests/takeover/lcp2.003.sh create mode 100755 ctdb/tests/takeover/lcp2.004.sh create mode 100755 ctdb/tests/takeover/lcp2.005.sh create mode 100755 ctdb/tests/takeover/lcp2.006.sh create mode 100755 ctdb/tests/takeover/lcp2.007.sh create mode 100755 ctdb/tests/takeover/lcp2.008.sh create mode 100755 ctdb/tests/takeover/lcp2.009.sh create mode 100755 ctdb/tests/takeover/lcp2.010.sh create mode 100755 ctdb/tests/takeover/lcp2.011.sh create mode 100755 ctdb/tests/takeover/lcp2.012.sh create mode 100755 ctdb/tests/takeover/lcp2.013.sh delete mode 100755 ctdb/tests/takeover/mgmt_simple.py delete mode 100755 ctdb/tests/takeover/node_group.py delete mode 100755 ctdb/tests/takeover/node_group_extra.py delete mode 100755 ctdb/tests/takeover/node_group_simple.py create mode 100755 ctdb/tests/takeover/nondet.001.sh create mode 100755 ctdb/tests/takeover/nondet.002.sh create mode 100755 ctdb/tests/takeover/nondet.003.sh delete mode 100755 ctdb/tests/takeover/nondet_path_01.py delete mode 100755 ctdb/tests/takeover/run_tests.sh create mode 100644 ctdb/tests/takeover/scripts/local.sh create mode 100644 ctdb/tests/takeover/simulation/README create mode 100755 ctdb/tests/takeover/simulation/ctdb_takeover.py create mode 100755 ctdb/tests/takeover/simulation/hey_jude.py create mode 100755 ctdb/tests/takeover/simulation/ip_groups1.py create mode 100755 ctdb/tests/takeover/simulation/ip_groups2.py create mode 100755 ctdb/tests/takeover/simulation/ip_groups3.py create mode 100755 ctdb/tests/takeover/simulation/ip_groups4.py create mode 100755 ctdb/tests/takeover/simulation/ip_groups5.py create mode 100755 ctdb/tests/takeover/simulation/mgmt_simple.py create mode 100755 ctdb/tests/takeover/simulation/node_group.py create mode 100755 ctdb/tests/takeover/simulation/node_group_extra.py create mode 100755 ctdb/tests/takeover/simulation/node_group_simple.py create mode 100755 ctdb/tests/takeover/simulation/nondet_path_01.py delete mode 100755 ctdb/tests/takeover/testcases/lcp2.001.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.002.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.003.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.004.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.005.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.006.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.007.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.008.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.009.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.010.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.011.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.012.sh delete mode 100755 ctdb/tests/takeover/testcases/lcp2.013.sh delete mode 100755 ctdb/tests/takeover/testcases/nondet.001.sh delete mode 100755 ctdb/tests/takeover/testcases/nondet.002.sh delete mode 100755 ctdb/tests/takeover/testcases/nondet.003.sh diff --git a/ctdb/tests/takeover/README b/ctdb/tests/takeover/README deleted file mode 100644 index 4a8267bf6e..0000000000 --- a/ctdb/tests/takeover/README +++ /dev/null @@ -1,6 +0,0 @@ -This contains a Python simulation of CTDB's IP reallocation algorithm. - -It is useful for experimenting with improvements. - -To use this on RHEL5 you'll need python2.6 from EPEL -. diff --git a/ctdb/tests/takeover/common.sh b/ctdb/tests/takeover/common.sh deleted file mode 100644 index 716e4d5fc7..0000000000 --- a/ctdb/tests/takeover/common.sh +++ /dev/null @@ -1,70 +0,0 @@ -# Hey Emacs, this is a -*- shell-script -*- !!! :-) - -# Print a message and exit. -die () { echo "$@" >&2 ; exit 1 ; } - -test_prog="$(dirname ${TAKEOVER_TESTS_DIR})/bin/ctdb_takeover_tests ctdb_takeover_run_core" - -define_test () -{ - _f="$0" - _f="${_f#./}" # strip leading ./ - _f="${_f#testcases/}" # strip leading testcases/ - _f="${_f%.sh}" # strip off .sh suffix if any - - case "$_f" in - nondet.*) - algorithm="nondet" - export CTDB_LCP2="no" - ;; - lcp2.*) - algorithm="lcp2" - export CTDB_LCP2="yes" - ;; - *) - die "Unknown algorithm for testcase \"$_f\"" - esac - - printf "%-12s - %s\n" "$_f" "$1" -} - -required_result () -{ - required_rc="${1:-0}" - required_output=$(cat) -} - -simple_test () -{ - _states="$1" - _out=$($test_prog $_states 2>&1) - _rc=$? - - if [ "$algorithm" = "lcp2" -a -n "$CTDB_TEST_LOGLEVEL" ] ; then - OUT_FILTER='s@^.*:@DATE TIME \[PID\]:@' - fi - - if [ -n "$OUT_FILTER" ] ; then - _fout=$(echo "$_out" | sed -r "$OUT_FILTER") - else - _fout="$_out" - fi - - if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then - echo "PASSED" - else - cat <. - - -import os -import sys -# Use optparse since newer argparse not available in RHEL5/EPEL. -from optparse import OptionParser -import copy -import random -import itertools - -# For parsing IP addresses -import socket -import struct - -# For external algorithm -import subprocess -import re - -options = None - -def process_args(extra_options=[]): - global options - - parser = OptionParser(option_list=extra_options) - - parser.add_option("--nd", - action="store_false", dest="deterministic_public_ips", - default=True, - help="turn off deterministic_public_ips") - parser.add_option("--ni", - action="store_true", dest="no_ip_failback", default=False, - help="turn on no_ip_failback") - parser.add_option("-L", "--lcp2", - action="store_true", dest="lcp2", default=False, - help="use LCP2 IP rebalancing algorithm [default: %default]") - parser.add_option("-e", "--external", - action="store_true", dest="external", default=False, - help="use external test program to implement IP allocation algorithm [default: %default]") - parser.add_option("-b", "--balance", - action="store_true", dest="balance", default=False, - help="show (im)balance information after each event") - parser.add_option("-d", "--diff", - action="store_true", dest="diff", default=False, - help="show IP address movements for each event") - parser.add_option("-n", "--no-print", - action="store_false", dest="show", default=True, - help="don't show IP address layout after each event") - parser.add_option("-v", "--verbose", - action="count", dest="verbose", default=0, - help="print information and actions taken to stdout") - parser.add_option("-r", "--retries", - action="store", type="int", dest="retries", default=5, - help="number of retry loops for rebalancing non-deterministic failback [default: %default]") - parser.add_option("-i", "--iterations", - action="store", type="int", dest="iterations", - default=1000, - help="number of iterations to run in test [default: %default]") - parser.add_option("-o", "--odds", - action="store", type="int", dest="odds", default=4, - help="make the chances of a failover 1 in ODDS [default: %default]") - parser.add_option("-A", "--aggressive", - action="store_true", dest="aggressive", default=False, - help="apply ODDS to try to flip each node [default: %default]") - - def seed_callback(option, opt, value, parser): - random.seed(value) - parser.add_option("-s", "--seed", - action="callback", type="int", callback=seed_callback, - help="initial random number seed for random events") - - parser.add_option("-x", "--exit", - action="store_true", dest="exit", default=False, - help="exit on the 1st gratuitous IP move or IP imbalance") - parser.add_option("-H", "--hard-imbalance-limit", - action="store", type="int", dest="hard_limit", default=1, - help="exceeding this limit causes termination [default: %default]") - parser.add_option("-S", "--soft-imbalance-limit", - action="store", type="int", dest="soft_limit", default=1, - help="exceeding this limit increments a counter [default: %default]") - - (options, args) = parser.parse_args() - - if len(args) != 0: - parser.error("too many arguments") - - # Could use a callback for this or change the default, but - # laziness is sometimes a virtue. ;-) - if options.lcp2: - options.deterministic_public_ips = False - -def print_begin(t, delim='='): - print delim * 40 - print "%s:" % (t) - -def print_end(): - print "-" * 40 - -def verbose_begin(t): - if options.verbose > 0: - print_begin(t) - -def verbose_end(): - if options.verbose > 0: - print_end() - -def verbose_print(t): - if options.verbose > 0: - if not type(t) == list: - t = [t] - if t != []: - print "\n".join([str(i) for i in t]) - -# more than this and we switch to the logging module... :-) -def debug_begin(t): - if options.verbose > 1: - print_begin(t, '-') - -def debug_end(): - if options.verbose > 1: - print_end() - -def debug_print(t): - if options.verbose > 1: - if not type(t) == list: - t = [t] - if t != []: - print "\n".join([str(i) for i in t]) - -def ip_to_list_of_ints(ip): - # Be lazy... but only expose errors in IPv4 addresses, since - # they'll be more commonly used. :-) - try: - l = socket.inet_pton(socket.AF_INET6, ip) - except: - # Pad with leading 0s. This makes IPv4 addresses comparable - # with IPv6 but reduces the overall effectiveness of the - # algorithm. The alternative would be to treat these - # addresses separately while trying to keep all the IPs in - # overall balance. - l = "".join(itertools.repeat("\0", 12)) + \ - socket.inet_pton(socket.AF_INET, ip) - - return map(lambda x: struct.unpack('B', x)[0], l) - -def ip_distance(ip1, ip2): - """Calculate the distance between 2 IPs. - - This is the length of the longtest common prefix between the IPs. - It is calculated by XOR-ing the 2 IPs together and counting the - number of leading zeroes.""" - - distance = 0 - for (o1, o2) in zip(ip_to_list_of_ints(ip1), ip_to_list_of_ints(ip2)): - # XOR this pair of octets - x = o1 ^ o2 - # count number leading zeroes - if x == 0: - distance += 8 - else: - # bin() gives minimal length '0bNNN' string - distance += (8 - (len(bin(x)) - 2)) - break - - return distance - -def ip_distance_2_sum(ip, ips): - """Calculate the IP distance for the given IP relative to IPs. - - This could be made more efficient by insering ip_distance_2 into - the loop in this function. However, that would result in some - loss of clarity and also will not be necessary in a C - implemntation.""" - - sum = 0 - for i in ips: - sum += ip_distance(ip, i) ** 2 - - return sum - -def imbalance_metric(ips): - """Return the imbalance metric for a group of IPs. - - This is the sum of squares of the IP distances between each pair of IPs.""" - if len(ips) > 1: - (h, t) = (ips[0], ips[1:]) - return ip_distance_2_sum(h, t) + imbalance_metric(t) - else: - return 0 - -def mean(l): - return float(sum(l))/len(l) - -class Node(object): - def __init__(self, public_addresses): - # List of list allows groups of IPs to be passed in. They're - # not actually used in the algorithm but are just used by - # calculate_imbalance() for checking the simulation. Note - # that people can pass in garbage and make this code - # fail... but we're all friends here in simulation world... - # :-) - if type(public_addresses[0]) is str: - self.public_addresses = set(public_addresses) - self.ip_groups = [] - else: - # flatten - self.public_addresses = set([i for s in public_addresses for i in s]) - self.ip_groups = public_addresses - - self.current_addresses = set() - self.healthy = True - self.imbalance = -1 - - def __str__(self): - return "%s %s%s" % \ - ("*" if len(self.public_addresses) == 0 else \ - (" " if self.healthy else "#"), - sorted(list(self.current_addresses)), - " %d" % self.imbalance if options.lcp2 else "") - - def can_node_serve_ip(self, ip): - return ip in self.public_addresses - - def node_ip_coverage(self, ips=None): - return len([a for a in self.current_addresses if ips == None or a in ips]) - - def set_imbalance(self, imbalance=-1): - """Set the imbalance metric to the given value. If none given - then calculate it.""" - - if imbalance != -1: - self.imbalance = imbalance - else: - self.imbalance = imbalance_metric(list(self.current_addresses)) - - def get_imbalance(self): - return self.imbalance - -class Cluster(object): - def __init__(self): - self.nodes = [] - self.deterministic_public_ips = options.deterministic_public_ips - self.no_ip_failback = options.no_ip_failback - self.all_public_ips = set() - - # Statistics - self.ip_moves = [] - self.grat_ip_moves = [] - self.imbalance = [] - self.imbalance_groups = [] - self.imbalance_count = 0 - self.imbalance_groups_count = itertools.repeat(0) - self.imbalance_metric = [] - self.events = -1 - self.num_unhealthy = [] - - self.prev = None - - def __str__(self): - return "\n".join(["%2d %s" % (i, n) \ - for (i, n) in enumerate(self.nodes)]) - - # This is naive. It assumes that IP groups are indicated by the - # 1st node having IP groups. - def have_ip_groups(self): - return (len(self.nodes[0].ip_groups) > 0) - - def print_statistics(self): - print_begin("STATISTICS") - print "Events: %6d" % self.events - print "Total IP moves: %6d" % sum(self.ip_moves) - print "Gratuitous IP moves: %6d" % sum(self.grat_ip_moves) - print "Max imbalance: %6d" % max(self.imbalance) - if self.have_ip_groups(): - print "Max group imbalance counts: ", map(max, zip(*self.imbalance_groups)) - print "Mean imbalance: %f" % mean(self.imbalance) - if self.have_ip_groups(): - print "Mean group imbalances counts: ", map(mean, zip(*self.imbalance_groups)) - print "Final imbalance: %6d" % self.imbalance[-1] - if self.have_ip_groups(): - print "Final group imbalances: ", self.imbalance_groups[-1] - if options.lcp2: - print "Max LCP2 imbalance : %6d" % max(self.imbalance_metric) - print "Soft imbalance count: %6d" % self.imbalance_count - if self.have_ip_groups(): - print "Soft imbalance group counts: ", self.imbalance_groups_count - if options.lcp2: - print "Final LCP2 imbalance : %6d" % self.imbalance_metric[-1] - print "Maximum unhealthy: %6d" % max(self.num_unhealthy) - print_end() - - def find_pnn_with_ip(self, ip): - for (i, n) in enumerate(self.nodes): - if ip in n.current_addresses: - return i - return -1 - - def quietly_remove_ip(self, ip): - # Remove address from old node. - old = self.find_pnn_with_ip(ip) - if old != -1: - self.nodes[old].current_addresses.remove(ip) - - def add_node(self, node): - self.nodes.append(node) - self.all_public_ips |= node.public_addresses - - def healthy(self, *pnns): - verbose_begin("HEALTHY") - - for pnn in pnns: - self.nodes[pnn].healthy = True - verbose_print(pnn) - - verbose_end() - - def unhealthy(self, *pnns): - - verbose_begin("UNHEALTHY") - - for pnn in pnns: - self.nodes[pnn].healthy = False - verbose_print(pnn) - - verbose_end() - - def do_something_random(self): - - """Make random node(s) healthy or unhealthy. - - If options.aggressive is False then: If all nodes are healthy - or unhealthy, then invert one of them; otherwise, there's a 1 - in options.odds chance of making another node unhealthy. - - If options.aggressive is True then: For each node there is a 1 - in options.odds chance of flipping the state of that node - between healthy and unhealthy.""" - - if not options.aggressive: - num_nodes = len(self.nodes) - healthy_pnns = [i for (i,n) in enumerate(self.nodes) if n.healthy] - num_healthy = len(healthy_pnns) - - if num_nodes == num_healthy: - self.unhealthy(random.randint(0, num_nodes-1)) - elif num_healthy == 0: - self.healthy(random.randint(0, num_nodes-1)) - elif random.randint(1, options.odds) == 1: - self.unhealthy(random.choice(healthy_pnns)) - else: - all_pnns = range(num_nodes) - unhealthy_pnns = sorted(list(set(all_pnns) - set(healthy_pnns))) - self.healthy(random.choice(unhealthy_pnns)) - else: - # We need to make at least one change or we retry...x - changed = False - while not changed: - for (pnn, n) in enumerate(self.nodes): - if random.randint(1, options.odds) == 1: - changed = True - if n.healthy: - self.unhealthy(pnn) - else: - self.healthy(pnn) - - def random_iterations(self): - i = 1 - while i <= options.iterations: - verbose_begin("EVENT %d" % i) - verbose_end() - self.do_something_random() - if self.recover() and options.exit: - break - i += 1 - - self.print_statistics() - - def imbalance_for_ips(self, ips): - - imbalance = 0 - - maxnode = -1 - minnode = -1 - - for ip in ips: - for (i, n) in enumerate(self.nodes): - - if not n.healthy or not n.can_node_serve_ip(ip): - continue - - num = n.node_ip_coverage(ips) - - if maxnode == -1 or num > maxnum: - maxnode = i - maxnum = num - - if minnode == -1 or num < minnum: - minnode = i - minnum = num - - if maxnode == -1 or minnode == -1: - continue - - i = maxnum - minnum - #if i < 2: - # i = 0 - imbalance = max([imbalance, i]) - - return imbalance - - - def calculate_imbalance(self): - - # First, do all the assigned IPs. - assigned = sorted([ip - for n in self.nodes - for ip in n.current_addresses]) - - i = self.imbalance_for_ips(assigned) - - ig = [] - # FIXME? If dealing with IP groups, assume the nodes are all - # the same. - for ips in self.nodes[0].ip_groups: - gi = self.imbalance_for_ips(ips) - ig.append(gi) - - return (i, ig) - - - def diff(self): - """Calculate differences in IP assignments between self and prev. - - Gratuitous IP moves (from a healthy node to a healthy node) - are prefixed by !!.""" - - ip_moves = 0 - grat_ip_moves = 0 - details = [] - - for (new, n) in enumerate(self.nodes): - for ip in n.current_addresses: - old = self.prev.find_pnn_with_ip(ip) - if old != new: - ip_moves += 1 - if old != -1 and \ - self.prev.nodes[new].healthy and \ - self.nodes[new].healthy and \ - self.nodes[old].healthy and \ - self.prev.nodes[old].healthy: - prefix = "!!" - grat_ip_moves += 1 - else: - prefix = " " - details.append("%s %s: %d -> %d" % - (prefix, ip, old, new)) - - return (ip_moves, grat_ip_moves, details) - - def find_takeover_node(self, ip): - - pnn = -1 - min = 0 - for (i, n) in enumerate(self.nodes): - if not n.healthy: - continue - - if not n.can_node_serve_ip(ip): - continue - - num = n.node_ip_coverage() - - if (pnn == -1): - pnn = i - min = num - else: - if num < min: - pnn = i - min = num - - if pnn == -1: - verbose_print("Could not find node to take over public address %s" % ip) - return False - - self.nodes[pnn].current_addresses.add(ip) - - verbose_print("%s -> %d" % (ip, pnn)) - return True - - def basic_allocate_unassigned(self): - - assigned = set([ip for n in self.nodes for ip in n.current_addresses]) - unassigned = sorted(list(self.all_public_ips - assigned)) - - for ip in unassigned: - self.find_takeover_node(ip) - - def basic_failback(self, retries_l): - - assigned = sorted([ip - for n in self.nodes - for ip in n.current_addresses]) - for ip in assigned: - - maxnode = -1 - minnode = -1 - for (i, n) in enumerate(self.nodes): - if not n.healthy: - continue - - if not n.can_node_serve_ip(ip): - continue - - num = n.node_ip_coverage() - - if maxnode == -1: - maxnode = i - maxnum = num - else: - if num > maxnum: - maxnode = i - maxnum = num - if minnode == -1: - minnode = i - minnum = num - else: - if num < minnum: - minnode = i - minnum = num - - if maxnode == -1: - print "Could not find maxnode. May not be able to serve ip", ip - continue - - #if self.deterministic_public_ips: - # continue - - if maxnum > minnum + 1 and retries_l[0] < options.retries: - # Remove the 1st ip from maxnode - t = sorted(list(self.nodes[maxnode].current_addresses)) - realloc = t[0] - verbose_print("%s <- %d" % (realloc, maxnode)) - self.nodes[maxnode].current_addresses.remove(realloc) - # Redo the outer loop. - retries_l[0] += 1 - return True - - return False - - - def lcp2_allocate_unassigned(self): - - # Assign as many unassigned addresses as possible. Keep - # selecting the optimal assignment until we don't manage to - # assign anything. - assigned = set([ip for n in self.nodes for ip in n.current_addresses]) - unassigned = sorted(list(self.all_public_ips - assigned)) - - should_loop = True - while len(unassigned) > 0 and should_loop: - should_loop = False - - debug_begin(" CONSIDERING MOVES (UNASSIGNED)") - - minnode = -1 - mindsum = 0 - minip = None - - for ip in unassigned: - for dstnode in range(len(self.nodes)): - if self.nodes[dstnode].can_node_serve_ip(ip) and \ - self.nodes[dstnode].healthy: - dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses) - dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum - debug_print(" %s -> %d [+%d]" % \ - (ip, - dstnode, - dstimbl - self.nodes[dstnode].get_imbalance())) - - if (minnode == -1) or (dstdsum < mindsum): - minnode = dstnode - minimbl = dstimbl - mindsum = dstdsum - minip = ip - should_loop = True - debug_end() - - if minnode != -1: - self.nodes[minnode].current_addresses.add(minip) - self.nodes[minnode].set_imbalance(self.nodes[minnode].get_imbalance() + mindsum) - verbose_print("%s -> %d [+%d]" % (minip, minnode, mindsum)) - unassigned.remove(minip) - - for ip in unassigned: - verbose_print("Could not find node to take over public address %s" % ip) - - def lcp2_failback(self, targets): - - # Get the node with the highest imbalance metric. - srcnode = -1 - maximbl = 0 - for (pnn, n) in enumerate(self.nodes): - b = n.get_imbalance() - if (srcnode == -1) or (b > maximbl): - srcnode = pnn - maximbl = b - - # This means that all nodes had 0 or 1 addresses, so can't - # be imbalanced. - if maximbl == 0: - return False - - # We'll need this a few times... - ips = self.nodes[srcnode].current_addresses - - # Find an IP and destination node that best reduces imbalance. - optimum = None - debug_begin(" CONSIDERING MOVES FROM %d [%d]" % (srcnode, maximbl)) - for ip in ips: - # What is this IP address costing the source node? - srcdsum = ip_distance_2_sum(ip, ips - set([ip])) - srcimbl = maximbl - srcdsum - - # Consider this IP address would cost each potential - # destination node. Destination nodes are limited to - # those that are newly healthy, since we don't want to - # do gratuitous failover of IPs just to make minor - # balance improvements. - for dstnode in targets: - if self.nodes[dstnode].can_node_serve_ip(ip) and \ - self.nodes[dstnode].healthy: - dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses) - dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum - debug_print(" %d [%d] -> %s -> %d [+%d]" % \ - (srcnode, - srcimbl - self.nodes[srcnode].get_imbalance(), - ip, - dstnode, - dstimbl - self.nodes[dstnode].get_imbalance())) - - if (dstimbl < maximbl) and (dstdsum < srcdsum): - if optimum is None: - optimum = (ip, srcnode, srcimbl, dstnode, dstimbl) - else: - (x, sn, si, dn, di) = optimum - if (srcimbl + dstimbl) < (si + di): - optimum = (ip, srcnode, srcimbl, dstnode, dstimbl) - debug_end() - - if optimum is not None: - # We found a move that makes things better... - (ip, srcnode, srcimbl, dstnode, dstimbl) = optimum - ini_srcimbl = self.nodes[srcnode].get_imbalance() - ini_dstimbl = self.nodes[dstnode].get_imbalance() - - self.nodes[srcnode].current_addresses.remove(ip) - self.nodes[srcnode].set_imbalance(srcimbl) - - self.nodes[dstnode].current_addresses.add(ip) - self.nodes[dstnode].set_imbalance(dstimbl) - - verbose_print("%d [%d] -> %s -> %d [+%d]" % \ - (srcnode, - srcimbl - ini_srcimbl, - ip, - dstnode, - dstimbl - ini_dstimbl)) - - return True - - return False - - def ctdb_takeover_run_python(self): - - # Don't bother with the num_healthy stuff. It is an - # irrelevant detail. - - # We just keep the allocate IPs in the current_addresses field - # of the node. This needs to readable, not efficient! - - if self.deterministic_public_ips: - # Remap everything. - addr_list = sorted(list(self.all_public_ips)) - for (i, ip) in enumerate(addr_list): - self.quietly_remove_ip(ip) - # Add addresses to new node. - pnn = i % len(self.nodes) - self.nodes[pnn].current_addresses.add(ip) - verbose_print("%s -> %d" % (ip, pnn)) - - # Remove public addresses from unhealthy nodes. - for (pnn, n) in enumerate(self.nodes): - if not n.healthy: - verbose_print(["%s <- %d" % (ip, pnn) - for ip in n.current_addresses]) - n.current_addresses = set() - - # If a node can't serve an assigned address then remove it. - for n in self.nodes: - verbose_print(["%s <- %d" % (ip, pnn) - for ip in n.current_addresses - n.public_addresses]) - n.current_addresses &= n.public_addresses - - if options.lcp2: - newly_healthy = [pnn for (pnn, n) in enumerate(self.nodes) - if len(n.current_addresses) == 0 and n.healthy] - for n in self.nodes: - n.set_imbalance() - - # We'll only retry the balancing act up to options.retries - # times (for the basic non-deterministic algorithm). This - # nonsense gives us a reference on the retries count in - # Python. It will be easier in C. :-) - # For LCP2 we reassignas many IPs from heavily "loaded" nodes - # to nodes that are newly healthy, looping until we fail to - # reassign an IP. - retries_l = [0] - should_loop = True - while should_loop: - should_loop = False - - if options.lcp2: - self.lcp2_allocate_unassigned() - else: - self.basic_allocate_unassigned() - - if self.no_ip_failback or self.deterministic_public_ips: - break - - if options.lcp2: - if len(newly_healthy) == 0: - break - should_loop = self.lcp2_failback(newly_healthy) - else: - should_loop = self.basic_failback(retries_l) - - def ctdb_takeover_run_external(self): - - # Written while asleep... - - # Convert the cluster state to something that be fed to - # ctdb_takeover_tests ctdb_takeover_run_core ... - - in_lines = [] - for ip in sorted(list(self.all_public_ips)): - allowed = [] - assigned = -1 - for (i, n) in enumerate(self.nodes): - if n.can_node_serve_ip(ip): - allowed.append("%s" % i) - if ip in n.current_addresses: - assigned = i - line = "%s\t%d\t%s" % (ip, assigned, ",".join(allowed)) - in_lines.append(line) - - nodestates = ",".join(["0" if n.healthy else "1" for n in self.nodes]) - - if options.lcp2: - os.environ["CTDB_LCP2"] = "yes" - if options.verbose > 1: - os.environ["CTDB_TEST_LOGLEVEL"] = "4" - elif options.verbose == 1: - os.environ["CTDB_TEST_LOGLEVEL"] = "3" - else: - os.environ["CTDB_TEST_LOGLEVEL"] = "0" - - p = subprocess.Popen("../bin/ctdb_takeover_tests ctdb_takeover_run_core %s 2>&1" % nodestates, - shell=True, - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - p.stdin.write("\n".join(in_lines)) - p.stdin.close() - - # Flush all of the assigned IPs. - for n in self.nodes: - n.current_addresses = set() - - # Uses the results to populate the current_addresses for each - # node. - for line in p.stdout.read().split("\n"): - # Some lines are debug, some are the final IP - # configuration. Let's use a gross hack that assumes any - # line with 2 words is IP configuration. That will do for - # now. - words = re.split("\s+", line) - if len(words) == 2: - # Add the IP as current for the specified node. - self.nodes[int(words[1])].current_addresses.add(words[0]) - else: - # First 3 words are log date/time, remove them... - print " ".join(words[3:]) - - # Now fake up the LCP calculations. - for n in self.nodes: - n.set_imbalance() - - def ctdb_takeover_run(self): - - self.events += 1 - - if options.external: - return self.ctdb_takeover_run_external() - else: - return self.ctdb_takeover_run_python() - - def recover(self): - verbose_begin("TAKEOVER") - - self.ctdb_takeover_run() - - verbose_end() - - grat_ip_moves = 0 - - if self.prev is not None: - (ip_moves, grat_ip_moves, details) = self.diff() - self.ip_moves.append(ip_moves) - self.grat_ip_moves.append(grat_ip_moves) - - if options.diff: - print_begin("DIFF") - print "\n".join(details) - print_end() - - (imbalance, imbalance_groups) = self.calculate_imbalance() - self.imbalance.append(imbalance) - self.imbalance_groups.append(imbalance_groups) - - if imbalance > options.soft_limit: - self.imbalance_count += 1 - - # There must be a cleaner way... - t = [] - for (c, i) in zip(self.imbalance_groups_count, imbalance_groups): - if i > options.soft_limit: - t.append(c + i) - else: - t.append(c) - self.imbalance_groups_count = t - - imbalance_metric = max([n.get_imbalance() for n in self.nodes]) - self.imbalance_metric.append(imbalance_metric) - if options.balance: - print_begin("IMBALANCE") - print "ALL IPS:", imbalance - if self.have_ip_groups(): - print "IP GROUPS:", imbalance_groups - if options.lcp2: - print "LCP2 IMBALANCE:", imbalance_metric - print_end() - - num_unhealthy = len(self.nodes) - \ - len([n for n in self.nodes if n.healthy]) - self.num_unhealthy.append(num_unhealthy) - - if options.show: - print_begin("STATE") - print self - print_end() - - self.prev = None - self.prev = copy.deepcopy(self) - - # True is bad! - return (grat_ip_moves > 0) or \ - (not self.have_ip_groups() and imbalance > options.hard_limit) or \ - (self.have_ip_groups() and (max(imbalance_groups) > options.hard_limit)) diff --git a/ctdb/tests/takeover/ip_groups1.py b/ctdb/tests/takeover/ip_groups1.py deleted file mode 100755 index 0808f466cf..0000000000 --- a/ctdb/tests/takeover/ip_groups1.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -# 2 IP groups, both on the same 5 nodes, with each group on different -# interfaces/VLANs. One group has many more addresses to test how -# well an "imbalanced" configuration will balance... - -from ctdb_takeover import Cluster, Node, process_args - -process_args() - -addresses20 = ['192.168.20.%d' % n for n in range(1, 13)] -addresses128 = ['192.168.128.%d' % n for n in range(1, 5)] - -c = Cluster() - -for i in range(5): - c.add_node(Node([addresses20, addresses128])) - -#for i in range(3): -# c.add_node(Node([addresses20])) - - -c.recover() - -c.random_iterations() diff --git a/ctdb/tests/takeover/ip_groups2.py b/ctdb/tests/takeover/ip_groups2.py deleted file mode 100755 index c6c1026646..0000000000 --- a/ctdb/tests/takeover/ip_groups2.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python - -# 2 groups of addresses, combined into 1 pool so the checking -# algorithm doesn't know about the groups, across 2 nodes. - -from ctdb_takeover import Cluster, Node, process_args - -process_args() - -addresses20 = ['192.168.20.%d' % n for n in range(1, 13)] -addresses21 = ['192.168.21.%d' % n for n in range(1, 5)] - -c = Cluster() - -for i in range(2): - c.add_node(Node(addresses20 + addresses21)) - -c.recover() - -c.random_iterations() diff --git a/ctdb/tests/takeover/ip_groups3.py b/ctdb/tests/takeover/ip_groups3.py deleted file mode 100755 index 149946d72b..0000000000 --- a/ctdb/tests/takeover/ip_groups3.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -# 4 IP groups, across 10 nodes, with each group on different -# interfaces/VLANs. 80 addresses in total but not evenly balanced, to -# help check some of the more extreme behaviour. - -from ctdb_takeover import Cluster, Node, process_args - -process_args() - -addresses1 = ['192.168.1.%d' % n for n in range(1, 41)] -addresses2 = ['192.168.2.%d' % n for n in range(1, 21)] -addresses3 = ['192.168.3.%d' % n for n in range(1, 11)] -addresses4 = ['192.168.4.%d' % n for n in range(1, 11)] - -# Try detecting imbalance with square root of number of nodes? Or -# just with a parameter indicating how unbalanced you're willing to -# accept... - -c = Cluster() - -for i in range(10): - c.add_node(Node([addresses1, addresses2, addresses3, addresses4])) - -c.recover() - -c.random_iterations() diff --git a/ctdb/tests/takeover/ip_groups4.py b/ctdb/tests/takeover/ip_groups4.py deleted file mode 100755 index fdcef7f0a6..0000000000 --- a/ctdb/tests/takeover/ip_groups4.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python - -# 2 IP groups, across 2 nodes, with each group on different -# interfaces. 4 addresses per group. A nice little canonical 2 node -# configuration. - -from ctdb_takeover import Cluster, Node, process_args - -process_args() - -addresses1 = ['192.168.1.%d' % n for n in range(1, 5)] -addresses2 = ['192.168.2.%d' % n for n in range(1, 5)] - -# Try detecting imbalance with square root of number of nodes? Or -# just with a parameter indicating how unbalanced you're willing to -# accept... - -c = Cluster() - -for i in range(2): - c.add_node(Node([addresses1, addresses2])) - -c.recover() - -c.random_iterations() diff --git a/ctdb/tests/takeover/ip_groups5.py b/ctdb/tests/takeover/ip_groups5.py deleted file mode 100755 index 8c46150638..0000000000 --- a/ctdb/tests/takeover/ip_groups5.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python - -# 1 IP group, to test backward compatibility of LCP2 algorithm. 16 -# addresses across 4 nodes. - -from ctdb_takeover import Cluster, Node, process_args - -process_args() - -addresses1 = ['192.168.1.%d' % n for n in range(1, 17)] - -# Try detecting imbalance with square root of number of nodes? Or -# just with a parameter indicating how unbalanced you're willing to -# accept... - -c = Cluster() - -for i in range(4): - c.add_node(Node(addresses1)) - -c.recover() - -c.random_iterations() diff --git a/ctdb/tests/takeover/lcp2.001.sh b/ctdb/tests/takeover/lcp2.001.sh new file mode 100755 index 0000000000..8772318d6e --- /dev/null +++ b/ctdb/tests/takeover/lcp2.001.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "3 nodes, 3 -> 1 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < 2 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < all healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < all healthy, info logging" + +export CTDB_TEST_LOGLEVEL=3 + +required_result < 192.168.20.253 -> 0 [+0] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 1 +192.168.20.254 2 +192.168.20.253 0 +192.168.20.252 1 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 1 +EOF + +simple_test 0,0,0 < all healthy, debug logging" + +export CTDB_TEST_LOGLEVEL=4 + +required_result < 192.168.21.254 -> 0 [+0] +DATE TIME [PID]: 1 [-116718] -> 192.168.21.254 -> 2 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.253 -> 0 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.253 -> 2 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.252 -> 0 [+0] +DATE TIME [PID]: 1 [-116971] -> 192.168.21.252 -> 2 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.254 -> 0 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.254 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.252 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.252 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.251 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.250 -> 0 [+0] +DATE TIME [PID]: 1 [-121363] -> 192.168.20.250 -> 2 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.249 -> 0 [+0] +DATE TIME [PID]: 1 [-121110] -> 192.168.20.249 -> 2 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [418056] +DATE TIME [PID]: 1 [-102557] -> 192.168.21.254 -> 0 [+14161] +DATE TIME [PID]: 1 [-102557] -> 192.168.21.254 -> 2 [+0] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.253 -> 2 [+0] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.252 -> 0 [+14161] +DATE TIME [PID]: 1 [-102810] -> 192.168.21.252 -> 2 [+0] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.254 -> 0 [+15876] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.254 -> 2 [+0] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.252 -> 0 [+16129] +DATE TIME [PID]: 1 [-105234] -> 192.168.20.252 -> 2 [+0] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 0 [+15625] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.250 -> 0 [+15625] +DATE TIME [PID]: 1 [-105738] -> 192.168.20.250 -> 2 [+0] +DATE TIME [PID]: 1 [-105485] -> 192.168.20.249 -> 0 [+15625] +DATE TIME [PID]: 1 [-105485] -> 192.168.20.249 -> 2 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [312571] +DATE TIME [PID]: 1 [-88396] -> 192.168.21.254 -> 0 [+14161] +DATE TIME [PID]: 1 [-88396] -> 192.168.21.254 -> 2 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 2 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.252 -> 0 [+14161] +DATE TIME [PID]: 1 [-88649] -> 192.168.21.252 -> 2 [+14161] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.254 -> 0 [+15876] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.252 -> 0 [+16129] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.252 -> 2 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.250 -> 0 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.250 -> 2 [+16129] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.249 -> 0 [+15625] +DATE TIME [PID]: 1 [-89609] -> 192.168.20.249 -> 2 [+15876] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [222962] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.254 -> 0 [+30037] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.254 -> 2 [+14161] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-72520] -> 192.168.21.252 -> 2 [+14161] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 0 [+30037] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.252 -> 2 [+15625] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.250 -> 2 [+16129] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.249 -> 0 [+29786] +DATE TIME [PID]: 1 [-75448] -> 192.168.20.249 -> 2 [+15876] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [147514] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.254 -> 0 [+30037] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-58359] -> 192.168.21.252 -> 2 [+28322] +DATE TIME [PID]: 1 [-59572] -> 192.168.20.252 -> 0 [+30290] +DATE TIME [PID]: 1 [-59572] -> 192.168.20.252 -> 2 [+31501] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 2 [+31754] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.249 -> 0 [+29786] +DATE TIME [PID]: 1 [-59823] -> 192.168.20.249 -> 2 [+31501] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [87691] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 0 [+44198] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.252 -> 0 [+44451] +DATE TIME [PID]: 1 [-44198] -> 192.168.21.252 -> 2 [+28322] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.252 -> 0 [+45915] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.252 -> 2 [+31501] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.249 -> 0 [+45662] +DATE TIME [PID]: 1 [-43947] -> 192.168.20.249 -> 2 [+31501] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 0 [43947] +DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 0 [+28322] +DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 2 [+44198] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.253 -> 0 [+29786] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.253 -> 2 [+45662] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 0 [+29786] +DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 2 [+45915] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 2 [43947] +DATE TIME [PID]: 2 [-28322] -> 192.168.21.254 -> 0 [+44198] +DATE TIME [PID]: 2 [-28322] -> 192.168.21.254 -> 2 [+28322] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.254 -> 0 [+45662] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.254 -> 2 [+29786] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.251 -> 0 [+45915] +DATE TIME [PID]: 2 [-29786] -> 192.168.20.251 -> 2 [+29786] +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: ---------------------------------------- +DATE TIME [PID]: CONSIDERING MOVES FROM 1 [43744] +DATE TIME [PID]: 1 [-28322] -> 192.168.21.252 -> 0 [+44451] +DATE TIME [PID]: 1 [-28322] -> 192.168.21.252 -> 2 [+44198] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.252 -> 0 [+45915] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.252 -> 2 [+45662] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.249 -> 0 [+45662] +DATE TIME [PID]: 1 [-29786] -> 192.168.20.249 -> 2 [+45662] +DATE TIME [PID]: ---------------------------------------- +192.168.21.254 2 +192.168.21.253 0 +192.168.21.252 1 +192.168.20.254 2 +192.168.20.253 0 +192.168.20.252 1 +192.168.20.251 2 +192.168.20.250 0 +192.168.20.249 1 +EOF + +simple_test 0,0,0 < 1 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < 2 healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < all healthy" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < all disconnected" + +export CTDB_TEST_LOGLEVEL=0 + +required_result < all healthy" + +required_result </dev/null) -fi - -"$test_dir/scripts/run_tests" $opts "$@" $tests || exit 1 - -echo "All OK" -exit 0 diff --git a/ctdb/tests/takeover/scripts/local.sh b/ctdb/tests/takeover/scripts/local.sh new file mode 100644 index 0000000000..0e6230ab6f --- /dev/null +++ b/ctdb/tests/takeover/scripts/local.sh @@ -0,0 +1,58 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +test_prog="$(dirname ${TEST_SUBDIR})/bin/ctdb_takeover_tests ctdb_takeover_run_core" + +define_test () +{ + _f=$(basename "$0" ".sh") + + case "$_f" in + nondet.*) + algorithm="nondet" + export CTDB_LCP2="no" + ;; + lcp2.*) + algorithm="lcp2" + export CTDB_LCP2="yes" + ;; + *) + die "Unknown algorithm for testcase \"$_f\"" + esac + + printf "%-12s - %s\n" "$_f" "$1" +} + +simple_test () +{ + _states="$1" + _out=$($test_prog $_states 2>&1) + _rc=$? + + if [ "$algorithm" = "lcp2" -a -n "$CTDB_TEST_LOGLEVEL" ] ; then + OUT_FILTER='s@^.*:@DATE TIME \[PID\]:@' + fi + + if [ -n "$OUT_FILTER" ] ; then + _fout=$(echo "$_out" | sed -r "$OUT_FILTER") + else + _fout="$_out" + fi + + if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then + echo "PASSED" + else + cat <. diff --git a/ctdb/tests/takeover/simulation/ctdb_takeover.py b/ctdb/tests/takeover/simulation/ctdb_takeover.py new file mode 100755 index 0000000000..4b7ceef468 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ctdb_takeover.py @@ -0,0 +1,888 @@ +#!/usr/bin/env python + +# ctdb ip takeover code + +# Copyright (C) Martin Schwenke, Ronnie Sahlberg 2010, 2011 + +# Based on original CTDB C code: +# +# Copyright (C) Ronnie Sahlberg 2007 +# Copyright (C) Andrew Tridgell 2007 + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + + +import os +import sys +# Use optparse since newer argparse not available in RHEL5/EPEL. +from optparse import OptionParser +import copy +import random +import itertools + +# For parsing IP addresses +import socket +import struct + +# For external algorithm +import subprocess +import re + +options = None + +def process_args(extra_options=[]): + global options + + parser = OptionParser(option_list=extra_options) + + parser.add_option("--nd", + action="store_false", dest="deterministic_public_ips", + default=True, + help="turn off deterministic_public_ips") + parser.add_option("--ni", + action="store_true", dest="no_ip_failback", default=False, + help="turn on no_ip_failback") + parser.add_option("-L", "--lcp2", + action="store_true", dest="lcp2", default=False, + help="use LCP2 IP rebalancing algorithm [default: %default]") + parser.add_option("-e", "--external", + action="store_true", dest="external", default=False, + help="use external test program to implement IP allocation algorithm [default: %default]") + parser.add_option("-b", "--balance", + action="store_true", dest="balance", default=False, + help="show (im)balance information after each event") + parser.add_option("-d", "--diff", + action="store_true", dest="diff", default=False, + help="show IP address movements for each event") + parser.add_option("-n", "--no-print", + action="store_false", dest="show", default=True, + help="don't show IP address layout after each event") + parser.add_option("-v", "--verbose", + action="count", dest="verbose", default=0, + help="print information and actions taken to stdout") + parser.add_option("-r", "--retries", + action="store", type="int", dest="retries", default=5, + help="number of retry loops for rebalancing non-deterministic failback [default: %default]") + parser.add_option("-i", "--iterations", + action="store", type="int", dest="iterations", + default=1000, + help="number of iterations to run in test [default: %default]") + parser.add_option("-o", "--odds", + action="store", type="int", dest="odds", default=4, + help="make the chances of a failover 1 in ODDS [default: %default]") + parser.add_option("-A", "--aggressive", + action="store_true", dest="aggressive", default=False, + help="apply ODDS to try to flip each node [default: %default]") + + def seed_callback(option, opt, value, parser): + random.seed(value) + parser.add_option("-s", "--seed", + action="callback", type="int", callback=seed_callback, + help="initial random number seed for random events") + + parser.add_option("-x", "--exit", + action="store_true", dest="exit", default=False, + help="exit on the 1st gratuitous IP move or IP imbalance") + parser.add_option("-H", "--hard-imbalance-limit", + action="store", type="int", dest="hard_limit", default=1, + help="exceeding this limit causes termination [default: %default]") + parser.add_option("-S", "--soft-imbalance-limit", + action="store", type="int", dest="soft_limit", default=1, + help="exceeding this limit increments a counter [default: %default]") + + (options, args) = parser.parse_args() + + if len(args) != 0: + parser.error("too many arguments") + + # Could use a callback for this or change the default, but + # laziness is sometimes a virtue. ;-) + if options.lcp2: + options.deterministic_public_ips = False + +def print_begin(t, delim='='): + print delim * 40 + print "%s:" % (t) + +def print_end(): + print "-" * 40 + +def verbose_begin(t): + if options.verbose > 0: + print_begin(t) + +def verbose_end(): + if options.verbose > 0: + print_end() + +def verbose_print(t): + if options.verbose > 0: + if not type(t) == list: + t = [t] + if t != []: + print "\n".join([str(i) for i in t]) + +# more than this and we switch to the logging module... :-) +def debug_begin(t): + if options.verbose > 1: + print_begin(t, '-') + +def debug_end(): + if options.verbose > 1: + print_end() + +def debug_print(t): + if options.verbose > 1: + if not type(t) == list: + t = [t] + if t != []: + print "\n".join([str(i) for i in t]) + +def ip_to_list_of_ints(ip): + # Be lazy... but only expose errors in IPv4 addresses, since + # they'll be more commonly used. :-) + try: + l = socket.inet_pton(socket.AF_INET6, ip) + except: + # Pad with leading 0s. This makes IPv4 addresses comparable + # with IPv6 but reduces the overall effectiveness of the + # algorithm. The alternative would be to treat these + # addresses separately while trying to keep all the IPs in + # overall balance. + l = "".join(itertools.repeat("\0", 12)) + \ + socket.inet_pton(socket.AF_INET, ip) + + return map(lambda x: struct.unpack('B', x)[0], l) + +def ip_distance(ip1, ip2): + """Calculate the distance between 2 IPs. + + This is the length of the longtest common prefix between the IPs. + It is calculated by XOR-ing the 2 IPs together and counting the + number of leading zeroes.""" + + distance = 0 + for (o1, o2) in zip(ip_to_list_of_ints(ip1), ip_to_list_of_ints(ip2)): + # XOR this pair of octets + x = o1 ^ o2 + # count number leading zeroes + if x == 0: + distance += 8 + else: + # bin() gives minimal length '0bNNN' string + distance += (8 - (len(bin(x)) - 2)) + break + + return distance + +def ip_distance_2_sum(ip, ips): + """Calculate the IP distance for the given IP relative to IPs. + + This could be made more efficient by insering ip_distance_2 into + the loop in this function. However, that would result in some + loss of clarity and also will not be necessary in a C + implemntation.""" + + sum = 0 + for i in ips: + sum += ip_distance(ip, i) ** 2 + + return sum + +def imbalance_metric(ips): + """Return the imbalance metric for a group of IPs. + + This is the sum of squares of the IP distances between each pair of IPs.""" + if len(ips) > 1: + (h, t) = (ips[0], ips[1:]) + return ip_distance_2_sum(h, t) + imbalance_metric(t) + else: + return 0 + +def mean(l): + return float(sum(l))/len(l) + +class Node(object): + def __init__(self, public_addresses): + # List of list allows groups of IPs to be passed in. They're + # not actually used in the algorithm but are just used by + # calculate_imbalance() for checking the simulation. Note + # that people can pass in garbage and make this code + # fail... but we're all friends here in simulation world... + # :-) + if type(public_addresses[0]) is str: + self.public_addresses = set(public_addresses) + self.ip_groups = [] + else: + # flatten + self.public_addresses = set([i for s in public_addresses for i in s]) + self.ip_groups = public_addresses + + self.current_addresses = set() + self.healthy = True + self.imbalance = -1 + + def __str__(self): + return "%s %s%s" % \ + ("*" if len(self.public_addresses) == 0 else \ + (" " if self.healthy else "#"), + sorted(list(self.current_addresses)), + " %d" % self.imbalance if options.lcp2 else "") + + def can_node_serve_ip(self, ip): + return ip in self.public_addresses + + def node_ip_coverage(self, ips=None): + return len([a for a in self.current_addresses if ips == None or a in ips]) + + def set_imbalance(self, imbalance=-1): + """Set the imbalance metric to the given value. If none given + then calculate it.""" + + if imbalance != -1: + self.imbalance = imbalance + else: + self.imbalance = imbalance_metric(list(self.current_addresses)) + + def get_imbalance(self): + return self.imbalance + +class Cluster(object): + def __init__(self): + self.nodes = [] + self.deterministic_public_ips = options.deterministic_public_ips + self.no_ip_failback = options.no_ip_failback + self.all_public_ips = set() + + # Statistics + self.ip_moves = [] + self.grat_ip_moves = [] + self.imbalance = [] + self.imbalance_groups = [] + self.imbalance_count = 0 + self.imbalance_groups_count = itertools.repeat(0) + self.imbalance_metric = [] + self.events = -1 + self.num_unhealthy = [] + + self.prev = None + + def __str__(self): + return "\n".join(["%2d %s" % (i, n) \ + for (i, n) in enumerate(self.nodes)]) + + # This is naive. It assumes that IP groups are indicated by the + # 1st node having IP groups. + def have_ip_groups(self): + return (len(self.nodes[0].ip_groups) > 0) + + def print_statistics(self): + print_begin("STATISTICS") + print "Events: %6d" % self.events + print "Total IP moves: %6d" % sum(self.ip_moves) + print "Gratuitous IP moves: %6d" % sum(self.grat_ip_moves) + print "Max imbalance: %6d" % max(self.imbalance) + if self.have_ip_groups(): + print "Max group imbalance counts: ", map(max, zip(*self.imbalance_groups)) + print "Mean imbalance: %f" % mean(self.imbalance) + if self.have_ip_groups(): + print "Mean group imbalances counts: ", map(mean, zip(*self.imbalance_groups)) + print "Final imbalance: %6d" % self.imbalance[-1] + if self.have_ip_groups(): + print "Final group imbalances: ", self.imbalance_groups[-1] + if options.lcp2: + print "Max LCP2 imbalance : %6d" % max(self.imbalance_metric) + print "Soft imbalance count: %6d" % self.imbalance_count + if self.have_ip_groups(): + print "Soft imbalance group counts: ", self.imbalance_groups_count + if options.lcp2: + print "Final LCP2 imbalance : %6d" % self.imbalance_metric[-1] + print "Maximum unhealthy: %6d" % max(self.num_unhealthy) + print_end() + + def find_pnn_with_ip(self, ip): + for (i, n) in enumerate(self.nodes): + if ip in n.current_addresses: + return i + return -1 + + def quietly_remove_ip(self, ip): + # Remove address from old node. + old = self.find_pnn_with_ip(ip) + if old != -1: + self.nodes[old].current_addresses.remove(ip) + + def add_node(self, node): + self.nodes.append(node) + self.all_public_ips |= node.public_addresses + + def healthy(self, *pnns): + verbose_begin("HEALTHY") + + for pnn in pnns: + self.nodes[pnn].healthy = True + verbose_print(pnn) + + verbose_end() + + def unhealthy(self, *pnns): + + verbose_begin("UNHEALTHY") + + for pnn in pnns: + self.nodes[pnn].healthy = False + verbose_print(pnn) + + verbose_end() + + def do_something_random(self): + + """Make random node(s) healthy or unhealthy. + + If options.aggressive is False then: If all nodes are healthy + or unhealthy, then invert one of them; otherwise, there's a 1 + in options.odds chance of making another node unhealthy. + + If options.aggressive is True then: For each node there is a 1 + in options.odds chance of flipping the state of that node + between healthy and unhealthy.""" + + if not options.aggressive: + num_nodes = len(self.nodes) + healthy_pnns = [i for (i,n) in enumerate(self.nodes) if n.healthy] + num_healthy = len(healthy_pnns) + + if num_nodes == num_healthy: + self.unhealthy(random.randint(0, num_nodes-1)) + elif num_healthy == 0: + self.healthy(random.randint(0, num_nodes-1)) + elif random.randint(1, options.odds) == 1: + self.unhealthy(random.choice(healthy_pnns)) + else: + all_pnns = range(num_nodes) + unhealthy_pnns = sorted(list(set(all_pnns) - set(healthy_pnns))) + self.healthy(random.choice(unhealthy_pnns)) + else: + # We need to make at least one change or we retry...x + changed = False + while not changed: + for (pnn, n) in enumerate(self.nodes): + if random.randint(1, options.odds) == 1: + changed = True + if n.healthy: + self.unhealthy(pnn) + else: + self.healthy(pnn) + + def random_iterations(self): + i = 1 + while i <= options.iterations: + verbose_begin("EVENT %d" % i) + verbose_end() + self.do_something_random() + if self.recover() and options.exit: + break + i += 1 + + self.print_statistics() + + def imbalance_for_ips(self, ips): + + imbalance = 0 + + maxnode = -1 + minnode = -1 + + for ip in ips: + for (i, n) in enumerate(self.nodes): + + if not n.healthy or not n.can_node_serve_ip(ip): + continue + + num = n.node_ip_coverage(ips) + + if maxnode == -1 or num > maxnum: + maxnode = i + maxnum = num + + if minnode == -1 or num < minnum: + minnode = i + minnum = num + + if maxnode == -1 or minnode == -1: + continue + + i = maxnum - minnum + #if i < 2: + # i = 0 + imbalance = max([imbalance, i]) + + return imbalance + + + def calculate_imbalance(self): + + # First, do all the assigned IPs. + assigned = sorted([ip + for n in self.nodes + for ip in n.current_addresses]) + + i = self.imbalance_for_ips(assigned) + + ig = [] + # FIXME? If dealing with IP groups, assume the nodes are all + # the same. + for ips in self.nodes[0].ip_groups: + gi = self.imbalance_for_ips(ips) + ig.append(gi) + + return (i, ig) + + + def diff(self): + """Calculate differences in IP assignments between self and prev. + + Gratuitous IP moves (from a healthy node to a healthy node) + are prefixed by !!.""" + + ip_moves = 0 + grat_ip_moves = 0 + details = [] + + for (new, n) in enumerate(self.nodes): + for ip in n.current_addresses: + old = self.prev.find_pnn_with_ip(ip) + if old != new: + ip_moves += 1 + if old != -1 and \ + self.prev.nodes[new].healthy and \ + self.nodes[new].healthy and \ + self.nodes[old].healthy and \ + self.prev.nodes[old].healthy: + prefix = "!!" + grat_ip_moves += 1 + else: + prefix = " " + details.append("%s %s: %d -> %d" % + (prefix, ip, old, new)) + + return (ip_moves, grat_ip_moves, details) + + def find_takeover_node(self, ip): + + pnn = -1 + min = 0 + for (i, n) in enumerate(self.nodes): + if not n.healthy: + continue + + if not n.can_node_serve_ip(ip): + continue + + num = n.node_ip_coverage() + + if (pnn == -1): + pnn = i + min = num + else: + if num < min: + pnn = i + min = num + + if pnn == -1: + verbose_print("Could not find node to take over public address %s" % ip) + return False + + self.nodes[pnn].current_addresses.add(ip) + + verbose_print("%s -> %d" % (ip, pnn)) + return True + + def basic_allocate_unassigned(self): + + assigned = set([ip for n in self.nodes for ip in n.current_addresses]) + unassigned = sorted(list(self.all_public_ips - assigned)) + + for ip in unassigned: + self.find_takeover_node(ip) + + def basic_failback(self, retries_l): + + assigned = sorted([ip + for n in self.nodes + for ip in n.current_addresses]) + for ip in assigned: + + maxnode = -1 + minnode = -1 + for (i, n) in enumerate(self.nodes): + if not n.healthy: + continue + + if not n.can_node_serve_ip(ip): + continue + + num = n.node_ip_coverage() + + if maxnode == -1: + maxnode = i + maxnum = num + else: + if num > maxnum: + maxnode = i + maxnum = num + if minnode == -1: + minnode = i + minnum = num + else: + if num < minnum: + minnode = i + minnum = num + + if maxnode == -1: + print "Could not find maxnode. May not be able to serve ip", ip + continue + + #if self.deterministic_public_ips: + # continue + + if maxnum > minnum + 1 and retries_l[0] < options.retries: + # Remove the 1st ip from maxnode + t = sorted(list(self.nodes[maxnode].current_addresses)) + realloc = t[0] + verbose_print("%s <- %d" % (realloc, maxnode)) + self.nodes[maxnode].current_addresses.remove(realloc) + # Redo the outer loop. + retries_l[0] += 1 + return True + + return False + + + def lcp2_allocate_unassigned(self): + + # Assign as many unassigned addresses as possible. Keep + # selecting the optimal assignment until we don't manage to + # assign anything. + assigned = set([ip for n in self.nodes for ip in n.current_addresses]) + unassigned = sorted(list(self.all_public_ips - assigned)) + + should_loop = True + while len(unassigned) > 0 and should_loop: + should_loop = False + + debug_begin(" CONSIDERING MOVES (UNASSIGNED)") + + minnode = -1 + mindsum = 0 + minip = None + + for ip in unassigned: + for dstnode in range(len(self.nodes)): + if self.nodes[dstnode].can_node_serve_ip(ip) and \ + self.nodes[dstnode].healthy: + dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses) + dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum + debug_print(" %s -> %d [+%d]" % \ + (ip, + dstnode, + dstimbl - self.nodes[dstnode].get_imbalance())) + + if (minnode == -1) or (dstdsum < mindsum): + minnode = dstnode + minimbl = dstimbl + mindsum = dstdsum + minip = ip + should_loop = True + debug_end() + + if minnode != -1: + self.nodes[minnode].current_addresses.add(minip) + self.nodes[minnode].set_imbalance(self.nodes[minnode].get_imbalance() + mindsum) + verbose_print("%s -> %d [+%d]" % (minip, minnode, mindsum)) + unassigned.remove(minip) + + for ip in unassigned: + verbose_print("Could not find node to take over public address %s" % ip) + + def lcp2_failback(self, targets): + + # Get the node with the highest imbalance metric. + srcnode = -1 + maximbl = 0 + for (pnn, n) in enumerate(self.nodes): + b = n.get_imbalance() + if (srcnode == -1) or (b > maximbl): + srcnode = pnn + maximbl = b + + # This means that all nodes had 0 or 1 addresses, so can't + # be imbalanced. + if maximbl == 0: + return False + + # We'll need this a few times... + ips = self.nodes[srcnode].current_addresses + + # Find an IP and destination node that best reduces imbalance. + optimum = None + debug_begin(" CONSIDERING MOVES FROM %d [%d]" % (srcnode, maximbl)) + for ip in ips: + # What is this IP address costing the source node? + srcdsum = ip_distance_2_sum(ip, ips - set([ip])) + srcimbl = maximbl - srcdsum + + # Consider this IP address would cost each potential + # destination node. Destination nodes are limited to + # those that are newly healthy, since we don't want to + # do gratuitous failover of IPs just to make minor + # balance improvements. + for dstnode in targets: + if self.nodes[dstnode].can_node_serve_ip(ip) and \ + self.nodes[dstnode].healthy: + dstdsum = ip_distance_2_sum(ip, self.nodes[dstnode].current_addresses) + dstimbl = self.nodes[dstnode].get_imbalance() + dstdsum + debug_print(" %d [%d] -> %s -> %d [+%d]" % \ + (srcnode, + srcimbl - self.nodes[srcnode].get_imbalance(), + ip, + dstnode, + dstimbl - self.nodes[dstnode].get_imbalance())) + + if (dstimbl < maximbl) and (dstdsum < srcdsum): + if optimum is None: + optimum = (ip, srcnode, srcimbl, dstnode, dstimbl) + else: + (x, sn, si, dn, di) = optimum + if (srcimbl + dstimbl) < (si + di): + optimum = (ip, srcnode, srcimbl, dstnode, dstimbl) + debug_end() + + if optimum is not None: + # We found a move that makes things better... + (ip, srcnode, srcimbl, dstnode, dstimbl) = optimum + ini_srcimbl = self.nodes[srcnode].get_imbalance() + ini_dstimbl = self.nodes[dstnode].get_imbalance() + + self.nodes[srcnode].current_addresses.remove(ip) + self.nodes[srcnode].set_imbalance(srcimbl) + + self.nodes[dstnode].current_addresses.add(ip) + self.nodes[dstnode].set_imbalance(dstimbl) + + verbose_print("%d [%d] -> %s -> %d [+%d]" % \ + (srcnode, + srcimbl - ini_srcimbl, + ip, + dstnode, + dstimbl - ini_dstimbl)) + + return True + + return False + + def ctdb_takeover_run_python(self): + + # Don't bother with the num_healthy stuff. It is an + # irrelevant detail. + + # We just keep the allocate IPs in the current_addresses field + # of the node. This needs to readable, not efficient! + + if self.deterministic_public_ips: + # Remap everything. + addr_list = sorted(list(self.all_public_ips)) + for (i, ip) in enumerate(addr_list): + self.quietly_remove_ip(ip) + # Add addresses to new node. + pnn = i % len(self.nodes) + self.nodes[pnn].current_addresses.add(ip) + verbose_print("%s -> %d" % (ip, pnn)) + + # Remove public addresses from unhealthy nodes. + for (pnn, n) in enumerate(self.nodes): + if not n.healthy: + verbose_print(["%s <- %d" % (ip, pnn) + for ip in n.current_addresses]) + n.current_addresses = set() + + # If a node can't serve an assigned address then remove it. + for n in self.nodes: + verbose_print(["%s <- %d" % (ip, pnn) + for ip in n.current_addresses - n.public_addresses]) + n.current_addresses &= n.public_addresses + + if options.lcp2: + newly_healthy = [pnn for (pnn, n) in enumerate(self.nodes) + if len(n.current_addresses) == 0 and n.healthy] + for n in self.nodes: + n.set_imbalance() + + # We'll only retry the balancing act up to options.retries + # times (for the basic non-deterministic algorithm). This + # nonsense gives us a reference on the retries count in + # Python. It will be easier in C. :-) + # For LCP2 we reassignas many IPs from heavily "loaded" nodes + # to nodes that are newly healthy, looping until we fail to + # reassign an IP. + retries_l = [0] + should_loop = True + while should_loop: + should_loop = False + + if options.lcp2: + self.lcp2_allocate_unassigned() + else: + self.basic_allocate_unassigned() + + if self.no_ip_failback or self.deterministic_public_ips: + break + + if options.lcp2: + if len(newly_healthy) == 0: + break + should_loop = self.lcp2_failback(newly_healthy) + else: + should_loop = self.basic_failback(retries_l) + + def ctdb_takeover_run_external(self): + + # Written while asleep... + + # Convert the cluster state to something that be fed to + # ctdb_takeover_tests ctdb_takeover_run_core ... + + in_lines = [] + for ip in sorted(list(self.all_public_ips)): + allowed = [] + assigned = -1 + for (i, n) in enumerate(self.nodes): + if n.can_node_serve_ip(ip): + allowed.append("%s" % i) + if ip in n.current_addresses: + assigned = i + line = "%s\t%d\t%s" % (ip, assigned, ",".join(allowed)) + in_lines.append(line) + + nodestates = ",".join(["0" if n.healthy else "1" for n in self.nodes]) + + if options.lcp2: + os.environ["CTDB_LCP2"] = "yes" + if options.verbose > 1: + os.environ["CTDB_TEST_LOGLEVEL"] = "4" + elif options.verbose == 1: + os.environ["CTDB_TEST_LOGLEVEL"] = "3" + else: + os.environ["CTDB_TEST_LOGLEVEL"] = "0" + + p = subprocess.Popen("../../bin/ctdb_takeover_tests ctdb_takeover_run_core %s 2>&1" % nodestates, + shell=True, + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.write("\n".join(in_lines)) + p.stdin.close() + + # Flush all of the assigned IPs. + for n in self.nodes: + n.current_addresses = set() + + # Uses the results to populate the current_addresses for each + # node. + for line in p.stdout.read().split("\n"): + # Some lines are debug, some are the final IP + # configuration. Let's use a gross hack that assumes any + # line with 2 words is IP configuration. That will do for + # now. + words = re.split("\s+", line) + if len(words) == 2: + # Add the IP as current for the specified node. + self.nodes[int(words[1])].current_addresses.add(words[0]) + else: + # First 3 words are log date/time, remove them... + print " ".join(words[3:]) + + # Now fake up the LCP calculations. + for n in self.nodes: + n.set_imbalance() + + def ctdb_takeover_run(self): + + self.events += 1 + + if options.external: + return self.ctdb_takeover_run_external() + else: + return self.ctdb_takeover_run_python() + + def recover(self): + verbose_begin("TAKEOVER") + + self.ctdb_takeover_run() + + verbose_end() + + grat_ip_moves = 0 + + if self.prev is not None: + (ip_moves, grat_ip_moves, details) = self.diff() + self.ip_moves.append(ip_moves) + self.grat_ip_moves.append(grat_ip_moves) + + if options.diff: + print_begin("DIFF") + print "\n".join(details) + print_end() + + (imbalance, imbalance_groups) = self.calculate_imbalance() + self.imbalance.append(imbalance) + self.imbalance_groups.append(imbalance_groups) + + if imbalance > options.soft_limit: + self.imbalance_count += 1 + + # There must be a cleaner way... + t = [] + for (c, i) in zip(self.imbalance_groups_count, imbalance_groups): + if i > options.soft_limit: + t.append(c + i) + else: + t.append(c) + self.imbalance_groups_count = t + + imbalance_metric = max([n.get_imbalance() for n in self.nodes]) + self.imbalance_metric.append(imbalance_metric) + if options.balance: + print_begin("IMBALANCE") + print "ALL IPS:", imbalance + if self.have_ip_groups(): + print "IP GROUPS:", imbalance_groups + if options.lcp2: + print "LCP2 IMBALANCE:", imbalance_metric + print_end() + + num_unhealthy = len(self.nodes) - \ + len([n for n in self.nodes if n.healthy]) + self.num_unhealthy.append(num_unhealthy) + + if options.show: + print_begin("STATE") + print self + print_end() + + self.prev = None + self.prev = copy.deepcopy(self) + + # True is bad! + return (grat_ip_moves > 0) or \ + (not self.have_ip_groups() and imbalance > options.hard_limit) or \ + (self.have_ip_groups() and (max(imbalance_groups) > options.hard_limit)) diff --git a/ctdb/tests/takeover/simulation/hey_jude.py b/ctdb/tests/takeover/simulation/hey_jude.py new file mode 100755 index 0000000000..a6b14c5c9b --- /dev/null +++ b/ctdb/tests/takeover/simulation/hey_jude.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses10 = ['10.4.20.%d' % n for n in range(154, 168)] +addresses172a = ['172.20.106.%d' % n for n in range(110, 124)] +addresses172b = ['172.20.107.%d' % n for n in range(110, 117)] + +c = Cluster() + +#for i in range(7): +# c.add_node(Node([addresses10, addresses172])) + + +for i in range(4): + c.add_node(Node([addresses172a, addresses172b])) +for i in range(3): + c.add_node(Node(addresses10)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups1.py b/ctdb/tests/takeover/simulation/ip_groups1.py new file mode 100755 index 0000000000..0808f466cf --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups1.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +# 2 IP groups, both on the same 5 nodes, with each group on different +# interfaces/VLANs. One group has many more addresses to test how +# well an "imbalanced" configuration will balance... + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses20 = ['192.168.20.%d' % n for n in range(1, 13)] +addresses128 = ['192.168.128.%d' % n for n in range(1, 5)] + +c = Cluster() + +for i in range(5): + c.add_node(Node([addresses20, addresses128])) + +#for i in range(3): +# c.add_node(Node([addresses20])) + + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups2.py b/ctdb/tests/takeover/simulation/ip_groups2.py new file mode 100755 index 0000000000..c6c1026646 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups2.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +# 2 groups of addresses, combined into 1 pool so the checking +# algorithm doesn't know about the groups, across 2 nodes. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses20 = ['192.168.20.%d' % n for n in range(1, 13)] +addresses21 = ['192.168.21.%d' % n for n in range(1, 5)] + +c = Cluster() + +for i in range(2): + c.add_node(Node(addresses20 + addresses21)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups3.py b/ctdb/tests/takeover/simulation/ip_groups3.py new file mode 100755 index 0000000000..149946d72b --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups3.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +# 4 IP groups, across 10 nodes, with each group on different +# interfaces/VLANs. 80 addresses in total but not evenly balanced, to +# help check some of the more extreme behaviour. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['192.168.1.%d' % n for n in range(1, 41)] +addresses2 = ['192.168.2.%d' % n for n in range(1, 21)] +addresses3 = ['192.168.3.%d' % n for n in range(1, 11)] +addresses4 = ['192.168.4.%d' % n for n in range(1, 11)] + +# Try detecting imbalance with square root of number of nodes? Or +# just with a parameter indicating how unbalanced you're willing to +# accept... + +c = Cluster() + +for i in range(10): + c.add_node(Node([addresses1, addresses2, addresses3, addresses4])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups4.py b/ctdb/tests/takeover/simulation/ip_groups4.py new file mode 100755 index 0000000000..fdcef7f0a6 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups4.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +# 2 IP groups, across 2 nodes, with each group on different +# interfaces. 4 addresses per group. A nice little canonical 2 node +# configuration. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['192.168.1.%d' % n for n in range(1, 5)] +addresses2 = ['192.168.2.%d' % n for n in range(1, 5)] + +# Try detecting imbalance with square root of number of nodes? Or +# just with a parameter indicating how unbalanced you're willing to +# accept... + +c = Cluster() + +for i in range(2): + c.add_node(Node([addresses1, addresses2])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/ip_groups5.py b/ctdb/tests/takeover/simulation/ip_groups5.py new file mode 100755 index 0000000000..8c46150638 --- /dev/null +++ b/ctdb/tests/takeover/simulation/ip_groups5.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +# 1 IP group, to test backward compatibility of LCP2 algorithm. 16 +# addresses across 4 nodes. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['192.168.1.%d' % n for n in range(1, 17)] + +# Try detecting imbalance with square root of number of nodes? Or +# just with a parameter indicating how unbalanced you're willing to +# accept... + +c = Cluster() + +for i in range(4): + c.add_node(Node(addresses1)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/mgmt_simple.py b/ctdb/tests/takeover/simulation/mgmt_simple.py new file mode 100755 index 0000000000..f891199655 --- /dev/null +++ b/ctdb/tests/takeover/simulation/mgmt_simple.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +# This is an example showing a current SONAS configuration with 3 +# interface node and a management node. When run with deterministic +# IPs there are gratuitous IP reassignments. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] + +c = Cluster() + +for i in range(3): + c.add_node(Node(addresses)) + +c.add_node(Node([])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/node_group.py b/ctdb/tests/takeover/simulation/node_group.py new file mode 100755 index 0000000000..bf7de58aa9 --- /dev/null +++ b/ctdb/tests/takeover/simulation/node_group.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# This demonstrates a node group configurations. +# +# Node groups can be defined with the syntax "-g N@IP0,IP1-IP2,IP3". +# This says to create a group of N nodes with IPs IP0, IP1, ..., IP2, +# IP3. Run it with deterministic IPs causes lots of gratuitous IP +# reassignments. Running with --nd fixes this. + +import ctdb_takeover +import sys +from optparse import make_option +import string + +ctdb_takeover.process_args([ + make_option("-g", "--group", + action="append", type="string", dest="groups", + help="define a node group using N@IPs syntax"), + ]) + +def expand_range(r): + sr = r.split("-", 1) + if len(sr) == 2: + all = string.ascii_uppercase + string.ascii_lowercase + sr = list(all[all.index(sr[0]):all.index(sr[1])+1]) + return sr + +def add_node_group(s): + (count, ips_str) = s.split("@", 1) + ips = [i for r in ips_str.split(",") \ + for i in expand_range(r) if r != ""] + for i in range(int(count)): + c.add_node(ctdb_takeover.Node(ips)) + +c = ctdb_takeover.Cluster() + +if ctdb_takeover.options.groups is None: + print "Error: no node groups defined." + sys.exit(1) + +for g in ctdb_takeover.options.groups: + add_node_group(g) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/node_group_extra.py b/ctdb/tests/takeover/simulation/node_group_extra.py new file mode 100755 index 0000000000..7e9e518bdd --- /dev/null +++ b/ctdb/tests/takeover/simulation/node_group_extra.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# This example demonstrates a node group configuration. Is it meant +# to be the same as node_group_simple.py, but with a couple of nodes +# added later, so they are listed after the management node. + +# When run with deterministic IPs (use "-d" to show the problem) it +# does many gratuitous IP reassignments. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] + ['P', 'Q', 'R', 'S', 'T', 'U'] +addresses2 = ['I', 'J', 'K', 'L'] + +c = Cluster() + +for i in range(4): + c.add_node(Node(addresses1)) + +for i in range(3): + c.add_node(Node(addresses2)) + +c.add_node(Node([])) +c.add_node(Node(addresses1)) +c.add_node(Node(addresses2)) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/node_group_simple.py b/ctdb/tests/takeover/simulation/node_group_simple.py new file mode 100755 index 0000000000..3c58ef7314 --- /dev/null +++ b/ctdb/tests/takeover/simulation/node_group_simple.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# This example demonstrates a simple, sensible node group +# configuration. When run with deterministic IPs (use "-d" to show +# the problem) it does many gratuitous IP reassignments. + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] +addresses2 = ['I', 'J', 'K'] + +c = Cluster() + +for i in range(4): + c.add_node(Node(addresses1)) + +for i in range(3): + c.add_node(Node(addresses2)) + +c.add_node(Node([])) + +c.recover() + +c.random_iterations() diff --git a/ctdb/tests/takeover/simulation/nondet_path_01.py b/ctdb/tests/takeover/simulation/nondet_path_01.py new file mode 100755 index 0000000000..a62847a216 --- /dev/null +++ b/ctdb/tests/takeover/simulation/nondet_path_01.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +# This is a contrived example that makes the balancing algorithm fail +# for nondeterministic IPs (run with "-dv --nd" to see the failure). + +from ctdb_takeover import Cluster, Node, process_args + +process_args() + +addresses1 = ['A', 'B', 'C', 'D'] +addresses2 = ['B', 'E', 'F'] + +c = Cluster() + +for i in range(2): + c.add_node(Node(addresses1)) + +c.add_node(Node(addresses2)) + +c.recover() + +c.unhealthy(1) +c.recover() +c.healthy(1) +c.recover() diff --git a/ctdb/tests/takeover/testcases/lcp2.001.sh b/ctdb/tests/takeover/testcases/lcp2.001.sh deleted file mode 100755 index f792f5989f..0000000000 --- a/ctdb/tests/takeover/testcases/lcp2.001.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -. "${TAKEOVER_TESTS_DIR}/common.sh" - -define_test "3 nodes, 3 -> 1 healthy" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < 2 healthy" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < all healthy" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < all healthy, info logging" - -export CTDB_TEST_LOGLEVEL=3 - -required_result < 192.168.20.253 -> 0 [+0] -DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] -DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] -DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] -DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] -192.168.21.254 2 -192.168.21.253 0 -192.168.21.252 1 -192.168.20.254 2 -192.168.20.253 0 -192.168.20.252 1 -192.168.20.251 2 -192.168.20.250 0 -192.168.20.249 1 -EOF - -simple_test 0,0,0 < all healthy, debug logging" - -export CTDB_TEST_LOGLEVEL=4 - -required_result < 192.168.21.254 -> 0 [+0] -DATE TIME [PID]: 1 [-116718] -> 192.168.21.254 -> 2 [+0] -DATE TIME [PID]: 1 [-116971] -> 192.168.21.253 -> 0 [+0] -DATE TIME [PID]: 1 [-116971] -> 192.168.21.253 -> 2 [+0] -DATE TIME [PID]: 1 [-116971] -> 192.168.21.252 -> 0 [+0] -DATE TIME [PID]: 1 [-116971] -> 192.168.21.252 -> 2 [+0] -DATE TIME [PID]: 1 [-121110] -> 192.168.20.254 -> 0 [+0] -DATE TIME [PID]: 1 [-121110] -> 192.168.20.254 -> 2 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 2 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.252 -> 0 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.252 -> 2 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.251 -> 0 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.251 -> 2 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.250 -> 0 [+0] -DATE TIME [PID]: 1 [-121363] -> 192.168.20.250 -> 2 [+0] -DATE TIME [PID]: 1 [-121110] -> 192.168.20.249 -> 0 [+0] -DATE TIME [PID]: 1 [-121110] -> 192.168.20.249 -> 2 [+0] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: 1 [-121363] -> 192.168.20.253 -> 0 [+0] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 1 [418056] -DATE TIME [PID]: 1 [-102557] -> 192.168.21.254 -> 0 [+14161] -DATE TIME [PID]: 1 [-102557] -> 192.168.21.254 -> 2 [+0] -DATE TIME [PID]: 1 [-102810] -> 192.168.21.253 -> 0 [+14161] -DATE TIME [PID]: 1 [-102810] -> 192.168.21.253 -> 2 [+0] -DATE TIME [PID]: 1 [-102810] -> 192.168.21.252 -> 0 [+14161] -DATE TIME [PID]: 1 [-102810] -> 192.168.21.252 -> 2 [+0] -DATE TIME [PID]: 1 [-105234] -> 192.168.20.254 -> 0 [+15876] -DATE TIME [PID]: 1 [-105234] -> 192.168.20.254 -> 2 [+0] -DATE TIME [PID]: 1 [-105234] -> 192.168.20.252 -> 0 [+16129] -DATE TIME [PID]: 1 [-105234] -> 192.168.20.252 -> 2 [+0] -DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 0 [+15625] -DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] -DATE TIME [PID]: 1 [-105738] -> 192.168.20.250 -> 0 [+15625] -DATE TIME [PID]: 1 [-105738] -> 192.168.20.250 -> 2 [+0] -DATE TIME [PID]: 1 [-105485] -> 192.168.20.249 -> 0 [+15625] -DATE TIME [PID]: 1 [-105485] -> 192.168.20.249 -> 2 [+0] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: 1 [-105738] -> 192.168.20.251 -> 2 [+0] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 1 [312571] -DATE TIME [PID]: 1 [-88396] -> 192.168.21.254 -> 0 [+14161] -DATE TIME [PID]: 1 [-88396] -> 192.168.21.254 -> 2 [+14161] -DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] -DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 2 [+14161] -DATE TIME [PID]: 1 [-88649] -> 192.168.21.252 -> 0 [+14161] -DATE TIME [PID]: 1 [-88649] -> 192.168.21.252 -> 2 [+14161] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.254 -> 0 [+15876] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.254 -> 2 [+15625] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.252 -> 0 [+16129] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.252 -> 2 [+15625] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.250 -> 0 [+15625] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.250 -> 2 [+16129] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.249 -> 0 [+15625] -DATE TIME [PID]: 1 [-89609] -> 192.168.20.249 -> 2 [+15876] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: 1 [-88649] -> 192.168.21.253 -> 0 [+14161] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 1 [222962] -DATE TIME [PID]: 1 [-72520] -> 192.168.21.254 -> 0 [+30037] -DATE TIME [PID]: 1 [-72520] -> 192.168.21.254 -> 2 [+14161] -DATE TIME [PID]: 1 [-72520] -> 192.168.21.252 -> 0 [+30290] -DATE TIME [PID]: 1 [-72520] -> 192.168.21.252 -> 2 [+14161] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 0 [+30037] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.252 -> 0 [+30290] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.252 -> 2 [+15625] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.250 -> 0 [+29786] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.250 -> 2 [+16129] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.249 -> 0 [+29786] -DATE TIME [PID]: 1 [-75448] -> 192.168.20.249 -> 2 [+15876] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: 1 [-75448] -> 192.168.20.254 -> 2 [+15625] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 1 [147514] -DATE TIME [PID]: 1 [-58359] -> 192.168.21.254 -> 0 [+30037] -DATE TIME [PID]: 1 [-58359] -> 192.168.21.254 -> 2 [+28322] -DATE TIME [PID]: 1 [-58359] -> 192.168.21.252 -> 0 [+30290] -DATE TIME [PID]: 1 [-58359] -> 192.168.21.252 -> 2 [+28322] -DATE TIME [PID]: 1 [-59572] -> 192.168.20.252 -> 0 [+30290] -DATE TIME [PID]: 1 [-59572] -> 192.168.20.252 -> 2 [+31501] -DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] -DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 2 [+31754] -DATE TIME [PID]: 1 [-59823] -> 192.168.20.249 -> 0 [+29786] -DATE TIME [PID]: 1 [-59823] -> 192.168.20.249 -> 2 [+31501] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: 1 [-59823] -> 192.168.20.250 -> 0 [+29786] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 1 [87691] -DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 0 [+44198] -DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] -DATE TIME [PID]: 1 [-44198] -> 192.168.21.252 -> 0 [+44451] -DATE TIME [PID]: 1 [-44198] -> 192.168.21.252 -> 2 [+28322] -DATE TIME [PID]: 1 [-43947] -> 192.168.20.252 -> 0 [+45915] -DATE TIME [PID]: 1 [-43947] -> 192.168.20.252 -> 2 [+31501] -DATE TIME [PID]: 1 [-43947] -> 192.168.20.249 -> 0 [+45662] -DATE TIME [PID]: 1 [-43947] -> 192.168.20.249 -> 2 [+31501] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: 1 [-44198] -> 192.168.21.254 -> 2 [+28322] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES (UNASSIGNED) -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 0 [43947] -DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 0 [+28322] -DATE TIME [PID]: 0 [-28322] -> 192.168.21.253 -> 2 [+44198] -DATE TIME [PID]: 0 [-29786] -> 192.168.20.253 -> 0 [+29786] -DATE TIME [PID]: 0 [-29786] -> 192.168.20.253 -> 2 [+45662] -DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 0 [+29786] -DATE TIME [PID]: 0 [-29786] -> 192.168.20.250 -> 2 [+45915] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 2 [43947] -DATE TIME [PID]: 2 [-28322] -> 192.168.21.254 -> 0 [+44198] -DATE TIME [PID]: 2 [-28322] -> 192.168.21.254 -> 2 [+28322] -DATE TIME [PID]: 2 [-29786] -> 192.168.20.254 -> 0 [+45662] -DATE TIME [PID]: 2 [-29786] -> 192.168.20.254 -> 2 [+29786] -DATE TIME [PID]: 2 [-29786] -> 192.168.20.251 -> 0 [+45915] -DATE TIME [PID]: 2 [-29786] -> 192.168.20.251 -> 2 [+29786] -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: ---------------------------------------- -DATE TIME [PID]: CONSIDERING MOVES FROM 1 [43744] -DATE TIME [PID]: 1 [-28322] -> 192.168.21.252 -> 0 [+44451] -DATE TIME [PID]: 1 [-28322] -> 192.168.21.252 -> 2 [+44198] -DATE TIME [PID]: 1 [-29786] -> 192.168.20.252 -> 0 [+45915] -DATE TIME [PID]: 1 [-29786] -> 192.168.20.252 -> 2 [+45662] -DATE TIME [PID]: 1 [-29786] -> 192.168.20.249 -> 0 [+45662] -DATE TIME [PID]: 1 [-29786] -> 192.168.20.249 -> 2 [+45662] -DATE TIME [PID]: ---------------------------------------- -192.168.21.254 2 -192.168.21.253 0 -192.168.21.252 1 -192.168.20.254 2 -192.168.20.253 0 -192.168.20.252 1 -192.168.20.251 2 -192.168.20.250 0 -192.168.20.249 1 -EOF - -simple_test 0,0,0 < 1 healthy" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < 2 healthy" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < all healthy" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < all disconnected" - -export CTDB_TEST_LOGLEVEL=0 - -required_result < all healthy" - -required_result < Date: Mon, 16 Apr 2012 15:58:44 +1000 Subject: tests - run_tests ignores trailing '/' on directories Signed-off-by: Martin Schwenke (This used to be ctdb commit dbec696930327ff07b39282e3084eef4ded064c0) --- ctdb/tests/scripts/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index c2ff15a69b..8fce3ae978 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -148,7 +148,7 @@ run_one_test () for f ; do if [ -d "$f" ] ; then - for i in $(ls "$f/"*".sh" 2>/dev/null) ; do + for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do run_one_test "$i" done elif [ -f "$f" ] ; then -- cgit From df39a671db2a88de7e6b1e9a9ce4885f2b9bb393 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 16:24:12 +1000 Subject: tests/eventscripts - Restructure according to new convention Signed-off-by: Martin Schwenke (This used to be ctdb commit eb13507713ba6732271b7c3024bfddbda6da5ffc) --- ctdb/tests/eventscripts/10.interface.init.001.sh | 13 + ctdb/tests/eventscripts/10.interface.init.002.sh | 11 + .../tests/eventscripts/10.interface.monitor.001.sh | 13 + .../tests/eventscripts/10.interface.monitor.002.sh | 11 + .../tests/eventscripts/10.interface.monitor.003.sh | 15 + .../tests/eventscripts/10.interface.monitor.004.sh | 15 + .../tests/eventscripts/10.interface.monitor.005.sh | 15 + .../tests/eventscripts/10.interface.monitor.006.sh | 15 + .../tests/eventscripts/10.interface.monitor.007.sh | 13 + .../tests/eventscripts/10.interface.monitor.008.sh | 15 + .../tests/eventscripts/10.interface.monitor.009.sh | 17 + .../tests/eventscripts/10.interface.monitor.010.sh | 19 + .../tests/eventscripts/10.interface.monitor.011.sh | 19 + .../tests/eventscripts/10.interface.monitor.012.sh | 23 + .../tests/eventscripts/10.interface.monitor.013.sh | 15 + .../tests/eventscripts/10.interface.monitor.014.sh | 16 + .../tests/eventscripts/10.interface.monitor.015.sh | 22 + ctdb/tests/eventscripts/10.interface.multi.001.sh | 14 + .../eventscripts/10.interface.releaseip.001.sh | 13 + .../eventscripts/10.interface.releaseip.002.sh | 17 + .../tests/eventscripts/10.interface.startup.001.sh | 13 + .../tests/eventscripts/10.interface.startup.002.sh | 11 + ctdb/tests/eventscripts/10.interface.takeip.001.sh | 13 + ctdb/tests/eventscripts/10.interface.takeip.002.sh | 13 + ctdb/tests/eventscripts/10.interface.takeip.003.sh | 25 + ctdb/tests/eventscripts/13.per_ip_routing.001.sh | 16 + ctdb/tests/eventscripts/13.per_ip_routing.002.sh | 14 + ctdb/tests/eventscripts/13.per_ip_routing.003.sh | 24 + ctdb/tests/eventscripts/13.per_ip_routing.004.sh | 25 + ctdb/tests/eventscripts/13.per_ip_routing.005.sh | 38 + ctdb/tests/eventscripts/13.per_ip_routing.006.sh | 38 + ctdb/tests/eventscripts/13.per_ip_routing.007.sh | 40 + ctdb/tests/eventscripts/13.per_ip_routing.008.sh | 42 + ctdb/tests/eventscripts/13.per_ip_routing.009.sh | 44 + ctdb/tests/eventscripts/13.per_ip_routing.010.sh | 52 ++ ctdb/tests/eventscripts/13.per_ip_routing.011.sh | 41 + ctdb/tests/eventscripts/13.per_ip_routing.012.sh | 48 ++ ctdb/tests/eventscripts/40.vsftpd.monitor.001.sh | 11 + ctdb/tests/eventscripts/41.httpd.monitor.001.sh | 11 + ctdb/tests/eventscripts/50.samba.monitor.001.sh | 11 + ctdb/tests/eventscripts/50.samba.monitor.050.sh | 17 + ctdb/tests/eventscripts/50.samba.monitor.051.sh | 19 + ctdb/tests/eventscripts/50.samba.monitor.101.sh | 11 + ctdb/tests/eventscripts/50.samba.monitor.102.sh | 12 + ctdb/tests/eventscripts/50.samba.monitor.103.sh | 12 + ctdb/tests/eventscripts/50.samba.monitor.104.sh | 12 + ctdb/tests/eventscripts/50.samba.monitor.105.sh | 12 + ctdb/tests/eventscripts/50.samba.monitor.106.sh | 14 + ctdb/tests/eventscripts/50.samba.monitor.107.sh | 18 + ctdb/tests/eventscripts/50.samba.monitor.108.sh | 15 + ctdb/tests/eventscripts/50.samba.monitor.109.sh | 30 + ctdb/tests/eventscripts/50.samba.monitor.110.sh | 33 + ctdb/tests/eventscripts/50.samba.monitor.111.sh | 20 + ctdb/tests/eventscripts/60.nfs.monitor.001.sh | 11 + ctdb/tests/eventscripts/60.nfs.monitor.100.sh | 19 + ctdb/tests/eventscripts/60.nfs.monitor.101.sh | 11 + ctdb/tests/eventscripts/60.nfs.monitor.111.sh | 12 + ctdb/tests/eventscripts/60.nfs.monitor.112.sh | 15 + ctdb/tests/eventscripts/60.nfs.monitor.121.sh | 17 + ctdb/tests/eventscripts/60.nfs.monitor.122.sh | 19 + ctdb/tests/eventscripts/60.nfs.monitor.131.sh | 10 + ctdb/tests/eventscripts/60.nfs.monitor.132.sh | 15 + ctdb/tests/eventscripts/60.nfs.monitor.141.sh | 15 + ctdb/tests/eventscripts/60.nfs.monitor.142.sh | 14 + ctdb/tests/eventscripts/60.nfs.monitor.151.sh | 12 + ctdb/tests/eventscripts/60.nfs.monitor.152.sh | 18 + ctdb/tests/eventscripts/60.nfs.monitor.153.sh | 15 + ctdb/tests/eventscripts/60.nfs.monitor.161.sh | 13 + ctdb/tests/eventscripts/60.nfs.monitor.162.sh | 14 + ctdb/tests/eventscripts/60.nfs.multi.001.sh | 21 + ctdb/tests/eventscripts/60.nfs.multi.002.sh | 25 + ctdb/tests/eventscripts/60.nfs.multi.003.sh | 27 + ctdb/tests/eventscripts/60.nfs.multi.004.sh | 27 + ctdb/tests/eventscripts/60.nfs.multi.005.sh | 27 + ctdb/tests/eventscripts/60.nfs.multi.006.sh | 21 + ctdb/tests/eventscripts/common.sh | 938 --------------------- .../eventscripts/multievent/10.interface.001.sh | 14 - .../multievent/13.per_ip_routing.001.sh | 16 - .../multievent/13.per_ip_routing.002.sh | 14 - .../multievent/13.per_ip_routing.003.sh | 24 - .../multievent/13.per_ip_routing.004.sh | 25 - .../multievent/13.per_ip_routing.005.sh | 38 - .../multievent/13.per_ip_routing.006.sh | 38 - .../multievent/13.per_ip_routing.007.sh | 40 - .../multievent/13.per_ip_routing.008.sh | 42 - .../multievent/13.per_ip_routing.009.sh | 44 - .../multievent/13.per_ip_routing.010.sh | 52 -- .../multievent/13.per_ip_routing.011.sh | 41 - .../multievent/13.per_ip_routing.012.sh | 48 -- ctdb/tests/eventscripts/multievent/60.nfs.001.sh | 21 - ctdb/tests/eventscripts/multievent/60.nfs.002.sh | 25 - ctdb/tests/eventscripts/multievent/60.nfs.003.sh | 27 - ctdb/tests/eventscripts/multievent/60.nfs.004.sh | 27 - ctdb/tests/eventscripts/multievent/60.nfs.005.sh | 27 - ctdb/tests/eventscripts/multievent/60.nfs.006.sh | 21 - ctdb/tests/eventscripts/scripts/local.sh | 914 ++++++++++++++++++++ .../eventscripts/simple/10.interface.init.001.sh | 13 - .../eventscripts/simple/10.interface.init.002.sh | 11 - .../simple/10.interface.monitor.001.sh | 13 - .../simple/10.interface.monitor.002.sh | 11 - .../simple/10.interface.monitor.003.sh | 15 - .../simple/10.interface.monitor.004.sh | 15 - .../simple/10.interface.monitor.005.sh | 15 - .../simple/10.interface.monitor.006.sh | 15 - .../simple/10.interface.monitor.007.sh | 13 - .../simple/10.interface.monitor.008.sh | 15 - .../simple/10.interface.monitor.009.sh | 17 - .../simple/10.interface.monitor.010.sh | 19 - .../simple/10.interface.monitor.011.sh | 19 - .../simple/10.interface.monitor.012.sh | 23 - .../simple/10.interface.monitor.013.sh | 15 - .../simple/10.interface.monitor.014.sh | 16 - .../simple/10.interface.monitor.015.sh | 22 - .../simple/10.interface.releaseip.001.sh | 13 - .../simple/10.interface.releaseip.002.sh | 17 - .../simple/10.interface.startup.001.sh | 13 - .../simple/10.interface.startup.002.sh | 11 - .../eventscripts/simple/10.interface.takeip.001.sh | 13 - .../eventscripts/simple/10.interface.takeip.002.sh | 13 - .../eventscripts/simple/10.interface.takeip.003.sh | 25 - .../eventscripts/simple/40.vsftpd.monitor.001.sh | 11 - .../eventscripts/simple/41.httpd.monitor.001.sh | 11 - .../eventscripts/simple/50.samba.monitor.001.sh | 11 - .../eventscripts/simple/50.samba.monitor.050.sh | 17 - .../eventscripts/simple/50.samba.monitor.051.sh | 19 - .../eventscripts/simple/50.samba.monitor.101.sh | 11 - .../eventscripts/simple/50.samba.monitor.102.sh | 12 - .../eventscripts/simple/50.samba.monitor.103.sh | 12 - .../eventscripts/simple/50.samba.monitor.104.sh | 12 - .../eventscripts/simple/50.samba.monitor.105.sh | 12 - .../eventscripts/simple/50.samba.monitor.106.sh | 14 - .../eventscripts/simple/50.samba.monitor.107.sh | 18 - .../eventscripts/simple/50.samba.monitor.108.sh | 15 - .../eventscripts/simple/50.samba.monitor.109.sh | 30 - .../eventscripts/simple/50.samba.monitor.110.sh | 33 - .../eventscripts/simple/50.samba.monitor.111.sh | 20 - .../eventscripts/simple/60.nfs.monitor.001.sh | 11 - .../eventscripts/simple/60.nfs.monitor.100.sh | 19 - .../eventscripts/simple/60.nfs.monitor.101.sh | 11 - .../eventscripts/simple/60.nfs.monitor.111.sh | 12 - .../eventscripts/simple/60.nfs.monitor.112.sh | 15 - .../eventscripts/simple/60.nfs.monitor.121.sh | 17 - .../eventscripts/simple/60.nfs.monitor.122.sh | 19 - .../eventscripts/simple/60.nfs.monitor.131.sh | 10 - .../eventscripts/simple/60.nfs.monitor.132.sh | 15 - .../eventscripts/simple/60.nfs.monitor.141.sh | 15 - .../eventscripts/simple/60.nfs.monitor.142.sh | 14 - .../eventscripts/simple/60.nfs.monitor.151.sh | 12 - .../eventscripts/simple/60.nfs.monitor.152.sh | 18 - .../eventscripts/simple/60.nfs.monitor.153.sh | 15 - .../eventscripts/simple/60.nfs.monitor.161.sh | 13 - .../eventscripts/simple/60.nfs.monitor.162.sh | 14 - ctdb/tests/eventscripts/stubs/ctdb | 2 +- 153 files changed, 2359 insertions(+), 2383 deletions(-) create mode 100755 ctdb/tests/eventscripts/10.interface.init.001.sh create mode 100755 ctdb/tests/eventscripts/10.interface.init.002.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.001.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.002.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.003.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.004.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.005.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.006.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.007.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.008.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.009.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.010.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.011.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.012.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.013.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.014.sh create mode 100755 ctdb/tests/eventscripts/10.interface.monitor.015.sh create mode 100755 ctdb/tests/eventscripts/10.interface.multi.001.sh create mode 100755 ctdb/tests/eventscripts/10.interface.releaseip.001.sh create mode 100755 ctdb/tests/eventscripts/10.interface.releaseip.002.sh create mode 100755 ctdb/tests/eventscripts/10.interface.startup.001.sh create mode 100755 ctdb/tests/eventscripts/10.interface.startup.002.sh create mode 100755 ctdb/tests/eventscripts/10.interface.takeip.001.sh create mode 100755 ctdb/tests/eventscripts/10.interface.takeip.002.sh create mode 100755 ctdb/tests/eventscripts/10.interface.takeip.003.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.001.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.002.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.003.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.004.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.005.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.006.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.007.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.008.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.009.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.010.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.011.sh create mode 100755 ctdb/tests/eventscripts/13.per_ip_routing.012.sh create mode 100755 ctdb/tests/eventscripts/40.vsftpd.monitor.001.sh create mode 100755 ctdb/tests/eventscripts/41.httpd.monitor.001.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.001.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.050.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.051.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.101.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.102.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.103.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.104.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.105.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.106.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.107.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.108.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.109.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.110.sh create mode 100755 ctdb/tests/eventscripts/50.samba.monitor.111.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.001.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.100.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.101.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.111.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.112.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.121.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.122.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.131.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.132.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.141.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.142.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.151.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.152.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.153.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.161.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.monitor.162.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.multi.001.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.multi.002.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.multi.003.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.multi.004.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.multi.005.sh create mode 100755 ctdb/tests/eventscripts/60.nfs.multi.006.sh delete mode 100644 ctdb/tests/eventscripts/common.sh delete mode 100755 ctdb/tests/eventscripts/multievent/10.interface.001.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.002.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.003.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.004.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.005.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.006.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.007.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.008.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.009.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.010.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.011.sh delete mode 100755 ctdb/tests/eventscripts/multievent/13.per_ip_routing.012.sh delete mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.001.sh delete mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.002.sh delete mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.003.sh delete mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.004.sh delete mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.005.sh delete mode 100755 ctdb/tests/eventscripts/multievent/60.nfs.006.sh create mode 100644 ctdb/tests/eventscripts/scripts/local.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.init.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.init.002.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.002.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.003.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.004.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.005.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.006.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.007.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.008.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.009.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.010.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.011.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.012.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.013.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.014.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.releaseip.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.releaseip.002.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.startup.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.startup.002.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.takeip.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.takeip.002.sh delete mode 100755 ctdb/tests/eventscripts/simple/10.interface.takeip.003.sh delete mode 100755 ctdb/tests/eventscripts/simple/40.vsftpd.monitor.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/41.httpd.monitor.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.050.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.051.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.101.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.102.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.103.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.104.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.105.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.106.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.107.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.108.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.109.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.110.sh delete mode 100755 ctdb/tests/eventscripts/simple/50.samba.monitor.111.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.001.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.100.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.101.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.111.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.112.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.121.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.122.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.131.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.132.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.141.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.142.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.151.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.152.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.153.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh delete mode 100755 ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh diff --git a/ctdb/tests/eventscripts/10.interface.init.001.sh b/ctdb/tests/eventscripts/10.interface.init.001.sh new file mode 100755 index 0000000000..fae1a78d50 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.init.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "no public addresses" + +setup_ctdb + +export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" + +ok "No public addresses file found. Nothing to do for 10.interfaces" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.init.002.sh b/ctdb/tests/eventscripts/10.interface.init.002.sh new file mode 100755 index 0000000000..ba33f927c3 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.init.002.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up" + +setup_ctdb + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.001.sh b/ctdb/tests/eventscripts/10.interface.monitor.001.sh new file mode 100755 index 0000000000..42ef42d81b --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.001.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "no public addresses" + +setup_ctdb + +export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.002.sh b/ctdb/tests/eventscripts/10.interface.monitor.002.sh new file mode 100755 index 0000000000..ba33f927c3 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.002.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up" + +setup_ctdb + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.003.sh b/ctdb/tests/eventscripts/10.interface.monitor.003.sh new file mode 100755 index 0000000000..1eb7916b93 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.003.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 interface down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +ethtool_interfaces_down $iface + +required_result 1 "ERROR: No link on the public network interface $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.004.sh b/ctdb/tests/eventscripts/10.interface.monitor.004.sh new file mode 100755 index 0000000000..69ffbd00cd --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.004.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all interfaces up, 1 is a bond" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.005.sh b/ctdb/tests/eventscripts/10.interface.monitor.005.sh new file mode 100755 index 0000000000..8cf7bbc9c0 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.005.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 bond, no active slaves" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "None" + +required_result 1 "ERROR: No active slaves for bond device $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.006.sh b/ctdb/tests/eventscripts/10.interface.monitor.006.sh new file mode 100755 index 0000000000..3c483a3516 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.006.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 bond, active slaves, link down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "" "down" + +required_result 1 "ERROR: public network interface $iface is down" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.007.sh b/ctdb/tests/eventscripts/10.interface.monitor.007.sh new file mode 100755 index 0000000000..c45900e3b2 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.007.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "unknown interface, up" + +setup_ctdb + +export CTDB_PUBLIC_INTERFACE="dev999" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.008.sh b/ctdb/tests/eventscripts/10.interface.monitor.008.sh new file mode 100755 index 0000000000..f73302b30d --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.008.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "unknown interface, down, up" + +setup_ctdb + +iface="dev999" +export CTDB_PUBLIC_INTERFACE="$iface" + +#EVENTSCRIPTS_TESTS_TRACE="sh -x" +iterate_test 3 "ok_null" \ + 1 'ethtool_interfaces_down "$iface" ; required_result 1 "ERROR: No link on the public network interface $iface"' \ + 2 'ethtool_interfaces_up "$iface"' diff --git a/ctdb/tests/eventscripts/10.interface.monitor.009.sh b/ctdb/tests/eventscripts/10.interface.monitor.009.sh new file mode 100755 index 0000000000..1b785ffdc8 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.009.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down "$iface" + +ok "ERROR: No link on the public network interface $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.010.sh b/ctdb/tests/eventscripts/10.interface.monitor.010.sh new file mode 100755 index 0000000000..4d233193fb --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.010.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, all down" + +setup_ctdb + +ifaces=$(ctdb_get_interfaces) + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down $ifaces + +msg=$(for i in $ifaces ; do echo "ERROR: No link on the public network interface $i" ; done) + +required_result 1 "$msg" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.011.sh b/ctdb/tests/eventscripts/10.interface.monitor.011.sh new file mode 100755 index 0000000000..21775d41ce --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.011.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "None" + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down "$iface" + +ok "ERROR: No active slaves for bond device $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.012.sh b/ctdb/tests/eventscripts/10.interface.monitor.012.sh new file mode 100755 index 0000000000..dbe84b7729 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.012.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down" + +setup_ctdb + +ifaces=$(ctdb_get_interfaces) + +for i in $ifaces ; do + setup_bond $i "None" +done + +export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" + +ethtool_interfaces_down $ifaces + +msg=$(for i in $ifaces ; do echo "ERROR: No active slaves for bond device $i" ; done) + +required_result 1 "$msg" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.013.sh b/ctdb/tests/eventscripts/10.interface.monitor.013.sh new file mode 100755 index 0000000000..0fcdcd8d15 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.013.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "1 bond, active slaves, link down" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +setup_bond $iface "" "up" "down" + +required_result 1 "ERROR: No active slaves for 802.ad bond device $iface" + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.014.sh b/ctdb/tests/eventscripts/10.interface.monitor.014.sh new file mode 100755 index 0000000000..ab23d307c2 --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.014.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "spurious addresses on interface, no action" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +ip addr add 192.168.253.253/24 dev $iface +ip addr add 192.168.254.254/24 dev $iface + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/10.interface.monitor.015.sh b/ctdb/tests/eventscripts/10.interface.monitor.015.sh new file mode 100755 index 0000000000..ff54b69d9b --- /dev/null +++ b/ctdb/tests/eventscripts/10.interface.monitor.015.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "spurious addresses on interface, delete them" + +setup_ctdb + +iface=$(ctdb_get_1_interface) + +ip addr add 192.168.253.253/24 dev $iface +ip addr add 192.168.254.254/24 dev $iface + +export CTDB_DELETE_UNEXPECTED_IPS="yes" + +ok <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" + +ctdb_get_1_public_address | +{ + read dev ip bits + + net=$(ipv4_host_addr_to_net "$ip" "$bits") + gw="${net%.*}.1" # a dumb, calculated default + + ok_null + + simple_test_event "takeip" $dev $ip $bits + + ok <"$CTDB_PER_IP_ROUTING_CONF" + +# Now do a takeip for each IP on the "current" node +ctdb_get_my_public_addresses | +{ + policy_rules="" + policy_routes="" + while read dev ip bits ; do + + net=$(ipv4_host_addr_to_net "$ip" "$bits") + gw="${net%.*}.1" # a dumb, calculated default + + ok_null + + simple_test_event "takeip" $dev $ip $bits + + policy_rules="${policy_rules} +${CTDB_PER_IP_ROUTING_RULE_PREF}: from $ip lookup ctdb.$ip " + policy_routes="${policy_routes} +# ip route show table ctdb.$ip +$net dev $dev scope link +default via $gw dev $dev " + done + + ok <"$CTDB_PER_IP_ROUTING_CONF" <"$rpc_fail_limits_file" + +services="knfsd|mountd|rquotad|lockd|statd" + +echo "Doing rough check of file format..." + +! grep -v -E "^(${services}) " "$rpc_fail_limits_file" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.101.sh b/ctdb/tests/eventscripts/60.nfs.monitor.101.sh new file mode 100755 index 0000000000..1a68927810 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.101.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "all services available" + +setup_nfs + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.111.sh b/ctdb/tests/eventscripts/60.nfs.monitor.111.sh new file mode 100755 index 0000000000..414fcc8066 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.111.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "knfsd down, 1 iteration" + +setup_nfs +rpc_services_down "nfs" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.112.sh b/ctdb/tests/eventscripts/60.nfs.monitor.112.sh new file mode 100755 index 0000000000..666a38aaaf --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.112.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "knfsd down, 6 iterations" + +# knfsd fails and attempts to restart it fail. + +setup_nfs +rpc_services_down "nfs" + +iterate_test 6 'ok_null' \ + 2 'rpc_set_service_failure_response "knfsd"' \ + 4 'rpc_set_service_failure_response "knfsd"' \ + 6 'rpc_set_service_failure_response "knfsd"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.121.sh b/ctdb/tests/eventscripts/60.nfs.monitor.121.sh new file mode 100755 index 0000000000..6d27f60b27 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.121.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "lockd down, 15 iterations" + +# This simulates an ongoing failure in the eventscript's automated +# attempts to restart the service. That is, the eventscript is unable +# to restart the service. + +setup_nfs +rpc_services_down "nlockmgr" + +#EVENTSCRIPTS_TESTS_TRACE="sh -x" +iterate_test 15 "ok_null" \ + 10 "rpc_set_service_failure_response 'lockd'" \ + 15 "rpc_set_service_failure_response 'lockd'" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.122.sh b/ctdb/tests/eventscripts/60.nfs.monitor.122.sh new file mode 100755 index 0000000000..fc5cea87c7 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.122.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "lockd down, 15 iterations, back up after 10" + +# This simulates a success the eventscript's automated attempts to +# restart the service. + +setup_nfs +rpc_services_down "nlockmgr" + +# Iteration 10 should try to restart rpc.lockd. However, our test +# stub rpc.lockd does nothing, so we have to explicitly flag it as up. + +iterate_test 15 "ok_null" \ + 10 "rpc_set_service_failure_response 'lockd'" \ + 11 "rpc_services_up nlockmgr" + diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.131.sh b/ctdb/tests/eventscripts/60.nfs.monitor.131.sh new file mode 100755 index 0000000000..1cf72a9284 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.131.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "rquotad down, 5 iterations" + +setup_nfs +rpc_services_down "rquotad" + +iterate_test 5 'rpc_set_service_failure_response "rquotad"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.132.sh b/ctdb/tests/eventscripts/60.nfs.monitor.132.sh new file mode 100755 index 0000000000..b8f3f2b041 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.132.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "rquotad down, 5 iterations, back up after 1" + +# rquotad fails once but then comes back of its own accord after 1 +# failure. + +setup_nfs +rpc_services_down "rquotad" + +iterate_test 5 'ok_null' \ + 1 'rpc_set_service_failure_response "rquotad"' \ + 2 'rpc_services_up "rquotad"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.141.sh b/ctdb/tests/eventscripts/60.nfs.monitor.141.sh new file mode 100755 index 0000000000..c77b1a7b05 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.141.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "statd down, 6 iterations" + +# statd fails and attempts to restart it fail. + +setup_nfs +rpc_services_down "status" + +iterate_test 6 'ok_null' \ + 2 'rpc_set_service_failure_response "statd"' \ + 4 'rpc_set_service_failure_response "statd"' \ + 6 'rpc_set_service_failure_response "statd"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.142.sh b/ctdb/tests/eventscripts/60.nfs.monitor.142.sh new file mode 100755 index 0000000000..4373d8d642 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.142.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "statd down, 8 iterations, back up after 2" + +# statd fails and the first attempt to restart it succeeds. + +setup_nfs +rpc_services_down "status" + +iterate_test 8 'ok_null' \ + 2 'rpc_set_service_failure_response "statd"' \ + 3 'rpc_services_up "status"' diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.151.sh b/ctdb/tests/eventscripts/60.nfs.monitor.151.sh new file mode 100755 index 0000000000..ea9aa7830e --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.151.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "mountd down, 1 iteration" + +setup_nfs +rpc_services_down "mountd" + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.152.sh b/ctdb/tests/eventscripts/60.nfs.monitor.152.sh new file mode 100755 index 0000000000..c4eb4194ae --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.152.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "mountd down, 10 iterations" + +# This simulates an ongoing failure in the eventscript's automated +# attempts to restart the service. That is, the eventscript is unable +# to restart the service. + +setup_nfs +rpc_services_down "mountd" + +iterate_test 10 "ok_null" \ + 5 "rpc_set_service_failure_response 'mountd'" \ + 10 "rpc_set_service_failure_response 'mountd'" + +#export FAKE_NETSTAT_TCP_ESTABLISHED="10.0.0.1:2049|10.254.254.1:12301 10.0.0.1:2049|10.254.254.1:12302 10.0.0.1:2049|10.254.254.1:12303 10.0.0.1:2049|10.254.254.2:12304 10.0.0.1:2049|10.254.254.2:12305" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.153.sh b/ctdb/tests/eventscripts/60.nfs.monitor.153.sh new file mode 100755 index 0000000000..cf33e39bbc --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.153.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "mountd down, 10 iterations, back up after 5" + +setup_nfs +rpc_services_down "mountd" + +# Iteration 5 should try to restart rpc.mountd. However, our test +# stub rpc.mountd does nothing, so we have to explicitly flag it as +# up. +iterate_test 10 "ok_null" \ + 5 "rpc_set_service_failure_response 'mountd'" \ + 6 "rpc_services_up mountd" diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.161.sh b/ctdb/tests/eventscripts/60.nfs.monitor.161.sh new file mode 100755 index 0000000000..1e07c181c2 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.161.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "2nd share missing" + +setup_nfs + +shares_missing "ERROR: nfs directory \"%s\" not available" 2 + +required_result 1 "$MISSING_SHARES_TEXT" + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.monitor.162.sh b/ctdb/tests/eventscripts/60.nfs.monitor.162.sh new file mode 100755 index 0000000000..ccd4ca84ea --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.monitor.162.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "2nd share missing, skipping share checks" + +setup_nfs +export CTDB_NFS_SKIP_SHARE_CHECK="yes" + +shares_missing "ERROR: nfs directory \"%s\" not available" 2 + +ok_null + +simple_test diff --git a/ctdb/tests/eventscripts/60.nfs.multi.001.sh b/ctdb/tests/eventscripts/60.nfs.multi.001.sh new file mode 100755 index 0000000000..e578c56c32 --- /dev/null +++ b/ctdb/tests/eventscripts/60.nfs.multi.001.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. "${TEST_SCRIPTS_DIR}/unit.sh" + +define_test "takeip, ipreallocated -> reconfigure" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +ok_null + +simple_test_event "takeip" $public_address + +ok < reconfigure" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +ok_null + +simple_test_event "takeip" $public_address + +# This currently assumes that ctdb scriptstatus will always return a +# good status (when replaying). That should change and we will need +# to split this into 2 tests. +ok < reconfigure, replay error" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="foo: bar error occurred" + +ok_null + +simple_test_event "takeip" $public_address + +ctdb_fake_scriptstatus 1 "ERROR" "$err" + +required_result 1 < reconfigure, replay timedout" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="waiting, waiting..." + +ok_null + +simple_test_event "takeip" $public_address + +ctdb_fake_scriptstatus -62 "TIMEDOUT" "$err" + +required_result 1 < reconfigure, replay disabled" + +setup_nfs + +public_address=$(ctdb_get_1_public_address) + +err="" + +ok_null + +simple_test_event "takeip" $public_address + +ctdb_fake_scriptstatus -8 "DISABLED" "$err" + +ok <&2 ; exit 1 ; } - -# Augment PATH with relevant stubs/ directories. We do this by actually -# setting PATH, and also by setting $EVENTSCRIPTS_PATH and then -# prepending that to $PATH in rc.local to avoid the PATH reset in -# functions. - -EVENTSCRIPTS_PATH="" - -if [ -d "${EVENTSCRIPTS_TESTS_DIR}/stubs" ] ; then - EVENTSCRIPTS_PATH="${EVENTSCRIPTS_TESTS_DIR}/stubs" -fi - -export EVENTSCRIPTS_TESTCASE_DIR=$(dirname "$0") -if [ $(basename "$EVENTSCRIPTS_TESTCASE_DIR") = "eventscripts" ] ; then - # Just a test script, no testcase subdirectory. - EVENTSCRIPTS_TESTCASE_DIR="$EVENTSCRIPTS_TESTS_DIR" -else - if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/stubs" ] ; then - EVENTSCRIPTS_PATH="${EVENTSCRIPTS_TESTCASE_DIR}/stubs:${EVENTSCRIPTS_PATH}" - fi -fi - -export EVENTSCRIPTS_PATH - -PATH="${EVENTSCRIPTS_PATH}:${PATH}" - -if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc" ] ; then - CTDB_ETCDIR="${EVENTSCRIPTS_TESTCASE_DIR}/etc" -elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc" ] ; then - CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_DIR}/etc" -else - die "Unable to set \$CTDB_ETCDIR" -fi -export CTDB_ETCDIR - -if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc-ctdb" ] ; then - CTDB_BASE="${EVENTSCRIPTS_TESTCASE_DIR}/etc-ctdb" -elif [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" ] ; then - CTDB_BASE="${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" -elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" ] ; then - CTDB_BASE="${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" -else - die "Unable to set \$CTDB_BASE" -fi -export CTDB_BASE - -export EVENTSCRIPTS_TESTS_VAR_DIR="${EVENTSCRIPTS_TESTS_DIR}/var" -if [ "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/var" ] ; then - rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR" -fi -mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR" -export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb" - -###################################################################### - -if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] ; then - debug () { echo "$@" ; } -else - debug () { : ; } -fi - -eventscripts_tests_cleanup_hooks="" - -# This loses quoting! -eventscripts_test_add_cleanup () -{ - eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*" -} - -trap 'eval $eventscripts_tests_cleanup_hooks' 0 - - -###################################################################### - -# General setup fakery - -setup_generic () -{ - debug "Setting up shares (3 existing shares)" - # Create 3 fake shares/exports. - export FAKE_SHARES="" - for i in $(seq 1 3) ; do - _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing" - mkdir -p "$_s" - FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}" - done - - export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding" - mkdir -p "$FAKE_PROC_NET_BONDING" - rm -f "$FAKE_PROC_NET_BONDING"/* - - export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down" - mkdir -p "$FAKE_ETHTOOL_LINK_DOWN" - rm -f "$FAKE_ETHTOOL_LINK_DOWN"/* - - # This can only have 2 levels. We don't want to resort to usings - # something dangerous like "rm -r" setup time. - export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state" - mkdir -p "$FAKE_IP_STATE" - rm -f "$FAKE_IP_STATE"/*/* - rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true - rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true -} - -tcp_port_down () -{ - for _i ; do - debug "Marking TCP port \"${_i}\" as not listening" - FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g") - done -} - -shares_missing () -{ - _fmt="$1" ; shift - - # Replace some shares with non-existent ones. - _t="" - _n=1 - _nl=" -" - export MISSING_SHARES_TEXT="" - for _i in $FAKE_SHARES ; do - if [ $_n = "$1" ] ; then - shift - _i="${_i%_existing}_missing" - debug "Replacing share $_n with missing share \"$_i\"" - rmdir "$_i" 2>/dev/null || true - MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}") - fi - _t="${_t}${_t:+ }${_i}" - _n=$(($_n + 1)) - done - FAKE_SHARES="$_t" -} - -# Setup some fake /proc/net/bonding files with just enough info for -# the eventscripts. - -# arg1 is interface name, arg2 is currently active slave (use "None" -# if none), arg3 is MII status ("up" or "down"). -setup_bond () -{ - _iface="$1" - _slave="${2:-${_iface}_sl_0}" - _mii_s="${3:-up}" - _mii_subs="${4:-${_mii_s:-up}}" - echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s" - cat >"${FAKE_PROC_NET_BONDING}/$_iface" <> 8)) - done - - echo "${_net}/${_maskbits}" -} - -###################################################################### - -# CTDB fakery - -# Evaluate an expression that probably calls functions or uses -# variables from the CTDB functions file. This is used for test -# initialisation. -eventscript_call () -{ - ( - . "$CTDB_BASE/functions" - "$@" - ) -} - -# Set output for ctdb command. Option 1st argument is return code. -ctdb_set_output () -{ - _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out" - cat >"$_out" - - _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc" - echo "${1:-0}" >"$_rc" - - eventscripts_test_add_cleanup "rm -f $_out $_rc" -} - -setup_ctdb () -{ - setup_generic - - export FAKE_CTDB_NUMNODES="${1:-3}" - echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes" - - export FAKE_CTDB_PNN="${2:-0}" - echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}" - - if [ -n "$3" ] ; then - echo "Setting up CTDB_PUBLIC_ADDRESSES: $3" - export CTDB_PUBLIC_ADDRESSES=$(mktemp) - for _i in $3 ; do - _ip="${_i%@*}" - _ifaces="${_i#*@}" - echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES" - done - eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES" - fi - - export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb" - - export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down" - mkdir -p "$FAKE_CTDB_IFACES_DOWN" - rm -f "$FAKE_CTDB_IFACES_DOWN"/* - - export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus" - mkdir -p "$FAKE_CTDB_SCRIPTSTATUS" - rm -f "$FAKE_CTDB_SCRIPTSTATUS"/* -} - -ctdb_get_interfaces () -{ - # The echo/subshell forces all the output onto 1 line. - echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}') -} - -ctdb_get_1_interface () -{ - _t=$(ctdb_get_interfaces) - echo ${_t%% *} -} - -# Print all public addresses as: interface IP maskbits -# Each line is suitable for passing to takeip/releaseip -ctdb_get_all_public_addresses () -{ - _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" - while IFS="/$IFS" read _ip _maskbits _ifaces ; do - echo "$_ifaces $_ip $_maskbits" - done <"$_f" -} - -# Print public addresses on this node as: interface IP maskbits -# Each line is suitable for passing to takeip/releaseip -ctdb_get_my_public_addresses () -{ - ctdb ip -v -Y | { - read _x # skip header line - - while IFS=":" read _x _ip _x _iface _x ; do - [ -n "$_iface" ] || continue - while IFS="/$IFS" read _i _maskbits _x ; do - if [ "$_ip" = "$_i" ] ; then - echo $_iface $_ip $_maskbits - break - fi - done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" - done - } -} - -# Prints the 1st public address as: interface IP maskbits -# This is suitable for passing to takeip/releaseip -ctdb_get_1_public_address () -{ - ctdb_get_my_public_addresses | head -n 1 -} - -ctdb_not_implemented () -{ - export CTDB_NOT_IMPLEMENTED="$1" - ctdb_not_implemented="\ -DEBUG: ctdb: command \"$1\" not implemented in stub" -} - -ctdb_fake_scriptstatus () -{ - _code="$1" - _status="$2" - _err_out="$3" - - _d1=$(date '+%s.%N') - _d2=$(date '+%s.%N') - - echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script" -} - -setup_ctdb_policy_routing () -{ - export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing" - export CTDB_PER_IP_ROUTING_RULE_PREF=100 - export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000 - export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000 - - # Tests need to create and populate this file - rm -f "$CTDB_PER_IP_ROUTING_CONF" -} - -###################################################################### - -# Samba fakery - -setup_samba () -{ - setup_ctdb - - if [ "$1" != "down" ] ; then - - debug "Marking Samba services as up, listening and managed by CTDB" - # Get into known state. - for i in "samba" "winbind" ; do - eventscript_call ctdb_service_managed "$i" - done - # All possible service names for all known distros. - for i in "smb" "nmb" "winbind" "samba" ; do - service "$i" force-started - done - - export CTDB_SAMBA_SKIP_SHARE_CHECK="no" - export CTDB_MANAGED_SERVICES="foo samba winbind bar" - - export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139" - export FAKE_WBINFO_FAIL="no" - - # Some things in 50.samba are backgrounded and waited for. If - # we don't sleep at all then timeouts can happen. This avoids - # that... :-) - export FAKE_SLEEP_FORCE=0.1 - else - debug "Marking Samba services as down, not listening and not managed by CTDB" - # Get into known state. - for i in "samba" "winbind" ; do - eventscript_call ctdb_service_unmanaged "$i" - done - # All possible service names for all known distros. - for i in "smb" "nmb" "winbind" "samba" ; do - service "$i" force-stopped - done - - export CTDB_SAMBA_SKIP_SHARE_CHECK="no" - export CTDB_MANAGED_SERVICES="foo bar" - unset CTDB_MANAGES_SAMBA - unset CTDB_MANAGES_WINBIND - - export FAKE_TCP_LISTEN="" - export FAKE_WBINFO_FAIL="yes" - fi - - # This is ugly but if this file isn't removed before each test - # then configuration changes between tests don't stick. - rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache" -} - -wbinfo_down () -{ - debug "Making wbinfo commands fail" - FAKE_WBINFO_FAIL="yes" -} - -###################################################################### - -# NFS fakery - -setup_nfs () -{ - setup_ctdb - - export FAKE_RPCINFO_SERVICES="" - - export CTDB_NFS_SKIP_SHARE_CHECK="no" - export CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK="no" - - # Reset the failcounts for nfs services. - eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' - - rpc_fail_limits_file="${EVENTSCRIPTS_TESTS_VAR_DIR}/rpc_fail_limits" - - # Force this file to exist so tests can be individually run. - if [ ! -f "$rpc_fail_limits_file" ] ; then - # This is gross... but is needed to fake through the nfs monitor event. - eventscript_call ctdb_service_managed "nfs" - service "nfs" force-started # might not be enough - CTDB_RC_LOCAL="$CTDB_BASE/rc.local.nfs.monitor.get-limits" \ - CTDB_MANAGES_NFS="yes" \ - "${CTDB_BASE}/events.d/60.nfs" "monitor" >"$rpc_fail_limits_file" - fi - - if [ "$1" != "down" ] ; then - debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB" - - eventscript_call ctdb_service_managed "nfs" - service "nfs" force-started # might not be enough - - export CTDB_MANAGED_SERVICES="foo nfs bar" - - rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status" - else - debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB" - - eventscript_call ctdb_service_unmanaged "nfs" - service "nfs" force-stopped # might not be enough - eventscript_call startstop_nfs stop - - export CTDB_MANAGED_SERVICES="foo bar" - unset CTDB_MANAGES_NFS - fi -} - -rpc_services_down () -{ - for _i ; do - debug "Marking RPC service \"${_i}\" as unavailable" - FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g") - done -} - -rpc_services_up () -{ - for _i ; do - debug "Marking RPC service \"${_i}\" as available" - case "$_i" in - nfs) _t="2:3" ;; - mountd) _t="1:3" ;; - rquotad) _t="1:2" ;; - nlockmgr) _t="3:4" ;; - status) _t="1:1" ;; - *) die "Internal error - unsupported RPC service \"${_i}\"" ;; - esac - - FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}" - done -} - -# Set the required result for a particular RPC program having failed -# for a certain number of iterations. This is probably still a work -# in progress. Note that we could hook aggressively -# nfs_check_rpc_service() to try to implement this but we're better -# off testing nfs_check_rpc_service() using independent code... even -# if it is incomplete and hacky. So, if the 60.nfs eventscript -# changes and the tests start to fail then it may be due to this -# function being incomplete. -rpc_set_service_failure_response () -{ - _progname="$1" - # The number of failures defaults to the iteration number. This - # will be true when we fail from the 1st iteration... but we need - # the flexibility to set the number of failures. - _numfails="${2:-${iteration}}" - - _etc="$CTDB_ETCDIR" # shortcut for readability - for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do - if [ -r "$_c" ] ; then - . "$_c" - break - fi - done - - # A handy newline. :-) - _nl=" -" - - # Default - ok_null - - _ts=$(sed -n -e "s@^${_progname} @@p" "$rpc_fail_limits_file") - - while [ -n "$_ts" ] ; do - # Get the triple: operator, fail limit and actions. - _op="${_ts%% *}" ; _ts="${_ts#* }" - _li="${_ts%% *}" ; _ts="${_ts#* }" - # We've lost some of the quoting but we can simulate - # because we know an operator is always the first in a - # triple. - _actions="" - while [ -n "$_ts" ] ; do - # If this is an operator then we've got all of the - # actions. - case "$_ts" in - -*) break ;; - esac - - _actions="${_actions}${_actions:+ }${_ts%% *}" - # Special case for end of list. - if [ "$_ts" != "${_ts#* }" ] ; then - _ts="${_ts#* }" - else - _ts="" - fi - done - - if [ "$_numfails" "$_op" "$_li" ] ; then - _out="" - _rc=0 - for _action in $_actions ; do - case "$_action" in - verbose) - _ver=1 - _pn="$_progname" - case "$_progname" in - knfsd) _ver=3 ; _pn="nfs" ;; - lockd) _ver=4 ; _pn="nlockmgr" ;; - statd) _pn="status" ;; - esac - _out="\ -ERROR: $_pn failed RPC check: -rpcinfo: RPC: Program not registered -program $_pn version $_ver is not available" - ;; - restart|restart:*) - _opts="" - _p="rpc.${_progname}" - case "$_progname" in - statd) - _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}" - ;; - esac - case "${_progname}${_action#restart}" in - knfsd) - _t="\ -Trying to restart NFS service -Starting nfslock: OK -Starting nfs: OK" - ;; - knfsd:bs) - _t="Trying to restart NFS service" - ;; - lockd) - _t="\ -Trying to restart lock manager service -Stopping nfslock: OK -Starting nfslock: OK" - ;; - lockd:*) - _t="Trying to restart lock manager service" - ;; - *) - _t="Trying to restart $_progname [${_p}${_opts}]" - esac - _out="${_out}${_out:+${_nl}}${_t}" - ;; - unhealthy) - _rc=1 - esac - done - required_result $_rc "$_out" - return - fi - done -} - -###################################################################### - -# VSFTPD fakery - -setup_vsftpd () -{ - if [ "$1" != "down" ] ; then - die "setup_vsftpd up not implemented!!!" - else - debug "Setting up VSFTPD environment: service down, not managed by CTDB" - - eventscript_call ctdb_service_unmanaged vsftpd - service vsftpd force-stopped - - export CTDB_MANAGED_SERVICES="foo" - unset CTDB_MANAGES_VSFTPD - fi -} - -###################################################################### - -# HTTPD fakery - -setup_httpd () -{ - if [ "$1" != "down" ] ; then - die "setup_httpd up not implemented!!!" - else - debug "Setting up HTTPD environment: service down, not managed by CTDB" - - for i in "apache2" "httpd" ; do - eventscript_call ctdb_service_unmanaged "$i" - service "$i" force-stopped - done - - export CTDB_MANAGED_SERVICES="foo" - unset CTDB_MANAGES_HTTPD - fi -} - -###################################################################### - -# Result and test functions - -# Set some globals and print the summary. -define_test () -{ - desc="$1" - - _f="$0" - _f="${_f#./}" # strip leading ./ - _f="${_f#simple/}" # strip leading simple/ - _f="${_f#multievent/}" # strip leading multievent/ - _f="${_f%%/*}" # if subdir, strip off file - _f="${_f%.sh}" # strip off .sh suffix if any - - # Remaining format should be NN.service.event.NNN or NN.service.NNN: - _num="${_f##*.}" - _f="${_f%.*}" - case "$_f" in - *.*.*) - script="${_f%.*}" - event="${_f##*.}" - ;; - *.*) - script="$_f" - unset event - ;; - *) - die "Internal error - unknown testcase filename format" - esac - - printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" -} - -# Set the required result for a test. -# - Argument 1 is exit code. -# - Argument 2, if present is the required test output but "--" -# indicates empty output. -# If argument 2 is not present or null then read required test output -# from stdin. -required_result () -{ - required_rc="${1:-0}" - if [ -n "$2" ] ; then - if [ "$2" = "--" ] ; then - required_output="" - else - required_output="$2" - fi - else - if ! tty -s ; then - required_output=$(cat) - else - required_output="" - fi - fi -} - -ok () -{ - required_result 0 "$@" -} - -ok_null () -{ - ok -- -} - -result_print () -{ - _passed="$1" - _out="$2" - _rc="$3" - _iteration="$4" - - if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then - if [ -n "$_iteration" ] ; then - cat <"$_outr" - - _outf=$(mktemp) - echo "$_out" >"$_outf" - - cat <&1) - - result_check -} - -simple_test_event () -{ - # If something has previously failed then don't continue. - : ${_passed:=true} - $_passed || return 1 - - event="$1" ; shift - echo "##################################################" - simple_test "$@" -} - -simple_test_command () -{ - # If something has previously failed then don't continue. - : ${_passed:=true} - $_passed || return 1 - - echo "##################################################" - echo "Running command \"$*\"" - _out=$("$@" 2>&1) - - result_check -} - -# Run an eventscript iteratively. -# - 1st argument is the number of iterations. -# - 2nd argument is something to eval to do setup for every iteration. -# The easiest thing to do here is to define a function and pass it -# here. -# - Subsequent arguments come in pairs: an iteration number and -# something to eval for that iteration. Each time an iteration -# number is matched the associated argument is given to eval after -# the default setup is done. The iteration numbers need to be given -# in ascending order. -# -# Some optional args can be given *before* these, surrounded by extra -# "--" args. These args are passed to the eventscript. Quoting is -# lost. -# -# One use of the 2nd and further arguments is to call -# required_result() to change what is expected of a particular -# iteration. -iterate_test () -{ - [ -n "$event" ] || die 'simple_test: $event not set' - - args="" - if [ "$1" = "--" ] ; then - shift - while [ "$1" != "--" ] ; do - args="${args}${args:+ }$1" - shift - done - shift - fi - - _repeats="$1" - _setup_default="$2" - shift 2 - - echo "Running $_repeats iterations of \"$script $event\" $args" - - _result=true - - for iteration in $(seq 1 $_repeats) ; do - # This is inefficient because the iteration-specific setup - # might completely replace the default one. However, running - # the default is good because it allows you to revert to a - # particular result without needing to specify it explicitly. - eval $_setup_default - if [ $iteration = "$1" ] ; then - eval $2 - shift 2 - fi - - _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1) - _rc=$? - - if [ -n "$OUT_FILTER" ] ; then - _fout=$(echo "$_out" | eval sed -r $OUT_FILTER) - else - _fout="$_out" - fi - - if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then - _passed=true - else - _passed=false - _result=false - fi - - result_print "$_passed" "$_out" "$_rc" "$iteration" - done - - result_footer "$_result" -} diff --git a/ctdb/tests/eventscripts/multievent/10.interface.001.sh b/ctdb/tests/eventscripts/multievent/10.interface.001.sh deleted file mode 100755 index 5ef254c536..0000000000 --- a/ctdb/tests/eventscripts/multievent/10.interface.001.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "takeip, removeip" - -setup_ctdb - -public_address=$(ctdb_get_1_public_address) - -ok_null - -simple_test_event "takeip" $public_address -simple_test_event "releaseip" $public_address diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh deleted file mode 100755 index 0bcf8e3055..0000000000 --- a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "not configured" - -setup_ctdb - -ok <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" <"$CTDB_PER_IP_ROUTING_CONF" - -ctdb_get_1_public_address | -{ - read dev ip bits - - net=$(ipv4_host_addr_to_net "$ip" "$bits") - gw="${net%.*}.1" # a dumb, calculated default - - ok_null - - simple_test_event "takeip" $dev $ip $bits - - ok <"$CTDB_PER_IP_ROUTING_CONF" - -# Now do a takeip for each IP on the "current" node -ctdb_get_my_public_addresses | -{ - policy_rules="" - policy_routes="" - while read dev ip bits ; do - - net=$(ipv4_host_addr_to_net "$ip" "$bits") - gw="${net%.*}.1" # a dumb, calculated default - - ok_null - - simple_test_event "takeip" $dev $ip $bits - - policy_rules="${policy_rules} -${CTDB_PER_IP_ROUTING_RULE_PREF}: from $ip lookup ctdb.$ip " - policy_routes="${policy_routes} -# ip route show table ctdb.$ip -$net dev $dev scope link -default via $gw dev $dev " - done - - ok <"$CTDB_PER_IP_ROUTING_CONF" < reconfigure" - -setup_nfs - -public_address=$(ctdb_get_1_public_address) - -ok_null - -simple_test_event "takeip" $public_address - -ok < reconfigure" - -setup_nfs - -public_address=$(ctdb_get_1_public_address) - -ok_null - -simple_test_event "takeip" $public_address - -# This currently assumes that ctdb scriptstatus will always return a -# good status (when replaying). That should change and we will need -# to split this into 2 tests. -ok < reconfigure, replay error" - -setup_nfs - -public_address=$(ctdb_get_1_public_address) - -err="foo: bar error occurred" - -ok_null - -simple_test_event "takeip" $public_address - -ctdb_fake_scriptstatus 1 "ERROR" "$err" - -required_result 1 < reconfigure, replay timedout" - -setup_nfs - -public_address=$(ctdb_get_1_public_address) - -err="waiting, waiting..." - -ok_null - -simple_test_event "takeip" $public_address - -ctdb_fake_scriptstatus -62 "TIMEDOUT" "$err" - -required_result 1 < reconfigure, replay disabled" - -setup_nfs - -public_address=$(ctdb_get_1_public_address) - -err="" - -ok_null - -simple_test_event "takeip" $public_address - -ctdb_fake_scriptstatus -8 "DISABLED" "$err" - -ok </dev/null || true + rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true +} + +tcp_port_down () +{ + for _i ; do + debug "Marking TCP port \"${_i}\" as not listening" + FAKE_TCP_LISTEN=$(echo "$FAKE_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g") + done +} + +shares_missing () +{ + _fmt="$1" ; shift + + # Replace some shares with non-existent ones. + _t="" + _n=1 + _nl=" +" + export MISSING_SHARES_TEXT="" + for _i in $FAKE_SHARES ; do + if [ $_n = "$1" ] ; then + shift + _i="${_i%_existing}_missing" + debug "Replacing share $_n with missing share \"$_i\"" + rmdir "$_i" 2>/dev/null || true + MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}") + fi + _t="${_t}${_t:+ }${_i}" + _n=$(($_n + 1)) + done + FAKE_SHARES="$_t" +} + +# Setup some fake /proc/net/bonding files with just enough info for +# the eventscripts. + +# arg1 is interface name, arg2 is currently active slave (use "None" +# if none), arg3 is MII status ("up" or "down"). +setup_bond () +{ + _iface="$1" + _slave="${2:-${_iface}_sl_0}" + _mii_s="${3:-up}" + _mii_subs="${4:-${_mii_s:-up}}" + echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s" + cat >"${FAKE_PROC_NET_BONDING}/$_iface" <> 8)) + done + + echo "${_net}/${_maskbits}" +} + +###################################################################### + +# CTDB fakery + +# Evaluate an expression that probably calls functions or uses +# variables from the CTDB functions file. This is used for test +# initialisation. +eventscript_call () +{ + ( + . "$CTDB_BASE/functions" + "$@" + ) +} + +# Set output for ctdb command. Option 1st argument is return code. +ctdb_set_output () +{ + _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out" + cat >"$_out" + + _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc" + echo "${1:-0}" >"$_rc" + + eventscripts_test_add_cleanup "rm -f $_out $_rc" +} + +setup_ctdb () +{ + setup_generic + + export FAKE_CTDB_NUMNODES="${1:-3}" + echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes" + + export FAKE_CTDB_PNN="${2:-0}" + echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}" + + if [ -n "$3" ] ; then + echo "Setting up CTDB_PUBLIC_ADDRESSES: $3" + export CTDB_PUBLIC_ADDRESSES=$(mktemp) + for _i in $3 ; do + _ip="${_i%@*}" + _ifaces="${_i#*@}" + echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES" + done + eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES" + fi + + export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb" + + export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down" + mkdir -p "$FAKE_CTDB_IFACES_DOWN" + rm -f "$FAKE_CTDB_IFACES_DOWN"/* + + export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus" + mkdir -p "$FAKE_CTDB_SCRIPTSTATUS" + rm -f "$FAKE_CTDB_SCRIPTSTATUS"/* +} + +ctdb_get_interfaces () +{ + # The echo/subshell forces all the output onto 1 line. + echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}') +} + +ctdb_get_1_interface () +{ + _t=$(ctdb_get_interfaces) + echo ${_t%% *} +} + +# Print all public addresses as: interface IP maskbits +# Each line is suitable for passing to takeip/releaseip +ctdb_get_all_public_addresses () +{ + _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + while IFS="/$IFS" read _ip _maskbits _ifaces ; do + echo "$_ifaces $_ip $_maskbits" + done <"$_f" +} + +# Print public addresses on this node as: interface IP maskbits +# Each line is suitable for passing to takeip/releaseip +ctdb_get_my_public_addresses () +{ + ctdb ip -v -Y | { + read _x # skip header line + + while IFS=":" read _x _ip _x _iface _x ; do + [ -n "$_iface" ] || continue + while IFS="/$IFS" read _i _maskbits _x ; do + if [ "$_ip" = "$_i" ] ; then + echo $_iface $_ip $_maskbits + break + fi + done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + done + } +} + +# Prints the 1st public address as: interface IP maskbits +# This is suitable for passing to takeip/releaseip +ctdb_get_1_public_address () +{ + ctdb_get_my_public_addresses | head -n 1 +} + +ctdb_not_implemented () +{ + export CTDB_NOT_IMPLEMENTED="$1" + ctdb_not_implemented="\ +DEBUG: ctdb: command \"$1\" not implemented in stub" +} + +ctdb_fake_scriptstatus () +{ + _code="$1" + _status="$2" + _err_out="$3" + + _d1=$(date '+%s.%N') + _d2=$(date '+%s.%N') + + echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script" +} + +setup_ctdb_policy_routing () +{ + export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing" + export CTDB_PER_IP_ROUTING_RULE_PREF=100 + export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000 + export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000 + + # Tests need to create and populate this file + rm -f "$CTDB_PER_IP_ROUTING_CONF" +} + +###################################################################### + +# Samba fakery + +setup_samba () +{ + setup_ctdb + + if [ "$1" != "down" ] ; then + + debug "Marking Samba services as up, listening and managed by CTDB" + # Get into known state. + for i in "samba" "winbind" ; do + eventscript_call ctdb_service_managed "$i" + done + # All possible service names for all known distros. + for i in "smb" "nmb" "winbind" "samba" ; do + service "$i" force-started + done + + export CTDB_SAMBA_SKIP_SHARE_CHECK="no" + export CTDB_MANAGED_SERVICES="foo samba winbind bar" + + export FAKE_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139" + export FAKE_WBINFO_FAIL="no" + + # Some things in 50.samba are backgrounded and waited for. If + # we don't sleep at all then timeouts can happen. This avoids + # that... :-) + export FAKE_SLEEP_FORCE=0.1 + else + debug "Marking Samba services as down, not listening and not managed by CTDB" + # Get into known state. + for i in "samba" "winbind" ; do + eventscript_call ctdb_service_unmanaged "$i" + done + # All possible service names for all known distros. + for i in "smb" "nmb" "winbind" "samba" ; do + service "$i" force-stopped + done + + export CTDB_SAMBA_SKIP_SHARE_CHECK="no" + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_SAMBA + unset CTDB_MANAGES_WINBIND + + export FAKE_TCP_LISTEN="" + export FAKE_WBINFO_FAIL="yes" + fi + + # This is ugly but if this file isn't removed before each test + # then configuration changes between tests don't stick. + rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache" +} + +wbinfo_down () +{ + debug "Making wbinfo commands fail" + FAKE_WBINFO_FAIL="yes" +} + +###################################################################### + +# NFS fakery + +setup_nfs () +{ + setup_ctdb + + export FAKE_RPCINFO_SERVICES="" + + export CTDB_NFS_SKIP_SHARE_CHECK="no" + export CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK="no" + + # Reset the failcounts for nfs services. + eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' + + rpc_fail_limits_file="${EVENTSCRIPTS_TESTS_VAR_DIR}/rpc_fail_limits" + + # Force this file to exist so tests can be individually run. + if [ ! -f "$rpc_fail_limits_file" ] ; then + # This is gross... but is needed to fake through the nfs monitor event. + eventscript_call ctdb_service_managed "nfs" + service "nfs" force-started # might not be enough + CTDB_RC_LOCAL="$CTDB_BASE/rc.local.nfs.monitor.get-limits" \ + CTDB_MANAGES_NFS="yes" \ + "${CTDB_BASE}/events.d/60.nfs" "monitor" >"$rpc_fail_limits_file" + fi + + if [ "$1" != "down" ] ; then + debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB" + + eventscript_call ctdb_service_managed "nfs" + service "nfs" force-started # might not be enough + + export CTDB_MANAGED_SERVICES="foo nfs bar" + + rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status" + else + debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB" + + eventscript_call ctdb_service_unmanaged "nfs" + service "nfs" force-stopped # might not be enough + eventscript_call startstop_nfs stop + + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_NFS + fi +} + +rpc_services_down () +{ + for _i ; do + debug "Marking RPC service \"${_i}\" as unavailable" + FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g") + done +} + +rpc_services_up () +{ + for _i ; do + debug "Marking RPC service \"${_i}\" as available" + case "$_i" in + nfs) _t="2:3" ;; + mountd) _t="1:3" ;; + rquotad) _t="1:2" ;; + nlockmgr) _t="3:4" ;; + status) _t="1:1" ;; + *) die "Internal error - unsupported RPC service \"${_i}\"" ;; + esac + + FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}" + done +} + +# Set the required result for a particular RPC program having failed +# for a certain number of iterations. This is probably still a work +# in progress. Note that we could hook aggressively +# nfs_check_rpc_service() to try to implement this but we're better +# off testing nfs_check_rpc_service() using independent code... even +# if it is incomplete and hacky. So, if the 60.nfs eventscript +# changes and the tests start to fail then it may be due to this +# function being incomplete. +rpc_set_service_failure_response () +{ + _progname="$1" + # The number of failures defaults to the iteration number. This + # will be true when we fail from the 1st iteration... but we need + # the flexibility to set the number of failures. + _numfails="${2:-${iteration}}" + + _etc="$CTDB_ETCDIR" # shortcut for readability + for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do + if [ -r "$_c" ] ; then + . "$_c" + break + fi + done + + # A handy newline. :-) + _nl=" +" + + # Default + ok_null + + _ts=$(sed -n -e "s@^${_progname} @@p" "$rpc_fail_limits_file") + + while [ -n "$_ts" ] ; do + # Get the triple: operator, fail limit and actions. + _op="${_ts%% *}" ; _ts="${_ts#* }" + _li="${_ts%% *}" ; _ts="${_ts#* }" + # We've lost some of the quoting but we can simulate + # because we know an operator is always the first in a + # triple. + _actions="" + while [ -n "$_ts" ] ; do + # If this is an operator then we've got all of the + # actions. + case "$_ts" in + -*) break ;; + esac + + _actions="${_actions}${_actions:+ }${_ts%% *}" + # Special case for end of list. + if [ "$_ts" != "${_ts#* }" ] ; then + _ts="${_ts#* }" + else + _ts="" + fi + done + + if [ "$_numfails" "$_op" "$_li" ] ; then + _out="" + _rc=0 + for _action in $_actions ; do + case "$_action" in + verbose) + _ver=1 + _pn="$_progname" + case "$_progname" in + knfsd) _ver=3 ; _pn="nfs" ;; + lockd) _ver=4 ; _pn="nlockmgr" ;; + statd) _pn="status" ;; + esac + _out="\ +ERROR: $_pn failed RPC check: +rpcinfo: RPC: Program not registered +program $_pn version $_ver is not available" + ;; + restart|restart:*) + _opts="" + _p="rpc.${_progname}" + case "$_progname" in + statd) + _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}" + ;; + esac + case "${_progname}${_action#restart}" in + knfsd) + _t="\ +Trying to restart NFS service +Starting nfslock: OK +Starting nfs: OK" + ;; + knfsd:bs) + _t="Trying to restart NFS service" + ;; + lockd) + _t="\ +Trying to restart lock manager service +Stopping nfslock: OK +Starting nfslock: OK" + ;; + lockd:*) + _t="Trying to restart lock manager service" + ;; + *) + _t="Trying to restart $_progname [${_p}${_opts}]" + esac + _out="${_out}${_out:+${_nl}}${_t}" + ;; + unhealthy) + _rc=1 + esac + done + required_result $_rc "$_out" + return + fi + done +} + +###################################################################### + +# VSFTPD fakery + +setup_vsftpd () +{ + if [ "$1" != "down" ] ; then + die "setup_vsftpd up not implemented!!!" + else + debug "Setting up VSFTPD environment: service down, not managed by CTDB" + + eventscript_call ctdb_service_unmanaged vsftpd + service vsftpd force-stopped + + export CTDB_MANAGED_SERVICES="foo" + unset CTDB_MANAGES_VSFTPD + fi +} + +###################################################################### + +# HTTPD fakery + +setup_httpd () +{ + if [ "$1" != "down" ] ; then + die "setup_httpd up not implemented!!!" + else + debug "Setting up HTTPD environment: service down, not managed by CTDB" + + for i in "apache2" "httpd" ; do + eventscript_call ctdb_service_unmanaged "$i" + service "$i" force-stopped + done + + export CTDB_MANAGED_SERVICES="foo" + unset CTDB_MANAGES_HTTPD + fi +} + +###################################################################### + +# Result and test functions + +# Set some globals and print the summary. +define_test () +{ + desc="$1" + + _f=$(basename "$0" ".sh") + + # Remaining format should be NN.service.event.NNN or NN.service.NNN: + _num="${_f##*.}" + _f="${_f%.*}" + case "$_f" in + *.*.*) + script="${_f%.*}" + event="${_f##*.}" + ;; + *.*) + script="$_f" + unset event + ;; + *) + die "Internal error - unknown testcase filename format" + esac + + printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" +} + +# Set the required result for a test. +# - Argument 1 is exit code. +# - Argument 2, if present is the required test output but "--" +# indicates empty output. +# If argument 2 is not present or null then read required test output +# from stdin. +required_result () +{ + required_rc="${1:-0}" + if [ -n "$2" ] ; then + if [ "$2" = "--" ] ; then + required_output="" + else + required_output="$2" + fi + else + if ! tty -s ; then + required_output=$(cat) + else + required_output="" + fi + fi +} + +ok () +{ + required_result 0 "$@" +} + +ok_null () +{ + ok -- +} + +result_print () +{ + _passed="$1" + _out="$2" + _rc="$3" + _iteration="$4" + + if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then + if [ -n "$_iteration" ] ; then + cat <"$_outr" + + _outf=$(mktemp) + echo "$_out" >"$_outf" + + cat <&1) + + result_check +} + +simple_test_event () +{ + # If something has previously failed then don't continue. + : ${_passed:=true} + $_passed || return 1 + + event="$1" ; shift + echo "##################################################" + simple_test "$@" +} + +simple_test_command () +{ + # If something has previously failed then don't continue. + : ${_passed:=true} + $_passed || return 1 + + echo "##################################################" + echo "Running command \"$*\"" + _out=$("$@" 2>&1) + + result_check +} + +# Run an eventscript iteratively. +# - 1st argument is the number of iterations. +# - 2nd argument is something to eval to do setup for every iteration. +# The easiest thing to do here is to define a function and pass it +# here. +# - Subsequent arguments come in pairs: an iteration number and +# something to eval for that iteration. Each time an iteration +# number is matched the associated argument is given to eval after +# the default setup is done. The iteration numbers need to be given +# in ascending order. +# +# Some optional args can be given *before* these, surrounded by extra +# "--" args. These args are passed to the eventscript. Quoting is +# lost. +# +# One use of the 2nd and further arguments is to call +# required_result() to change what is expected of a particular +# iteration. +iterate_test () +{ + [ -n "$event" ] || die 'simple_test: $event not set' + + args="" + if [ "$1" = "--" ] ; then + shift + while [ "$1" != "--" ] ; do + args="${args}${args:+ }$1" + shift + done + shift + fi + + _repeats="$1" + _setup_default="$2" + shift 2 + + echo "Running $_repeats iterations of \"$script $event\" $args" + + _result=true + + for iteration in $(seq 1 $_repeats) ; do + # This is inefficient because the iteration-specific setup + # might completely replace the default one. However, running + # the default is good because it allows you to revert to a + # particular result without needing to specify it explicitly. + eval $_setup_default + if [ $iteration = "$1" ] ; then + eval $2 + shift 2 + fi + + _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1) + _rc=$? + + if [ -n "$OUT_FILTER" ] ; then + _fout=$(echo "$_out" | eval sed -r $OUT_FILTER) + else + _fout="$_out" + fi + + if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then + _passed=true + else + _passed=false + _result=false + fi + + result_print "$_passed" "$_out" "$_rc" "$iteration" + done + + result_footer "$_result" +} diff --git a/ctdb/tests/eventscripts/simple/10.interface.init.001.sh b/ctdb/tests/eventscripts/simple/10.interface.init.001.sh deleted file mode 100755 index 5a61ab3efd..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.init.001.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "no public addresses" - -setup_ctdb - -export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" - -ok "No public addresses file found. Nothing to do for 10.interfaces" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.init.002.sh b/ctdb/tests/eventscripts/simple/10.interface.init.002.sh deleted file mode 100755 index 43bacd5231..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.init.002.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "all interfaces up" - -setup_ctdb - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.001.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.001.sh deleted file mode 100755 index 411355f105..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.001.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "no public addresses" - -setup_ctdb - -export CTDB_PUBLIC_ADDRESSES="$CTDB_ETC/does/not/exist" - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.002.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.002.sh deleted file mode 100755 index 43bacd5231..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.002.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "all interfaces up" - -setup_ctdb - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.003.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.003.sh deleted file mode 100755 index 5dd432816b..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.003.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "1 interface down" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -ethtool_interfaces_down $iface - -required_result 1 "ERROR: No link on the public network interface $iface" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.004.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.004.sh deleted file mode 100755 index d215c40019..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.004.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "all interfaces up, 1 is a bond" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -setup_bond $iface - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.005.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.005.sh deleted file mode 100755 index 00db01fe63..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.005.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "1 bond, no active slaves" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -setup_bond $iface "None" - -required_result 1 "ERROR: No active slaves for bond device $iface" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.006.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.006.sh deleted file mode 100755 index 862a972373..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.006.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "1 bond, active slaves, link down" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -setup_bond $iface "" "down" - -required_result 1 "ERROR: public network interface $iface is down" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.007.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.007.sh deleted file mode 100755 index 004247ef67..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.007.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "unknown interface, up" - -setup_ctdb - -export CTDB_PUBLIC_INTERFACE="dev999" - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.008.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.008.sh deleted file mode 100755 index aa59151c2a..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.008.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "unknown interface, down, up" - -setup_ctdb - -iface="dev999" -export CTDB_PUBLIC_INTERFACE="$iface" - -#EVENTSCRIPTS_TESTS_TRACE="sh -x" -iterate_test 3 "ok_null" \ - 1 'ethtool_interfaces_down "$iface" ; required_result 1 "ERROR: No link on the public network interface $iface"' \ - 2 'ethtool_interfaces_up "$iface"' diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.009.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.009.sh deleted file mode 100755 index ab890f3442..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.009.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 down" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" - -ethtool_interfaces_down "$iface" - -ok "ERROR: No link on the public network interface $iface" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.010.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.010.sh deleted file mode 100755 index 476d2c9f69..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.010.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, all down" - -setup_ctdb - -ifaces=$(ctdb_get_interfaces) - -export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" - -ethtool_interfaces_down $ifaces - -msg=$(for i in $ifaces ; do echo "ERROR: No link on the public network interface $i" ; done) - -required_result 1 "$msg" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.011.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.011.sh deleted file mode 100755 index 3e5ba34ace..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.011.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -setup_bond $iface "None" - -export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" - -ethtool_interfaces_down "$iface" - -ok "ERROR: No active slaves for bond device $iface" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.012.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.012.sh deleted file mode 100755 index d27c423e44..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.012.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "CTDB_PARTIALLY_ONLINE_INTERFACES, 1 bond down" - -setup_ctdb - -ifaces=$(ctdb_get_interfaces) - -for i in $ifaces ; do - setup_bond $i "None" -done - -export CTDB_PARTIALLY_ONLINE_INTERFACES="yes" - -ethtool_interfaces_down $ifaces - -msg=$(for i in $ifaces ; do echo "ERROR: No active slaves for bond device $i" ; done) - -required_result 1 "$msg" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.013.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.013.sh deleted file mode 100755 index 88e884dc84..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.013.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "1 bond, active slaves, link down" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -setup_bond $iface "" "up" "down" - -required_result 1 "ERROR: No active slaves for 802.ad bond device $iface" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.014.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.014.sh deleted file mode 100755 index 97cf253c3b..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.014.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "spurious addresses on interface, no action" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -ip addr add 192.168.253.253/24 dev $iface -ip addr add 192.168.254.254/24 dev $iface - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh deleted file mode 100755 index 6810733b54..0000000000 --- a/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "spurious addresses on interface, delete them" - -setup_ctdb - -iface=$(ctdb_get_1_interface) - -ip addr add 192.168.253.253/24 dev $iface -ip addr add 192.168.254.254/24 dev $iface - -export CTDB_DELETE_UNEXPECTED_IPS="yes" - -ok <"$rpc_fail_limits_file" - -services="knfsd|mountd|rquotad|lockd|statd" - -echo "Doing rough check of file format..." - -! grep -v -E "^(${services}) " "$rpc_fail_limits_file" diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.101.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.101.sh deleted file mode 100755 index 657db97a07..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.101.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "all services available" - -setup_nfs - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.111.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.111.sh deleted file mode 100755 index 8dcde029dd..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.111.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "knfsd down, 1 iteration" - -setup_nfs -rpc_services_down "nfs" - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.112.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.112.sh deleted file mode 100755 index de796ebd07..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.112.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "knfsd down, 6 iterations" - -# knfsd fails and attempts to restart it fail. - -setup_nfs -rpc_services_down "nfs" - -iterate_test 6 'ok_null' \ - 2 'rpc_set_service_failure_response "knfsd"' \ - 4 'rpc_set_service_failure_response "knfsd"' \ - 6 'rpc_set_service_failure_response "knfsd"' diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.121.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.121.sh deleted file mode 100755 index 062c3f6f5c..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.121.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "lockd down, 15 iterations" - -# This simulates an ongoing failure in the eventscript's automated -# attempts to restart the service. That is, the eventscript is unable -# to restart the service. - -setup_nfs -rpc_services_down "nlockmgr" - -#EVENTSCRIPTS_TESTS_TRACE="sh -x" -iterate_test 15 "ok_null" \ - 10 "rpc_set_service_failure_response 'lockd'" \ - 15 "rpc_set_service_failure_response 'lockd'" diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.122.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.122.sh deleted file mode 100755 index b93dba7b97..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.122.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "lockd down, 15 iterations, back up after 10" - -# This simulates a success the eventscript's automated attempts to -# restart the service. - -setup_nfs -rpc_services_down "nlockmgr" - -# Iteration 10 should try to restart rpc.lockd. However, our test -# stub rpc.lockd does nothing, so we have to explicitly flag it as up. - -iterate_test 15 "ok_null" \ - 10 "rpc_set_service_failure_response 'lockd'" \ - 11 "rpc_services_up nlockmgr" - diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.131.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.131.sh deleted file mode 100755 index 84d20b761c..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.131.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "rquotad down, 5 iterations" - -setup_nfs -rpc_services_down "rquotad" - -iterate_test 5 'rpc_set_service_failure_response "rquotad"' diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.132.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.132.sh deleted file mode 100755 index fad9a4c590..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.132.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "rquotad down, 5 iterations, back up after 1" - -# rquotad fails once but then comes back of its own accord after 1 -# failure. - -setup_nfs -rpc_services_down "rquotad" - -iterate_test 5 'ok_null' \ - 1 'rpc_set_service_failure_response "rquotad"' \ - 2 'rpc_services_up "rquotad"' diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.141.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.141.sh deleted file mode 100755 index 377de6e229..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.141.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "statd down, 6 iterations" - -# statd fails and attempts to restart it fail. - -setup_nfs -rpc_services_down "status" - -iterate_test 6 'ok_null' \ - 2 'rpc_set_service_failure_response "statd"' \ - 4 'rpc_set_service_failure_response "statd"' \ - 6 'rpc_set_service_failure_response "statd"' diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.142.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.142.sh deleted file mode 100755 index 3ca3cf64a7..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.142.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "statd down, 8 iterations, back up after 2" - -# statd fails and the first attempt to restart it succeeds. - -setup_nfs -rpc_services_down "status" - -iterate_test 8 'ok_null' \ - 2 'rpc_set_service_failure_response "statd"' \ - 3 'rpc_services_up "status"' diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.151.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.151.sh deleted file mode 100755 index af2dd26f5d..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.151.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "mountd down, 1 iteration" - -setup_nfs -rpc_services_down "mountd" - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.152.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.152.sh deleted file mode 100755 index 9aad819b74..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.152.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "mountd down, 10 iterations" - -# This simulates an ongoing failure in the eventscript's automated -# attempts to restart the service. That is, the eventscript is unable -# to restart the service. - -setup_nfs -rpc_services_down "mountd" - -iterate_test 10 "ok_null" \ - 5 "rpc_set_service_failure_response 'mountd'" \ - 10 "rpc_set_service_failure_response 'mountd'" - -#export FAKE_NETSTAT_TCP_ESTABLISHED="10.0.0.1:2049|10.254.254.1:12301 10.0.0.1:2049|10.254.254.1:12302 10.0.0.1:2049|10.254.254.1:12303 10.0.0.1:2049|10.254.254.2:12304 10.0.0.1:2049|10.254.254.2:12305" diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.153.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.153.sh deleted file mode 100755 index 6b2750c82c..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.153.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "mountd down, 10 iterations, back up after 5" - -setup_nfs -rpc_services_down "mountd" - -# Iteration 5 should try to restart rpc.mountd. However, our test -# stub rpc.mountd does nothing, so we have to explicitly flag it as -# up. -iterate_test 10 "ok_null" \ - 5 "rpc_set_service_failure_response 'mountd'" \ - 6 "rpc_services_up mountd" diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh deleted file mode 100755 index 4abe68eaba..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.161.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "2nd share missing" - -setup_nfs - -shares_missing "ERROR: nfs directory \"%s\" not available" 2 - -required_result 1 "$MISSING_SHARES_TEXT" - -simple_test diff --git a/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh b/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh deleted file mode 100755 index 865b1c635a..0000000000 --- a/ctdb/tests/eventscripts/simple/60.nfs.monitor.162.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -. "${EVENTSCRIPTS_TESTS_DIR}/common.sh" - -define_test "2nd share missing, skipping share checks" - -setup_nfs -export CTDB_NFS_SKIP_SHARE_CHECK="yes" - -shares_missing "ERROR: nfs directory \"%s\" not available" 2 - -ok_null - -simple_test diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb index 81d275af70..e43784e3bb 100755 --- a/ctdb/tests/eventscripts/stubs/ctdb +++ b/ctdb/tests/eventscripts/stubs/ctdb @@ -126,7 +126,7 @@ ip_reallocate () fi _flags="${_flags}${_flags:+,}${_this}" done - "$(dirname ${EVENTSCRIPTS_TESTS_DIR})/bin/ctdb_takeover_tests" \ + "$(dirname ${TEST_SUBDIR})/bin/ctdb_takeover_tests" \ "ctdb_takeover_run_core" "$_flags" <"$FAKE_CTDB_IP_LAYOUT" | sort >"$_t" mv "$_t" "$FAKE_CTDB_IP_LAYOUT" -- cgit From feff13ff6328b85fae1581c53ba4b0d617b799c2 Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Mon, 16 Apr 2012 16:27:20 +1000 Subject: tests - run_tests needs to expand directories like "." and "..". Signed-off-by: Martin Schwenke (This used to be ctdb commit fd08fc6c88cb80190ce87325867de0391cf1af51) --- ctdb/tests/scripts/run_tests | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests index 8fce3ae978..5e2de17c80 100755 --- a/ctdb/tests/scripts/run_tests +++ b/ctdb/tests/scripts/run_tests @@ -148,6 +148,10 @@ run_one_test () for f ; do if [ -d "$f" ] ; then + # This expands the most probable problem cases like "." and "..". + if [ $(dirname "$f") = "." ] ; then + f=$(cd "$f" ; pwd) + fi for i in $(ls "${f%/}/"*".sh" 2>/dev/null) ; do run_one_test "$i" done -- cgit From aaa6565977e5f5a1ff6c908ab57e16f8978e044e Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Tue, 17 Apr 2012 12:57:46 +1000 Subject: tests/eventscripts: Tweak an error message in a policy routing test It currently has my home directory pasted into it. That can't be right... ;-) Signed-off-by: Martin Schwenke (This used to be ctdb commit 5d6e71cf6e1e0665ec96e7e4e2cfd31f40ebabae) --- ctdb/tests/eventscripts/13.per_ip_routing.002.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctdb/tests/eventscripts/13.per_ip_routing.002.sh b/ctdb/tests/eventscripts/13.per_ip_routing.002.sh index c54691fafc..3198ef36ed 100755 --- a/ctdb/tests/eventscripts/13.per_ip_routing.002.sh +++ b/ctdb/tests/eventscripts/13.per_ip_routing.002.sh @@ -8,7 +8,7 @@ setup_ctdb setup_ctdb_policy_routing required_result 1 < Date: Wed, 18 Apr 2012 10:35:25 +1000 Subject: tests/eventscripts: Share directories must be absolute in eventscript tests This fixes eventscripts/scripts/local.sh:setup_generic() so that directories listed in $FAKE_SHARES are absolute instead of potentially relative (and, therefore, ignored by the NFS share check). Signed-off-by: Martin Schwenke (This used to be ctdb commit 8544162b7a4efa23c92bc0d2774df95bb7f0f463) --- ctdb/tests/eventscripts/scripts/local.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index cba552e92f..2ee8fd20f0 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -67,6 +67,11 @@ setup_generic () for i in $(seq 1 3) ; do _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing" mkdir -p "$_s" + # Shares must begin with / + case "$_s" in + /*) : ;; + *) _s="${PWD}/$_s" + esac FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}" done -- cgit From 116f19b808ace4a74bcf56a638923dcd1e1e975d Mon Sep 17 00:00:00 2001 From: Martin Schwenke Date: Wed, 18 Apr 2012 10:37:45 +1000 Subject: tests: More unit test factoring/rationalisation and bug fixes Move some options from eventscripts/run_tests.sh to scripts/run_tests. Remove the former. Move some functions from eventscripts/scripts/local.sh to scripts/unit.sh. Both of these are modified during move so they are no longer eventscript-specific. Tweak */local.sh so that the new functions in unit.sh are used. Signed-off-by: Martin Schwenke (This used to be ctdb commit 7ff485687891732074c9fc9998502ca197663d02) --- .../eventscripts/etc-ctdb/interface_modify.sh | 1 - ctdb/tests/eventscripts/run_tests.sh | 53 -------- ctdb/tests/eventscripts/scripts/local.sh | 136 ++------------------- ctdb/tests/onnode/scripts/local.sh | 35 +++--- ctdb/tests/scripts/run_tests | 17 ++- ctdb/tests/scripts/unit.sh | 127 ++++++++++++++++++- ctdb/tests/takeover/scripts/local.sh | 32 +---- ctdb/tests/tool/scripts/local.sh | 38 +----- 8 files changed, 173 insertions(+), 266 deletions(-) delete mode 120000 ctdb/tests/eventscripts/etc-ctdb/interface_modify.sh delete mode 100755 ctdb/tests/eventscripts/run_tests.sh diff --git a/ctdb/tests/eventscripts/etc-ctdb/interface_modify.sh b/ctdb/tests/eventscripts/etc-ctdb/interface_modify.sh deleted file mode 120000 index 94f555c30d..0000000000 --- a/ctdb/tests/eventscripts/etc-ctdb/interface_modify.sh +++ /dev/null @@ -1 +0,0 @@ -../../../config/interface_modify.sh \ No newline at end of file diff --git a/ctdb/tests/eventscripts/run_tests.sh b/ctdb/tests/eventscripts/run_tests.sh deleted file mode 100755 index 406689a0d6..0000000000 --- a/ctdb/tests/eventscripts/run_tests.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh - -# Eventscript unit test harness. - -cd $(dirname "$0") -export EVENTSCRIPTS_TESTS_DIR=$(pwd) - -test_dir=$(dirname "$EVENTSCRIPTS_TESTS_DIR") - -export EVENTSCRIPT_TESTS_CAT_RESULTS_OPTS="" -export EVENTSCRIPT_TESTS_DIFF_RESULTS=false - -opts="-d" - -for i ; do - case "$i" in - -v) - export EVENTSCRIPT_TESTS_VERBOSE="yes" - shift - ;; - -T) - # This will cause tests to fail but is good for debugging - # individual tests when they fail. - export EVENTSCRIPTS_TESTS_TRACE="sh -x" - shift - ;; - -A) - # Useful for detecting whitespace differences in results - export EVENTSCRIPT_TESTS_CAT_RESULTS_OPTS="-A" - shift - ;; - -D) - # Useful for detecting whitespace differences in results - export EVENTSCRIPT_TESTS_DIFF_RESULTS=true - shift - ;; - -*) - opts="$opts $i" - shift - ;; - *) - break - esac -done - -tests="" -if [ -z "$*" ] ; then - tests=$(ls simple/[0-9][0-9].*.*.[0-9][0-9][0-9].sh simple/[0-9][0-9].*.*.[0-9][0-9][0-9]/run_test.sh 2>/dev/null) -fi - -"$test_dir/scripts/run_tests" $opts "$@" $tests || exit 1 - -exit 0 diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh index 2ee8fd20f0..6d7de89465 100644 --- a/ctdb/tests/eventscripts/scripts/local.sh +++ b/ctdb/tests/eventscripts/scripts/local.sh @@ -38,7 +38,7 @@ export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb" ###################################################################### -if [ "$TEST_VERBOSE" = "yes" ] ; then +if "$TEST_VERBOSE" ; then debug () { echo "$@" ; } else debug () { : ; } @@ -672,133 +672,13 @@ define_test () printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" } -# Set the required result for a test. -# - Argument 1 is exit code. -# - Argument 2, if present is the required test output but "--" -# indicates empty output. -# If argument 2 is not present or null then read required test output -# from stdin. -required_result () +_extra_header () { - required_rc="${1:-0}" - if [ -n "$2" ] ; then - if [ "$2" = "--" ] ; then - required_output="" - else - required_output="$2" - fi - else - if ! tty -s ; then - required_output=$(cat) - else - required_output="" - fi - fi -} - -ok () -{ - required_result 0 "$@" -} - -ok_null () -{ - ok -- -} - -result_print () -{ - _passed="$1" - _out="$2" - _rc="$3" - _iteration="$4" - - if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then - if [ -n "$_iteration" ] ; then - cat <"$_outr" - - _outf=$(mktemp) - echo "$_out" >"$_outf" - - cat <&1) + _out=$($TEST_COMMAND_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1) - result_check + result_check "$_extra_header" } simple_test_event () @@ -912,8 +794,8 @@ iterate_test () _result=false fi - result_print "$_passed" "$_out" "$_rc" "$iteration" + result_print "$_passed" "$_out" "$_rc" "Iteration $iteration" done - result_footer "$_result" + result_footer "$_result" "$(_extra_header)" } diff --git a/ctdb/tests/onnode/scripts/local.sh b/ctdb/tests/onnode/scripts/local.sh index a990c695f8..d6dd41c293 100644 --- a/ctdb/tests/onnode/scripts/local.sh +++ b/ctdb/tests/onnode/scripts/local.sh @@ -49,10 +49,14 @@ ctdb_set_output () trap "rm -f $_out $_rc" 0 } -required_result () +_extra_header () { - required_rc="${1:-0}" - required_output=$(cat) + cat <&1) _rc=$? _out=$(echo "$_out" | $_sort ) - if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then - echo "PASSED" - else - cat <"$_outr" + + _outf=$(mktemp) + echo "$_out" >"$_outf" + + cat <&1) - _rc=$? - + # Do some filtering of the output to replace date/time. if [ "$algorithm" = "lcp2" -a -n "$CTDB_TEST_LOGLEVEL" ] ; then - OUT_FILTER='s@^.*:@DATE TIME \[PID\]:@' - fi - - if [ -n "$OUT_FILTER" ] ; then - _fout=$(echo "$_out" | sed -r "$OUT_FILTER") - else - _fout="$_out" + OUT_FILTER='s@^.*:@DATE\ TIME\ \[PID\]:@' fi - if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then - echo "PASSED" - else - cat <&1) -################################################## -Required output (Exit status: ${required_rc}): -################################################## -$required_output -################################################## -Actual output (Exit status: ${_rc}): -################################################## -$_out -EOF - return 1 - fi + result_check "Algorithm: $algorithm" } diff --git a/ctdb/tests/tool/scripts/local.sh b/ctdb/tests/tool/scripts/local.sh index fe8416ee98..1105628a4b 100644 --- a/ctdb/tests/tool/scripts/local.sh +++ b/ctdb/tests/tool/scripts/local.sh @@ -1,8 +1,5 @@ # Hey Emacs, this is a -*- shell-script -*- !!! :-) -# Print a message and exit. -die () { echo "$@" >&2 ; exit 1 ; } - test_bin="$(dirname ${TEST_SUBDIR})/bin" define_test () @@ -29,42 +26,13 @@ define_test () simple_test () { - _out=$($test_prog "$@" 2>&1) - _rc=$? - # Most of the tests when the tool fails will have a date/time/pid # prefix. Strip that because it isn't possible to match it. if [ $required_rc -ne 0 ] ; then - OUT_FILTER='s@^[0-9/]+ [0-9:\.]+ \[[ 0-9]+\]:@DATE TIME \[PID\]:@' + OUT_FILTER='s@^[0-9/]+\ [0-9:\.]+\ \[[\ 0-9]+\]:@DATE\ TIME\ \[PID\]:@' fi - if [ -n "$OUT_FILTER" ] ; then - _fout=$(echo "$_out" | sed -r "$OUT_FILTER") - else - _fout="$_out" - fi + _out=$($test_prog "$@" 2>&1) - if [ "$_fout" = "$required_output" -a $_rc = $required_rc ] ; then - if [ "$TEST_VERBOSE" = "yes" ] ; then - cat <