diff options
Diffstat (limited to 'ctdb/tests/eventscripts/common.sh')
| -rw-r--r-- | ctdb/tests/eventscripts/common.sh | 779 |
1 files changed, 779 insertions, 0 deletions
diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh new file mode 100644 index 0000000000..ab7abb9fac --- /dev/null +++ b/ctdb/tests/eventscripts/common.sh @@ -0,0 +1,779 @@ +# Hey Emacs, this is a -*- shell-script -*- !!! :-) + +# Augment PATH with relevant stubs/ directories. We do this by actually +# setting PATH, and also by setting $EVENTSCRIPTS_PATH and then +# prepending that to $PATH in rc.local to avoid the PATH reset in +# functions. + +EVENTSCRIPTS_PATH="" + +if [ -d "${EVENTSCRIPTS_TESTS_DIR}/stubs" ] ; then + EVENTSCRIPTS_PATH="${EVENTSCRIPTS_TESTS_DIR}/stubs" +fi + +export EVENTSCRIPTS_TESTCASE_DIR=$(dirname "$0") +if [ $(basename "$EVENTSCRIPTS_TESTCASE_DIR") = "eventscripts" ] ; then + # Just a test script, no testcase subdirectory. + EVENTSCRIPTS_TESTCASE_DIR="$EVENTSCRIPTS_TESTS_DIR" +else + if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/stubs" ] ; then + EVENTSCRIPTS_PATH="${EVENTSCRIPTS_TESTCASE_DIR}/stubs:${EVENTSCRIPTS_PATH}" + fi +fi + +export EVENTSCRIPTS_PATH + +PATH="${EVENTSCRIPTS_PATH}:${PATH}" + +if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc" ] ; then + CTDB_ETCDIR="${EVENTSCRIPTS_TESTCASE_DIR}/etc" +elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc" ] ; then + CTDB_ETCDIR="${EVENTSCRIPTS_TESTS_DIR}/etc" +else + echo "Unable to set \$CTDB_ETCDIR" >&2 + exit 1 +fi +export CTDB_ETCDIR + +if [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc-ctdb" ] ; then + CTDB_BASE="${EVENTSCRIPTS_TESTCASE_DIR}/etc-ctdb" +elif [ -d "${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" ] ; then + CTDB_BASE="${EVENTSCRIPTS_TESTCASE_DIR}/etc/ctdb" +elif [ -d "${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" ] ; then + CTDB_BASE="${EVENTSCRIPTS_TESTS_DIR}/etc-ctdb" +else + echo "Unable to set \$CTDB_BASE" >&2 + exit 1 +fi +export CTDB_BASE + +export EVENTSCRIPTS_TESTS_VAR_DIR="${EVENTSCRIPTS_TESTS_DIR}/var" +if [ "$EVENTSCRIPTS_TESTS_VAR_DIR" != "/var" ] ; then + rm -r "$EVENTSCRIPTS_TESTS_VAR_DIR" +fi +mkdir -p "$EVENTSCRIPTS_TESTS_VAR_DIR" +export CTDB_VARDIR="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb" + +###################################################################### + +if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] ; then + debug () { echo "$@" ; } +else + debug () { : ; } +fi + +eventscripts_tests_cleanup_hooks="" + +# This loses quoting! +eventscripts_test_add_cleanup () +{ + eventscripts_tests_cleanup_hooks="${eventscripts_tests_cleanup_hooks}${eventscripts_tests_cleanup_hooks:+ ; }$*" +} + +trap 'eval $eventscripts_tests_cleanup_hooks' 0 + + +###################################################################### + +# General setup fakery + +setup_generic () +{ + debug "Setting up shares (3 existing shares)" + # Create 3 fake shares/exports. + export FAKE_SHARES="" + for i in $(seq 1 3) ; do + _s="${EVENTSCRIPTS_TESTS_VAR_DIR}/shares/${i}_existing" + mkdir -p "$_s" + FAKE_SHARES="${FAKE_SHARES}${FAKE_SHARES:+ }${_s}" + done + + export FAKE_PROC_NET_BONDING="$EVENTSCRIPTS_TESTS_VAR_DIR/proc-net-bonding" + mkdir -p "$FAKE_PROC_NET_BONDING" + rm -f "$FAKE_PROC_NET_BONDING"/* + + export FAKE_ETHTOOL_LINK_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/ethtool-link-down" + mkdir -p "$FAKE_ETHTOOL_LINK_DOWN" + rm -f "$FAKE_ETHTOOL_LINK_DOWN"/* + + # This can only have 2 levels. We don't want to resort to usings + # something dangerous like "rm -r" setup time. + export FAKE_IP_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ip-state" + mkdir -p "$FAKE_IP_STATE" + rm -f "$FAKE_IP_STATE"/*/* + rm -f "$FAKE_IP_STATE"/* 2>/dev/null || true + rmdir "$FAKE_IP_STATE"/* 2>/dev/null || true +} + +tcp_port_down () +{ + for _i ; do + debug "Marking TCP port \"${_i}\" as not listening" + FAKE_NETSTAT_TCP_LISTEN=$(echo "$FAKE_NETSTAT_TCP_LISTEN" | sed -r -e "s@[[:space:]]*[\.0-9]+:${_i}@@g") + done +} + +shares_missing () +{ + _fmt="$1" ; shift + + # Replace some shares with non-existent ones. + _t="" + _n=1 + _nl=" +" + export MISSING_SHARES_TEXT="" + for _i in $FAKE_SHARES ; do + if [ $_n = "$1" ] ; then + shift + _i="${_i%_existing}_missing" + debug "Replacing share $_n with missing share \"$_i\"" + rmdir "$_i" 2>/dev/null || true + MISSING_SHARES_TEXT="${MISSING_SHARES_TEXT}${MISSING_SHARES_TEXT:+${_nl}}"$(printf "$_fmt" "${_i}") + fi + _t="${_t}${_t:+ }${_i}" + _n=$(($_n + 1)) + done + FAKE_SHARES="$_t" +} + +# Setup some fake /proc/net/bonding files with just enough info for +# the eventscripts. + +# arg1 is interface name, arg2 is currently active slave (use "None" +# if none), arg3 is MII status ("up" or "down"). +setup_bond () +{ + _iface="$1" + _slave="${2:-${_iface}_sl_0}" + _mii_s="${3:-up}" + _mii_subs="${4:-${_mii_s:-up}}" + echo "Setting $_iface to be a bond with active slave $_slave and MII status $_mii_s" + cat >"${FAKE_PROC_NET_BONDING}/$_iface" <<EOF +Bonding Mode: IEEE 802.3ad Dynamic link aggregation +Currently Active Slave: $_slave +# Status of the bond +MII Status: $_mii_s +# Status of 1st pretend adapter +MII Status: $_mii_subs +# Status of 2nd pretend adapter +MII Status: $_mii_subs +EOF +} + +ethtool_interfaces_down () +{ + for _i ; do + echo "Marking interface $_i DOWN for ethtool" + touch "${FAKE_ETHTOOL_LINK_DOWN}/${_i}" + done +} + +ethtool_interfaces_up () +{ + for _i ; do + echo "Marking interface $_i UP for ethtool" + rm -f "${FAKE_ETHTOOL_LINK_DOWN}/${_i}" + done +} + +###################################################################### + +# CTDB fakery + +# Evaluate an expression that probably calls functions or uses +# variables from the CTDB functions file. This is used for test +# initialisation. +eventscript_call () +{ + ( + . "$CTDB_BASE/functions" + "$@" + ) +} + +# Set output for ctdb command. Option 1st argument is return code. +ctdb_set_output () +{ + _out="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.out" + cat >"$_out" + + _rc="$EVENTSCRIPTS_TESTS_VAR_DIR/ctdb.rc" + echo "${1:-0}" >"$_rc" + + eventscripts_test_add_cleanup "rm -f $_out $_rc" +} + +setup_ctdb () +{ + setup_generic + + export FAKE_CTDB_NUMNODES="${1:-3}" + echo "Setting up CTDB with ${FAKE_CTDB_NUMNODES} fake nodes" + + export FAKE_CTDB_PNN="${2:-0}" + echo "Setting up CTDB with PNN ${FAKE_CTDB_PNN}" + + if [ -n "$3" ] ; then + echo "Setting up CTDB_PUBLIC_ADDRESSES: $3" + export CTDB_PUBLIC_ADDRESSES=$(mktemp) + for _i in $3 ; do + _ip="${_i%@*}" + _ifaces="${_i#*@}" + echo "${_ip} ${_ifaces}" >>"$CTDB_PUBLIC_ADDRESSES" + done + eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES" + fi + + export FAKE_CTDB_IFACES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb/ifaces-down" + mkdir -p "$FAKE_CTDB_IFACES_DOWN" + rm -f "$FAKE_CTDB_IFACES_DOWN"/* + + export FAKE_CTDB_NODES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/nodes-down" + mkdir -p "$FAKE_CTDB_NODES_DOWN" + rm -f "$FAKE_CTDB_NODES_DOWN"/* +} + + + +ctdb_nodes_up () +{ + for _i ; do + rm -f "${FAKE_CTDB_NODES_DOWN}/${_i}" + done +} + +ctdb_nodes_down () +{ + for _i ; do + touch "${FAKE_CTDB_NODES_DOWN}/${_i}" + done +} + +ctdb_get_interfaces () +{ + # The echo/subshell forces all the output onto 1 line. + echo $(ctdb ifaces -Y | awk -F: 'FNR > 1 {print $2}') +} + +ctdb_get_1_interface () +{ + _t=$(ctdb_get_interfaces) + echo ${_t%% *} +} + +ctdb_get_public_addresses () +{ + _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}" + echo $(if [ -r "$_f" ] ; then + while read _ip _iface ; do + echo "${_ip}@${_iface}" + done <"$_f" + fi) +} + +# Prints the 1st public address as: interface IP maskbits +# This is suitable for passing to 10.interfaces takeip/releaseip +ctdb_get_1_public_address () +{ + _addrs=$(ctdb_get_public_addresses) + echo "${_addrs%% *}" | sed -r -e 's#(.*)/(.*)@(.*)#\3 \1 \2#g' +} + +###################################################################### + +# Samba fakery + +setup_samba () +{ + setup_ctdb + + if [ "$1" != "down" ] ; then + + debug "Marking Samba services as up, listening and managed by CTDB" + # Get into known state. + for i in "samba" "winbind" ; do + eventscript_call ctdb_service_managed "$i" + done + # All possible service names for all known distros. + for i in "smb" "nmb" "winbind" "samba" ; do + service "$i" force-started + done + + export CTDB_SAMBA_SKIP_SHARE_CHECK="no" + export CTDB_MANAGED_SERVICES="foo samba winbind bar" + + export FAKE_NETSTAT_TCP_LISTEN="0.0.0.0:445 0.0.0.0:139" + export FAKE_WBINFO_FAIL="no" + else + debug "Marking Samba services as down, not listening and not managed by CTDB" + # Get into known state. + for i in "samba" "winbind" ; do + eventscript_call ctdb_service_unmanaged "$i" + done + # All possible service names for all known distros. + for i in "smb" "nmb" "winbind" "samba" ; do + service "$i" force-stopped + done + + export CTDB_SAMBA_SKIP_SHARE_CHECK="no" + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_SAMBA + unset CTDB_MANAGES_WINBIND + + export FAKE_NETSTAT_TCP_LISTEN="" + export FAKE_WBINFO_FAIL="yes" + fi + + # This is ugly but if this file isn't removed before each test + # then configuration changes between tests don't stick. + rm -f "$CTDB_VARDIR/state/samba/smb.conf.cache" +} + +wbinfo_down () +{ + debug "Making wbinfo commands fail" + FAKE_WBINFO_FAIL="yes" +} + +###################################################################### + +# NFS fakery + +setup_nfs () +{ + setup_ctdb + + export FAKE_RPCINFO_SERVICES="" + + export CTDB_NFS_SKIP_SHARE_CHECK="no" + export CTDB_NFS_SKIP_KNFSD_ALIVE_CHECK="no" + + # Reset the failcounts for nfs services. + eventscript_call eval rm -f '$ctdb_fail_dir/nfs_*' + + rpc_fail_limits_file="${EVENTSCRIPTS_TESTS_VAR_DIR}/rpc_fail_limits" + + # Force this file to exist so tests can be individually run. + if [ ! -f "$rpc_fail_limits_file" ] ; then + # This is gross... but is needed to fake through the nfs monitor event. + eventscript_call ctdb_service_managed "nfs" + service "nfs" force-started # might not be enough + CTDB_RC_LOCAL="$CTDB_BASE/rc.local.nfs.monitor.get-limits" \ + CTDB_MANAGES_NFS="yes" \ + "${CTDB_BASE}/events.d/60.nfs" "monitor" >"$rpc_fail_limits_file" + fi + + if [ "$1" != "down" ] ; then + debug "Setting up NFS environment: all RPC services up, NFS managed by CTDB" + + eventscript_call ctdb_service_managed "nfs" + service "nfs" force-started # might not be enough + + export CTDB_MANAGED_SERVICES="foo nfs bar" + + rpc_services_up "nfs" "mountd" "rquotad" "nlockmgr" "status" + else + debug "Setting up NFS environment: all RPC services down, NFS not managed by CTDB" + + eventscript_call ctdb_service_unmanaged "nfs" + service "nfs" force-stopped # might not be enough + eventscript_call startstop_nfs stop + + export CTDB_MANAGED_SERVICES="foo bar" + unset CTDB_MANAGES_NFS + fi +} + +rpc_services_down () +{ + for _i ; do + debug "Marking RPC service \"${_i}\" as unavailable" + FAKE_RPCINFO_SERVICES=$(echo "$FAKE_RPCINFO_SERVICES" | sed -r -e "s@[[:space:]]*${_i}:[0-9]+:[0-9]+@@g") + done +} + +rpc_services_up () +{ + for _i ; do + debug "Marking RPC service \"${_i}\" as available" + case "$_i" in + nfs) _t="2:3" ;; + mountd) _t="1:3" ;; + rquotad) _t="1:2" ;; + nlockmgr) _t="3:4" ;; + status) _t="1:1" ;; + *) + echo "Internal error - unsupported RPC service \"${_i}\"" + exit 1 + esac + + FAKE_RPCINFO_SERVICES="${FAKE_RPCINFO_SERVICES}${FAKE_RPCINFO_SERVICES:+ }${_i}:${_t}" + done +} + +# Set the required result for a particular RPC program having failed +# for a certain number of iterations. This is probably still a work +# in progress. Note that we could hook aggressively +# nfs_check_rpc_service() to try to implement this but we're better +# off testing nfs_check_rpc_service() using independent code... even +# if it is incomplete and hacky. So, if the 60.nfs eventscript +# changes and the tests start to fail then it may be due to this +# function being incomplete. +rpc_set_service_failure_response () +{ + _progname="$1" + # The number of failures defaults to the iteration number. This + # will be true when we fail from the 1st iteration... but we need + # the flexibility to set the number of failures. + _numfails="${2:-${iteration}}" + + _c="${CTDB_ETCDIR}/sysconfig/nfs" + if [ -r "$_c" ] ; then + . "$_c" + fi + + # A handy newline. :-) + _nl=" +" + + # Default + ok_null + + _ts=$(sed -n -e "s@^${_progname} @@p" "$rpc_fail_limits_file") + + while [ -n "$_ts" ] ; do + # Get the triple: operator, fail limit and actions. + _op="${_ts%% *}" ; _ts="${_ts#* }" + _li="${_ts%% *}" ; _ts="${_ts#* }" + # We've lost some of the quoting but we can simulate + # because we know an operator is always the first in a + # triple. + _actions="" + while [ -n "$_ts" ] ; do + # If this is an operator then we've got all of the + # actions. + case "$_ts" in + -*) break ;; + esac + + _actions="${_actions}${_actions:+ }${_ts%% *}" + # Special case for end of list. + if [ "$_ts" != "${_ts#* }" ] ; then + _ts="${_ts#* }" + else + _ts="" + fi + done + + if [ "$_numfails" "$_op" "$_li" ] ; then + _out="" + _rc=0 + for _action in $_actions ; do + case "$_action" in + verbose) + _ver=1 + _pn="$_progname" + case "$_progname" in + knfsd) _ver=3 ; _pn="nfs" ;; + lockd) _ver=4 ; _pn="nlockmgr" ;; + statd) _pn="status" ;; + esac + _out="\ +ERROR: $_pn failed RPC check: +rpcinfo: RPC: Program not registered +program $_pn version $_ver is not available" + ;; + restart|restart:*) + _opts="" + _p="rpc.${_progname}" + case "$_progname" in + statd) + _opts="${STATD_HOSTNAME:+ -n }${STATD_HOSTNAME}" + ;; + esac + case "${_progname}${_action#restart}" in + knfsd) + _t="\ +Trying to restart NFS service +Starting nfslock: OK +Starting nfs: OK" + ;; + knfsd:bs) + _t="Trying to restart NFS service" + ;; + lockd) + _t="\ +Trying to restart lock manager service +Stopping nfslock: OK +Starting nfslock: OK" + ;; + lockd:*) + _t="Trying to restart lock manager service" + ;; + *) + _t="Trying to restart $_progname [${_p}${_opts}]" + esac + _out="${_out}${_out:+${_nl}}${_t}" + ;; + unhealthy) + _rc=1 + esac + done + required_result $_rc "$_out" + return + fi + done +} + +###################################################################### + +# VSFTPD fakery + +setup_vsftpd () +{ + if [ "$1" != "down" ] ; then + echo "setup_vsftpd up not implemented!!!" + exit 1 + else + debug "Setting up VSFTPD environment: service down, not managed by CTDB" + + eventscript_call ctdb_service_unmanaged vsftpd + service vsftpd force-stopped + + export CTDB_MANAGED_SERVICES="foo" + unset CTDB_MANAGES_VSFTPD + fi +} + +###################################################################### + +# HTTPD fakery + +setup_httpd () +{ + if [ "$1" != "down" ] ; then + echo "setup_httpd up not implemented!!!" + exit 1 + else + debug "Setting up HTTPD environment: service down, not managed by CTDB" + + for i in "apache2" "httpd" ; do + eventscript_call ctdb_service_unmanaged "$i" + service "$i" force-stopped + done + + export CTDB_MANAGED_SERVICES="foo" + unset CTDB_MANAGES_HTTPD + fi +} + +###################################################################### + +# Result and test functions + +# Set some globals and print the summary. +define_test () +{ + desc="$1" + + _f="$0" + _f="${_f#./}" # strip leading ./ + _f="${_f#simple/}" # strip leading simple/ + _f="${_f%%/*}" # if subdir, strip off file + _f="${_f%.sh}" # strip off .sh suffix if any + + # Remaining format should be NN.service.event.NNN: + _num="${_f##*.}" + _f="${_f%.*}" + event="${_f##*.}" + script="${_f%.*}" + + printf "%-14s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc" +} + +# Set the required result for a test. +# - Argument 1 is exit code. +# - Argument 2, if present is the required test output but "--" +# indicates empty output. +# If argument 2 is not present or null then read required test output +# from stdin. +required_result () +{ + required_rc="${1:-0}" + if [ -n "$2" ] ; then + if [ "$2" = "--" ] ; then + required_output="" + else + required_output="$2" + fi + else + if ! tty -s ; then + required_output=$(cat) + else + required_output="" + fi + fi +} + +ok () +{ + required_result 0 "$@" +} + +ok_null () +{ + ok -- +} + +result_print () +{ + _passed="$1" + _out="$2" + _rc="$3" + _iteration="$4" + + if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then + if [ -n "$_iteration" ] ; then + cat <<EOF + +################################################## +################################################## +Iteration $_iteration +EOF + fi + +cat <<EOF +################################################## +Output (Exit status: ${_rc}): +################################################## +$_out +EOF + fi + + if ! $_passed ; then + cat <<EOF +################################################## +Required output (Exit status: ${required_rc}): +################################################## +$required_output +EOF + fi +} + +result_footer () +{ + _passed="$1" + + if [ "$EVENTSCRIPT_TESTS_VERBOSE" = "yes" ] || ! $_passed ; then + + cat <<EOF +################################################## +CTDB_BASE="$CTDB_BASE" +CTDB_ETCDIR="$CTDB_ETCDIR" +ctdb client is "$(which ctdb)" +################################################## +EOF + fi + + echo + + if $_passed ; then + echo "PASSED" + return 0 + else + echo "FAILED" + return 1 + fi +} + +# Run an eventscript once. The test passes if the return code and +# output match those required. + +# Any args are passed to the eventscript. + +# Eventscript tracing can be done by setting: +# EVENTSCRIPTS_TESTS_TRACE="sh -x" + +# or similar. This will almost certainly make a test fail but is +# useful for debugging. +simple_test () +{ + echo "Running \"${CTDB_BASE}/events.d/$script $event\"" + _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1) + _rc=$? + + if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then + _passed=true + else + _passed=false + fi + + result_print "$_passed" "$_out" "$_rc" + result_footer "$_passed" +} + +# Run an eventscript iteratively. +# - 1st argument is the number of iterations. +# - 2nd argument is something to eval to do setup for every iteration. +# The easiest thing to do here is to define a function and pass it +# here. +# - Subsequent arguments come in pairs: an iteration number and +# something to eval for that iteration. Each time an iteration +# number is matched the associated argument is given to eval after +# the default setup is done. The iteration numbers need to be given +# in ascending order. +# +# Some optional args can be given *before* these, surrounded by extra +# "--" args. These args are passed to the eventscript. Quoting is +# lost. +# +# One use of the 2nd and further arguments is to call +# required_result() to change what is expected of a particular +# iteration. +iterate_test () +{ + args="" + if [ "$1" = "--" ] ; then + shift + while [ "$1" != "--" ] ; do + args="${args}${args:+ }$1" + shift + done + shift + fi + + _repeats="$1" + _setup_default="$2" + shift 2 + + echo "Running $_repeats iterations of \"${CTDB_BASE}/events.d/$script $event\" $args" + + _result=true + + for iteration in $(seq 1 $_repeats) ; do + # This is inefficient because the iteration-specific setup + # might completely replace the default one. However, running + # the default is good because it allows you to revert to a + # particular result without needing to specify it explicitly. + eval $_setup_default + if [ $iteration = "$1" ] ; then + eval $2 + shift 2 + fi + + _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" $args 2>&1) + _rc=$? + + if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then + _passed=true + else + _passed=false + _result=false + fi + + result_print "$_passed" "$_out" "$_rc" "$iteration" + done + + result_footer "$_result" +} |