ctdb-tests: Add some tests for 11.natgw eventscript

This includes adding support for:

* Configuring fake NATGW state in the eventscript unit tests

* "natgwlist" and "setnatgwstate" in ctdb command stub

* ip command stub to default to "main table" when no table specified,
  allow routes to be added without "dev" option (just add a default
  dev), support "metric" option

Signed-off-by: Martin Schwenke <martin@meltin.net>

9 files changed, 376 insertions, 18 deletions

diff --git a/ctdb/tests/eventscripts/11.natgw.001.sh b/ctdb/tests/eventscripts/11.natgw.001.sh
new file mode 100755
index 00000000000..afcc0972ddf
--- /dev/null
+++ b/ctdb/tests/eventscripts/11.natgw.001.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "not configured"
+
+setup_ctdb
+
+ok_null
+simple_test_event "ipreallocate"
+
+check_routes 0
diff --git a/ctdb/tests/eventscripts/11.natgw.011.sh b/ctdb/tests/eventscripts/11.natgw.011.sh
new file mode 100755
index 00000000000..4c5a937d553
--- /dev/null
+++ b/ctdb/tests/eventscripts/11.natgw.011.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "master node, basic configuration"
+
+setup_ctdb
+
+setup_ctdb_natgw <<EOF
+192.168.1.21 master
+192.168.1.22
+192.168.1.23
+192.168.1.24
+EOF
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok "default via ${CTDB_NATGW_DEFAULT_GATEWAY} dev ethXXX  metric 10 "
+simple_test_command ip route show
+
+ok_natgw_master_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
diff --git a/ctdb/tests/eventscripts/11.natgw.012.sh b/ctdb/tests/eventscripts/11.natgw.012.sh
new file mode 100755
index 00000000000..6ec98a03f58
--- /dev/null
+++ b/ctdb/tests/eventscripts/11.natgw.012.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "slave node, basic configuration"
+
+setup_ctdb
+
+setup_ctdb_natgw <<EOF
+192.168.1.21
+192.168.1.22 master
+192.168.1.23
+192.168.1.24
+EOF
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok "default via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX  metric 10 "
+simple_test_command ip route show
+
+ok_natgw_slave_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
diff --git a/ctdb/tests/eventscripts/11.natgw.013.sh b/ctdb/tests/eventscripts/11.natgw.013.sh
new file mode 100755
index 00000000000..16f06221805
--- /dev/null
+++ b/ctdb/tests/eventscripts/11.natgw.013.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "master node, no gateway"
+
+setup_ctdb
+
+setup_ctdb_natgw <<EOF
+192.168.1.21 master
+192.168.1.22
+192.168.1.23
+192.168.1.24
+EOF
+
+CTDB_NATGW_DEFAULT_GATEWAY=""
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok_null
+simple_test_command ip route show
+
+ok_natgw_master_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
diff --git a/ctdb/tests/eventscripts/11.natgw.014.sh b/ctdb/tests/eventscripts/11.natgw.014.sh
new file mode 100755
index 00000000000..462756c062d
--- /dev/null
+++ b/ctdb/tests/eventscripts/11.natgw.014.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "slave node, no gateway"
+
+setup_ctdb
+
+setup_ctdb_natgw <<EOF
+192.168.1.21
+192.168.1.22 master
+192.168.1.23
+192.168.1.24
+EOF
+
+CTDB_NATGW_DEFAULT_GATEWAY=""
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok "default via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX  metric 10 "
+simple_test_command ip route show
+
+ok_natgw_slave_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
diff --git a/ctdb/tests/eventscripts/11.natgw.015.sh b/ctdb/tests/eventscripts/11.natgw.015.sh
new file mode 100755
index 00000000000..d202ae670bc
--- /dev/null
+++ b/ctdb/tests/eventscripts/11.natgw.015.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "basic configuration, multiple transitions"
+
+setup_ctdb
+
+echo "*** Master node..."
+
+setup_ctdb_natgw <<EOF
+192.168.1.21 master
+192.168.1.22
+192.168.1.23
+192.168.1.24
+EOF
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok "default via ${CTDB_NATGW_DEFAULT_GATEWAY} dev ethXXX  metric 10 "
+simple_test_command ip route show
+
+ok_natgw_master_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
+
+echo "*** Slave node..."
+
+setup_ctdb_natgw <<EOF
+192.168.1.21
+192.168.1.22 master
+192.168.1.23
+192.168.1.24
+EOF
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok "default via ${FAKE_CTDB_NATGW_MASTER} dev ethXXX  metric 10 "
+simple_test_command ip route show
+
+ok_natgw_slave_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
+
+echo "*** Master node again..."
+
+setup_ctdb_natgw <<EOF
+192.168.1.21 master
+192.168.1.22
+192.168.1.23
+192.168.1.24
+EOF
+
+ok_null
+simple_test_event "ipreallocated"
+
+ok "default via ${CTDB_NATGW_DEFAULT_GATEWAY} dev ethXXX  metric 10 "
+simple_test_command ip route show
+
+ok_natgw_master_ip_addr_show
+simple_test_command ip addr show "$CTDB_NATGW_PUBLIC_IFACE"
+
diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index 85894a41b82..898405ce2a5 100644
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -505,6 +505,64 @@ EOF
 
 ######################################################################
 
+setup_ctdb_natgw ()
+{
+    debug "Setting up NAT gateway"
+
+    natgw_config_dir="${TEST_VAR_DIR}/natgw_config"
+    mkdir -p "$natgw_config_dir"
+
+    # These will accumulate, 1 per test... but will be cleaned up at
+    # the end.
+    export CTDB_NATGW_NODES=$(mktemp --tmpdir="$natgw_config_dir")
+
+    # Read from stdin
+    while read _ip _master _dev ; do
+	echo "$_ip"
+	if [ "$_master" = "master" ] ; then
+	    export FAKE_CTDB_NATGW_MASTER="$_ip"
+	fi
+    done >"$CTDB_NATGW_NODES"
+
+    # Assume all of the nodes are on a /24 network and have IPv4
+    # addresses:
+    read _ip <"$CTDB_NATGW_NODES"
+    export CTDB_NATGW_PRIVATE_NETWORK="${_ip%.*}.0/24"
+
+    # These are fixed.  Probably don't use the same network for the
+    # private node IPs.  To unset the default gateway just set it to
+    # "".  :-)
+    export CTDB_NATGW_PUBLIC_IP="10.1.1.121/24"
+    export CTDB_NATGW_PUBLIC_IFACE="eth1"
+    export CTDB_NATGW_DEFAULT_GATEWAY="10.1.1.254"
+}
+
+ok_natgw_master_ip_addr_show ()
+{
+    _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
+
+    # This is based on CTDB_NATGW_PUBLIC_IP
+    _brd="10.1.1.255"
+
+ok <<EOF
+1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
+    link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
+    inet ${CTDB_NATGW_PUBLIC_IP} brd ${_brd} scope global ${CTDB_NATGW_PUBLIC_IFACE}
+       valid_lft forever preferred_lft forever
+EOF
+}
+
+ok_natgw_slave_ip_addr_show ()
+{
+    _mac=$(echo "$CTDB_NATGW_PUBLIC_IFACE" | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
+ok <<EOF
+1: ${CTDB_NATGW_PUBLIC_IFACE}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
+    link/ether ${_mac} brd ff:ff:ff:ff:ff:ff
+EOF
+}
+
+######################################################################
+
 # Samba/winbind fakery
 
 setup_samba ()
diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index da84ed7cdfd..51c17635fc0 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -2,6 +2,12 @@
 
 prog="ctdb"
 
+# Print a message and exit.
+die ()
+{
+    echo "$1" >&2 ; exit ${2:-1}
+}
+
 not_implemented_exit_code=1
 
 usage ()
@@ -231,6 +237,66 @@ ctdb_shutdown ()
 
 ######################################################################
 
+FAKE_CTDB_NATGW_STATE="${FAKE_CTDB_STATE}/natgw_state"
+
+ctdb_setnatgwstate ()
+{
+    echo "$2" >"$FAKE_CTDB_NATGW_STATE"
+}
+
+ctdb_natgwlist ()
+{
+    [ -r "$CTDB_NATGW_NODES" ] || \
+	die "error: missing CTDB_NATGW_NODES=${CTDB_NATGW_NODES}"
+
+    # Determine the master node
+    _master="-1 0.0.0.0"
+    if [ -r "$FAKE_CTDB_NATGW_STATE" ] ; then
+	read _state <"$FAKE_CTDB_NATGW_STATE"
+	if [ "$_state" = "on" ] ; then
+	    _pnn=0
+	    while read _ip ; do
+		if [ "$FAKE_CTDB_NATGW_MASTER" = "$_ip" ] ; then
+		    _master="${_pnn} ${_ip}"
+		    break
+		fi
+		_pnn=$(($_pnn + 1))
+	    done <"$CTDB_NATGW_NODES"
+	fi
+    fi
+    echo "$_master"
+
+    # Now print the node information - it is clearer to do this in a
+    # second pass.  Any nodes before the master that have state not
+    # "off" are tagged as unhealthy, just so the output makes some
+    # sense.
+    _pnn=0
+    _found_master=false
+    while read _ip ; do
+	if [ "$FAKE_CTDB_NATGW_MASTER" = "$_ip" ] ; then
+	    _found_master=true
+	fi
+	if $_found_master ; then
+	    _outstate="HEALTHY"
+	else
+	    if [ $FAKE_CTDB_PNN -eq $_pnn -a "$_state" = "off" ] ; then
+		_outstate="HEALTHY"
+	    else
+		_outstate="UNHEALTHY"
+	    fi
+	fi
+	if [ $FAKE_CTDB_PNN -eq $_pnn ] ; then
+	    _outstate="${_outstate} (THIS NODE)"
+	fi
+	printf "pnn:%d %-16s ${_outstate}\n" $_pnn "$_ip"
+
+	_pnn=$(($_pnn + 1))
+    done <"$CTDB_NATGW_NODES"
+
+}
+
+######################################################################
+
 case "$1" in
     gettickles)
 	setup_tickles
@@ -323,12 +389,14 @@ case "$1" in
 	done
 	;;
     gratiousarp) : ;;  # Do nothing for now
-    killtcp)	 ctdb_killtcp "$@" ;;
-    ip)          ctdb_ip "$@" ;;
-    pnn|xpnn)    ctdb_pnn ;;
-    enable)      ctdb_enable "$@";;
-    disable)     ctdb_disable "$@";;
-    moveip)      ctdb_moveip "$@";;
-    shutdown)    ctdb_shutdown "$@";;
+    killtcp)	   ctdb_killtcp "$@" ;;
+    ip)            ctdb_ip "$@" ;;
+    pnn|xpnn)      ctdb_pnn ;;
+    enable)        ctdb_enable "$@";;
+    disable)       ctdb_disable "$@";;
+    moveip)        ctdb_moveip "$@";;
+    shutdown)      ctdb_shutdown "$@";;
+    setnatgwstate) ctdb_setnatgwstate "$@" ;;
+    natgwlist)     ctdb_natgwlist "$@" ;;
     *) not_implemented "$1" ;;
 esac
diff --git a/ctdb/tests/eventscripts/stubs/ip b/ctdb/tests/eventscripts/stubs/ip
index 860f6a54089..e8f17d84131 100755
--- a/ctdb/tests/eventscripts/stubs/ip
+++ b/ctdb/tests/eventscripts/stubs/ip
@@ -80,12 +80,17 @@ ip_check_table ()
 {
     _cmd="$1"
 
+    if [ "$_cmd" = "route" -a -z "$_table" ] ;then
+	_table="main"
+    fi
+
     [ -n "$_table" ] || not_implemented "ip rule/route without \"table\""
 
-    # Only allow tables names from 13.per_ip_routing.  This is a cheap
-    # way of avoiding implementing the default/main/local tables.
+    # Only allow tables names from 13.per_ip_routing and "main".  This
+    # is a cheap way of avoiding implementing the default/local
+    # tables.
     case "$_table" in
-	ctdb.*)
+	ctdb.*|main)
 	    if $IP_ROUTE_BAD_TABLE_ID ; then
 		# Ouch.  Simulate inconsistent errors from ip.  :-(
 		case "$_cmd" in
@@ -449,14 +454,17 @@ ip_route ()
 	show|list)    shift ; ip_route_show  "$@" ;;
 	flush)        shift ; ip_route_flush "$@" ;;
 	add)          shift ; ip_route_add   "$@" ;;
+	del*)         shift ; ip_route_del   "$@" ;;
 	*) not_implemented "$1 in \"ip route\"" ;;
     esac
 }
 
 ip_route_common ()
 {
-    [ "$1" = table ] || not_implemented "$1 in \"$orig_args\""
-    _table="$2"
+    if [ "$1" = table ] ; then
+	_table="$2"
+	shift 2
+    fi
 
     ip_check_table "route"
 }
@@ -487,6 +495,7 @@ ip_route_add ()
     _dev=""
     _gw=""
     _table=""
+    _metric=""
 
     while [ -n "$1" ] ; do
 	case "$1" in
@@ -495,13 +504,15 @@ ip_route_add ()
 	    dev)   _dev="$2"   ; shift 2 ;;
 	    via)   _gw="$2"    ; shift 2 ;;
 	    table) _table="$2" ; shift 2 ;;
+	    metric) _metric="$2" ; shift 2 ;;
 	    *) not_implemented "$1 in \"$orig_args\"" ;;
 	esac
     done
 
     ip_check_table "route"
     [ -n "$_prefix" ] || not_implemented "ip route without inet prefix in \"$orig_args\""
-    [ -n "$_dev" ] || not_implemented "ip route without \"dev\" in \"$orig_args\""
+    # This can't be easily deduced, so print some garbage.
+    [ -n "$_dev" ] || _dev="ethXXX"
 
     # Alias or add missing bits
     case "$_prefix" in
@@ -517,14 +528,65 @@ ip_route_add ()
     (
 	flock 0
 
-	if [ -n "$_gw" ] ; then
-	    echo "${_prefix} via ${_gw} dev ${_dev} "
-	else
-	    echo "${_prefix} dev ${_dev}  scope link "
-	fi >>"$_f"
+	_out="${_prefix} "
+	[ -z "$_gw" ] || _out="${_out}via ${_gw} "
+	[ -z "$_dev" ] || _out="${_out}dev ${_dev} "
+	[ -n "$_gw" ] || _out="${_out} scope link "
+	[ -z "$_metric" ] || _out="${_out} metric ${_metric} "
+	echo "$_out" >>"$_f"
     ) <"$_f"
 }
 
+ip_route_del ()
+{
+    _prefix=""
+    _dev=""
+    _gw=""
+    _table=""
+    _metric=""
+
+    while [ -n "$1" ] ; do
+	case "$1" in
+	    *.*.*.*/*|*.*.*.*) _prefix="$1" ; shift 1 ;;
+	    local) _prefix="$2" ; shift 2 ;;
+	    dev)   _dev="$2"   ; shift 2 ;;
+	    via)   _gw="$2"    ; shift 2 ;;
+	    table) _table="$2" ; shift 2 ;;
+	    metric) _metric="$2" ; shift 2 ;;
+	    *) not_implemented "$1 in \"$orig_args\"" ;;
+	esac
+    done
+
+    ip_check_table "route"
+    [ -n "$_prefix" ] || not_implemented "ip route without inet prefix in \"$orig_args\""
+    # This can't be easily deduced, so print some garbage.
+    [ -n "$_dev" ] || _dev="ethXXX"
+
+    # Alias or add missing bits
+    case "$_prefix" in
+	0.0.0.0/0) _prefix="default" ;;
+	*/*) : ;;
+	*) _prefix="${_prefix}/32" ;;
+    esac
+
+    _f="$FAKE_IP_STATE/routes/${_table}"
+    mkdir -p "$FAKE_IP_STATE/routes"
+    touch "$_f"
+
+    (
+	flock 0
+
+	# Escape some dots
+	[ -z "$_gw" ] || _gw=$(echo "$_gw" | sed -e 's@\.@\\.@g')
+	_prefix=$(echo "$_prefix" | sed -e 's@\.@\\.@g' -e 's@/@\\/@')
+
+	_re="^${_prefix}\>.*"
+	[ -z "$_gw" ] || _re="${_re}\<via ${_gw}\>.*"
+	[ -z "$_dev" ] || _re="${_re}\<dev ${_dev}\>.*"
+	[ -z "$_metric" ] || _re="${_re}.*\<metric ${_metric}\>.*"
+	sed -i -e "/${_re}/d" "$_f"
+    ) <"$_f"
+}
 
 ######################################################################