Merge remote branch 'martins/policy_routing'

(This used to be ctdb commit 7293762046e25a9c96e6417e34e18c214ebc6e55)

26 files changed, 1508 insertions, 945 deletions

diff --git a/ctdb/.gitignore b/ctdb/.gitignore
index ef8ae6ad3c..77ffc3b793 100644
--- a/ctdb/.gitignore
+++ b/ctdb/.gitignore
@@ -26,3 +26,5 @@ tests/events.d/00.ctdb_test_trigger
 tests/var
 tests/takeover/ctdb_takeover.pyc
 tests/eventscripts/var
+tests/eventscripts/etc/iproute2
+tests/eventscripts/etc-ctdb/policy_routing
diff --git a/ctdb/Makefile.in b/ctdb/Makefile.in
index 9055a8f64d..90d7b085cc 100755
--- a/ctdb/Makefile.in
+++ b/ctdb/Makefile.in
@@ -315,7 +315,6 @@ install: all $(PMDA_INSTALL)
 	${INSTALLCMD} -m 644 include/ctdb_typesafe_cb.h $(DESTDIR)$(includedir)
 	${INSTALLCMD} -m 644 config/functions $(DESTDIR)$(etcdir)/ctdb
 	${INSTALLCMD} -m 755 config/statd-callout $(DESTDIR)$(etcdir)/ctdb
-	${INSTALLCMD} -m 755 config/interface_modify.sh $(DESTDIR)$(etcdir)/ctdb
 	${INSTALLCMD} -m 644 config/events.d/README $(DESTDIR)$(docdir)/ctdb/README.eventscripts
 	${INSTALLCMD} -m 644 doc/recovery-process.txt $(DESTDIR)$(docdir)/ctdb/recovery-process.txt
 	${INSTALLCMD} -m 755 config/events.d/00.ctdb $(DESTDIR)$(etcdir)/ctdb/events.d
diff --git a/ctdb/config/README b/ctdb/config/README
index f2457a7618..ffbeb0e690 100644
--- a/ctdb/config/README
+++ b/ctdb/config/README
@@ -14,12 +14,6 @@ Selected highlights:
 
     Support functions, sourced by eventscripts and other scripts.
 
-  interface_modify.sh
-
-    Script to support add/remove IPs and other funky stuff.  Not sure
-    why this is separate... but it certainly allows easy wrapping by
-    flock.
-
   statd-callout
 
     rpc.statd high-availability callout to support lock migration on
diff --git a/ctdb/config/events.d/13.per_ip_routing b/ctdb/config/events.d/13.per_ip_routing
index d8a064493e..f51d5edf5f 100755
--- a/ctdb/config/events.d/13.per_ip_routing
+++ b/ctdb/config/events.d/13.per_ip_routing
@@ -3,492 +3,350 @@
 . $CTDB_BASE/functions
 loadconfig
 
-ctdb_setup_service_state_dir "per_ip_routing"
+# Do nothing if unconfigured 
+[ -n "$CTDB_PER_IP_ROUTING_CONF" ] || exit 0
 
-[ -z "$CTDB_PER_IP_ROUTING_STATE" ] && {
-	CTDB_PER_IP_ROUTING_STATE="$service_state_dir"
-}
+table_id_prefix="ctdb."
 
-AUTO_LINK_LOCAL="no"
-
-case "$CTDB_PER_IP_ROUTING_CONF" in
-	__auto_link_local__)
-		AUTO_LINK_LOCAL="yes"
-		CTDB_PER_IP_ROUTING_CONF="$CTDB_PER_IP_ROUTING_STATE/auto_link_local.conf"
-		;;
-	*)
-		[ -z "$CTDB_PER_IP_ROUTING_CONF" ] && {
-			#echo "No config file found. Nothing to do for 13.per_ip_routing"
-			exit 0;
-		}
-		;;
-esac
+[ -n "$CTDB_PER_IP_ROUTING_RULE_PREF" ] || \
+    die "error: CTDB_PER_IP_ROUTING_RULE_PREF not configured"
 
-_low=$CTDB_PER_IP_ROUTING_TABLE_ID_LOW
-_high=$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH
+[ "$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" -lt "$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" ] 2>/dev/null || \
+    die "error: CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$CTDB_PER_IP_ROUTING_TABLE_ID_LOW] and/or CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH] improperly configured"
 
-test -z "$_low" && {
-	echo "$0: CTDB_PER_IP_ROUTING_TABLE_ID_LOW not configured";
-	exit 1;
-}
-test -z "$_high" && {
-	echo "$0: CTDB_PER_IP_ROUTING_TABLE_ID_HIGH not configured";
-	exit 1;
-}
-test "$_low" -ge "$_high" && {
-	echo "$0: CTDB_PER_IP_ROUTING_TABLE_ID_LOW[$_low] needs to be below CTDB_PER_IP_ROUTING_TABLE_ID_HIGH[$_high]";
-	exit 1;
-}
+######################################################################
 
-test -z "$CTDB_PER_IP_ROUTING_RULE_PREF" && {
-	echo "$0: CTDB_PER_IP_ROUTING_RULE_PREF not configured";
-	exit 1;
-}
-
-locknesting=0
-lock_root="$CTDB_PER_IP_ROUTING_STATE"
-host=`hostname`
-
-lock_debug()
+ipv4_is_valid_addr()
 {
-	echo -n ""
-}
-
-############################
-# grab a lock file. Not atomic, but close :)
-# tries to cope with NFS
-lock_file() {
-	if [ -z "$lock_root" ]; then
-		lock_root=`pwd`;
-	fi
-	lckf="$lock_root/$1"
-	machine=`cat "$lckf" 2> /dev/null | cut -d: -f1`
-	pid=`cat "$lckf" 2> /dev/null | cut -d: -f2`
-
-	if [ "$pid" = "$$" ]; then
-		locknesting=`expr $locknesting + 1`
-		lock_debug "lock nesting now $locknesting"
-		return 0
+    _ip="$1"
+
+    _count=0
+    # Get the shell to break up the address into 1 word per octet 
+    for _o in $(export IFS="." ; echo $_ip) ; do
+	# The 2>/dev/null stops output from failures where an "octet"
+	# is not numeric.  The test will still fail.
+	if ! [ 0 -le $_o -a $_o -le 255 ] 2>/dev/null ; then
+	    return 1
 	fi
+	_count=$(($_count + 1))
+    done
 
-	if test -f "$lckf"; then
-		test $machine = $host || {
-			lock_debug "lock file $lckf is valid for other machine $machine"
-			stat -c%y "$lckf"
-			return 1
-		}
-		kill -0 $pid && {
-			lock_debug "lock file $lckf is valid for process $pid"
-			stat -c%y "$lckf"
-			return 1
-		}
-		lock_debug "stale lock file $lckf for $machine:$pid"
-		cat "$lckf"
-		rm -f "$lckf"
-	fi
-	echo "$host:$$" > "$lckf"
-	return 0
+    # A valid IPv4 address has 4 octets
+    [ $_count -eq 4 ]
 }
 
-############################
-# unlock a lock file
-unlock_file() {
-	if [ -z "$lock_root" ]; then
-		lock_root=`pwd`;
-	fi
-	if [ "$locknesting" != "0" ]; then
-		locknesting=`expr $locknesting - 1`
-		lock_debug "lock nesting now $locknesting"
-	else
-		lckf="$lock_root/$1"
-		rm -f "$lckf"
-	fi
-}
-
-generate_table_id () {
-	local _ip=$1
-	local _ipsdir="$CTDB_PER_IP_ROUTING_STATE/ips"
-	local _ipdir="$_ipsdir/$_ip"
+ensure_ipv4_is_valid_addr ()
+{
+    _event="$1"
+    _ip="$2"
 
-	mkdir -p $_ipdir
+    ipv4_is_valid_addr "$_ip" || {
+	echo "$0: $_event not an ipv4 address skipping IP:$_ip"
+	exit 0
+    }
+}
 
-	#echo "generate_table_id $_ip"
+ipv4_host_addr_to_net ()
+{
+    _host="$1"
+    _maskbits="$2"
+
+    # Convert the host address to an unsigned long by splitting out
+    # the octets and doing the math.
+    _host_ul=0
+    for _o in $(export IFS="." ; echo $_host) ; do
+	_host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
+    done
+
+    # Calculate the mask and apply it.
+    _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
+    _net_ul=$(( $_host_ul & $_mask_ul ))
+ 
+    # Now convert to a network address one byte at a time.
+    _net=""
+    for _o in $(seq 1 4) ; do
+	_net="$(($_net_ul & 255))${_net:+.}${_net}"
+	_net_ul=$(($_net_ul >> 8))
+    done
+
+    echo "${_net}/${_maskbits}"
+}
 
-	local _id=`cat $_ipdir/table_id 2>/dev/null| xargs`
-	test -n "$_id" && {
-		#echo "IP: $_ip => OLD TABLE: $_id"
-		table_id=$_id
-		return 0;
-	}
+######################################################################
 
-	local _low="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW"
-	local _high="$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH"
-
-	local _newid=""
-	for _id in `seq $_low $_high | xargs`; do
-		local _table_lck="table_id_$_id.lock"
-		lock_file $_table_lck 2>/dev/null || {
-			continue;
-		}
-		local _taken=`grep "^$_id$" $_ipsdir/*/table_id 2>/dev/null| wc -l | xargs`
-		test x"$_taken" != x"0" && {
-			unlock_file $_table_lck
-			#echo "tableid: $_id taken"
-			continue
-		}
-		_newid=$_id;
-		echo "$_newid" > $_ipdir/table_id
-		unlock_file $_table_lck
-		break;
+# Setup a table id to use for the given IP.  We don't need to know it,
+# it just needs to exist in /etc/iproute2/rt_tables.  Fail if no free
+# table id could be found in the configured range.
+ensure_table_id_for_ip ()
+{
+    _ip=$1
+
+    _f="$CTDB_ETCDIR/iproute2/rt_tables"
+    # This file should always exist, but...
+    if [ ! -f "$_f" ] ; then
+	mkdir -p $(dirname "$_f")
+	touch "$_f"
+    fi
+
+    # Maintain a table id for each IP address we've ever seen in
+    # rt_tables.  We use a "ctdb." prefix on the label.
+    _label="${table_id_prefix}${_ip}"
+
+    # This finds either the table id corresponding to the label or a
+    # new unused one (that is greater than all the used ones in the
+    # range).
+    (
+	# Note that die() just gets us out of the subshell...
+	flock --timeout 30 0 || \
+	    die "ensure_table_id_for_ip: failed to lock file $_f"
+
+	_new=$CTDB_PER_IP_ROUTING_TABLE_ID_LOW
+	while read _t _l ; do
+	    # Skip comments
+	    case "$_t" in
+		\#*) continue ;;
+	    esac
+	    # Found existing: done
+	    if [ "$_l" = "$_label" ] ; then
+		return 0
+	    fi
+	    # Potentially update the new table id to be used.  The
+	    # redirect stops error spam for a non-numeric value.
+	    if [ $_new -le $_t -a \
+		$_t -le $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH ] 2>/dev/null ; then
+		_new=$(($_t + 1))
+	    fi
 	done
 
-	test -z "$_newid" && {
-		echo "generate_table_id: out of table ids: $_low - $_high"
-		exit 1;
-	}
-
-	#echo "IP: $_ip => NEW TABLE: $_newid"
-	table_id=$_newid
-	return 0;
+	# If the new table id is legal then add it to the file and
+	# print it.
+	if [ $_new -le $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH ] ; then
+	    printf "%d\t%s\n" "$_new" "$_label" >>"$_f"
+	    return 0
+	else
+	    return 1
+	fi
+    ) <"$_f"
 }
 
-run_release_script_once()
+# Clean up all the table ids that we might own.
+clean_up_table_ids ()
 {
-	local _script=$1
-
-	#echo "run_release_script_once[$_script]"
-
-	test -x "$_script" && {
-		#echo "run it: start"
-		$_script || {
-			echo "release_script: $_script - failed $?"
-			return $?;
-		}
-		#echo "run it: end"
-	}
-
-	echo '#!/bin/sh' > $_script
-	echo '#' >> $_script
-	echo >> $_script
+    _f="$CTDB_ETCDIR/iproute2/rt_tables"
+    # Even if this didn't exist on the system, adding a route will
+    # have created it.  What if we startup and immediately shutdown?
+    if [ ! -f "$_f" ] ; then
+	mkdir -p $(basename "$_f")
+	touch "$_f"
+    fi
+
+    (
+	# Note that die() just gets us out of the subshell...
+	flock --timeout 30 0 || \
+	    die "clean_up_table_ids: failed to lock file $_f"
+
+	# Delete any items from the file that have a table id in our
+	# range or a label matching our label.  Preserve comments.
+	_tmp="${_f}.$$.ctdb"
+	awk -v min="$CTDB_PER_IP_ROUTING_TABLE_ID_LOW" \
+	    -v max="$CTDB_PER_IP_ROUTING_TABLE_ID_HIGH" \
+	    -v pre="$table_id_prefix" \
+	    '/^#/ || \
+	     !(min <= $1 && $1 <= max) && \
+	     !(index($2, pre) == 1) \
+	     { print $0 }' "$_f" >"$_tmp"
+
+	mv "$_tmp" "$_f"
+	# The lock is gone - don't do anything else here
+    ) <"$_f"
+}
 
-	chmod +x $_script
+######################################################################
 
-	return 0;
+# This prints the config for an IP, which is either relevant entries
+# from the config file or, if set to the magic link local value, some
+# link local routing config for the IP.
+# NOTE: non-zero return indicates missing configuration file
+get_config_for_ip ()
+{
+    _ip="$1"
+
+    if [ "$CTDB_PER_IP_ROUTING_CONF" = "__auto_link_local__" ] ; then
+	# When parsing public_addresses also split on '/'.  This means
+	# that we get the maskbits as item #2 without further parsing.
+	while IFS="/$IFS" read _i _maskbits _x ; do
+	    if [ "$_ip" = "$_i" ] ; then
+		echo -n "$_ip "; ipv4_host_addr_to_net "$_ip" "$_maskbits"
+	    fi
+	done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE:-/dev/null}${CTDB:+/public_addresses}}"
+    else
+	[ -f "$CTDB_PER_IP_ROUTING_CONF" ] || return 1
+
+	while read _i _rest ; do
+	    if [ "$_ip" = "$_i" ] ; then
+		printf "%s\t%s\n" "$_ip" "$_rest"
+	    fi
+	done <"$CTDB_PER_IP_ROUTING_CONF"
+    fi
 }
 
-generate_auto_link_local()
+ip_has_configuration ()
 {
-	local _ip=$1
-	local _maskbits=$2
-
-	#echo "generate_auto_link_local $_ip $_maskbits"
-
-	local _netip=`ipv4_host_addr_to_net_addr $_ip $_maskbits`
-
-	local _line="$_ip $_netip/$_maskbits"
-
-	local _lockfile="$CTDB_PER_IP_ROUTING_CONF.lock"
-	local _script="$CTDB_PER_IP_ROUTING_CONF.$$.sh"
-
-	echo "#!/bin/sh" > $_script
-	echo "#" >> $_script
-	echo "" >> $_script
-	echo "_config=\`cat $CTDB_PER_IP_ROUTING_CONF 2>/dev/null\`" >> $_script
-	echo "_exact=\`echo -n \"\$_config\" | grep \"^$_line\$\" | wc -l | xargs\`" >> $_script
-	echo "" >> $_script
-
-	echo "test x\"\$_exact\" = x\"1\" && {" >> $_script
-	echo "    exit 0;" >> $_script
-	echo "}" >> $_script
-	echo "" >> $_script
-
-	echo "_tmp=\"$CTDB_PER_IP_ROUTING_CONF.$$.tmp\"" >> $_script
-	echo "echo -n \"\$_config\" | grep -v \"^$_ip \" | cat > \$_tmp || {" >> $_script
-	echo "    echo \"echo -n \\\"\$_config\\\" | grep -v \\\"^$_ip \\\" > \$_tmp - failed\"" >> $_script
-	echo "    exit 1;" >> $_script
-	echo "}" >> $_script
-	echo "echo \"$_line\" >> \$_tmp || {" >> $_script
-	echo "    echo \"echo \\\"$_line\\\" >> \$_tmp - failed\"" >> $_script
-	echo "    exit 1;" >> $_script
-	echo "}" >> $_script
-	echo "" >> $_script
-
-	echo "mv \$_tmp $CTDB_PER_IP_ROUTING_CONF || {" >> $_script
-	echo "    echo \"mv \$_tmp $CTDB_PER_IP_ROUTING_CONF - failed\"" >> $_script
-	echo "    exit 1;" >> $_script
-	echo "}" >> $_script
-	echo "" >> $_script
-
-	echo "echo \"Added '$_line' to $CTDB_PER_IP_ROUTING_CONF\"">> $_script
-	echo "exit 0" >> $_script
-
-	chmod +x $_script
-
-	test -f $_lockfile || {
-		touch $_lockfile
-	}
+    _ip="$1"
 
-	flock --timeout 30 $_lockfile $_script
-	ret=$?
-	rm $_script
-	return $ret
+    _config="$(get_config_for_ip $_ip)" || \
+	die "error: CTDB_PER_IP_ROUTING_CONF=$CTDB_PER_IP_ROUTING_CONF file not found"
+    [ -n "$_config" ]
 }
 
-generate_per_ip_routing()
+add_routing_for_ip ()
 {
-	local _ip=$1
-	local _maskbits=$2
-	local _iface=$3
-	local _readonly=$4
-	local _ipdir="$CTDB_PER_IP_ROUTING_STATE/ips/$_ip"
-
-	table_id=""
-	release_script="$_ipdir/per_ip_routing_release.sh"
-	setup_script="$_ipdir/per_ip_routing_setup.sh"
-
-	test x"$_readonly" = x"yes" && {
-		test -d $_ipdir || {
-			return 1;
-		}
-		return 0;
-	}
+    _iface="$1"
+    _ip="$2"
 
-	mkdir -p $_ipdir || {
-		echo "mkdir -p $_ipdir failed"
-		return 1;
-	}
-	echo "$_ip" > $_ipdir/ip
+    # Do nothing if no config for this IP.
+    ip_has_configuration "$_ip" || return 0
 
-	generate_table_id $_ip
+    ensure_table_id_for_ip "$_ip" || \
+	die "add_routing_for_ip: out of table ids in range $CTDB_PER_IP_ROUTING_TABLE_ID_LOW - $CTDB_PER_IP_ROUTING_TABLE_ID_HIGH"
 
-	test x"$AUTO_LINK_LOCAL" = x"yes" && {
-		generate_auto_link_local $_ip $_maskbits
-	}
+    _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
+    _table_id="${table_id_prefix}${_ip}"
 
-	run_release_script_once $release_script
+    del_routing_for_ip "$_ip"
 
-	echo '#!/bin/sh' > $setup_script
-	echo '#' >> $setup_script
-	echo >> $setup_script
-	chmod +x $setup_script
+    ip rule add from "$_ip" pref "$_pref" table "$_table_id" || \
+	die "add_routing_for_ip: failed to add rule for $_ip"
 
-	return 0;
+    # Add routes to table for any lines matching the IP.
+    get_config_for_ip "$_ip" |
+    while read _i _dest _gw ; do
+	_r="$_dest ${_gw:+via} $_gw dev $_iface table $_table_id"
+	ip route add $_r || \
+	    die "add_routing_for_ip: failed to add route: $_r"
+    done
 }
 
-setup_per_ip_routing()
+del_routing_for_ip ()
 {
-	local _ip=$1
-	local _iface=$2
-	local _table_id=$3
-	local _release_script=$4
-	local _setup_script=$5
-
-	local _config=`cat $CTDB_PER_IP_ROUTING_CONF`
-	local _lines=`echo -n "$_config" | grep -n "^$_ip " | cut -d ':' -f1 | xargs`
+    _ip="$1"
 
-	local _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
+    _pref="$CTDB_PER_IP_ROUTING_RULE_PREF"
+    _table_id="${table_id_prefix}${_ip}"
 
-	test -n "$_lines" && {
-		echo "ip rule del from $_ip pref $_pref table $_table_id" >> $_release_script
-		echo "ip route flush table $_table_id 2>/dev/null" >> $_release_script
-
-		cmd="ip rule del from $_ip pref $_pref 2>/dev/null"
-		echo "$cmd" >> $_setup_script
+    # Do this unconditionally since we own any matching table ids...
+    ip rule del from $_ip pref $_pref table $_table_id 2>/dev/null
+    ip route flush table $_table_id 2>/dev/null
+}
 
-		cmd="ip route flush table $_table_id 2>/dev/null"
-		echo "$cmd" >> $_setup_script
+######################################################################
 
-		cmd="ip rule add from $_ip pref $_pref table $_table_id"
-		echo "$cmd || {" >> $_setup_script
-		echo "    echo \"$cmd - failed \$ret\"" >> $_setup_script
-		echo "    exit \$ret" >> $_setup_script
-		echo "}" >> $_setup_script
-	}
-	local _l
-	for _l in $_lines; do
-		local _line=`echo -n "$_config" | head -n $_l | tail -n 1`
-		local _dest=`echo -n "$_line" | cut -d ' ' -f 2`
-		local _gw=`echo -n "$_line" | cut -d ' ' -f 3`
-
-		local _via=""
-		test -n "$_gw" && {
-			_via="via $_gw"
-		}
-
-		cmd="ip route add $_dest $_via dev $_iface table $_table_id"
-		echo "$cmd || {" >> $_setup_script
-		echo "    echo \"$cmd - failed \$ret\"" >> $_setup_script
-		echo "    exit \$ret" >> $_setup_script
-		echo "}" >> $_setup_script
+flush_rules_and_routes ()
+{
+	ip rule show |
+	while read _p _x _i _x _t ; do
+	    # Remove trailing colon after priority/preference.
+	    _p="${_p%:}"
+	    # Only remove rules that match our priority/preference.
+	    [ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
+
+	    echo "Removing ip rule for public address $_i for routing table $_t"
+	    ip rule del from "$_i" table "$_t" pref "$_p"
+	    ip route flush table "$_t" 2>/dev/null
 	done
+}
 
-	$_setup_script
-	return $?;
+# Add any missing routes.  Some might have gone missing if, for
+# example, all IPs on the network were removed (possibly if the
+# primary was removed).
+add_missing_routes ()
+{
+    ctdb ip -v -Y | {
+	read _x # skip header line
+
+	# Read the rest of the lines.  We're only interested in the
+	# "IP" and "ActiveInterface" columns.  The latter is only set
+	# for addresses local to this node, making it easy to skip
+	# non-local addresses.  For each IP local address we check if
+	# the relevant routing table is populated and populate it if
+	# not.
+	while IFS=":" read _x _ip _x _iface _x ; do
+	    [ -n "$_iface" ] || continue
+	    
+	    _table_id="${table_id_prefix}${_ip}"
+	    if [ -z "$(ip route show table $_table_id 2>/dev/null)" ]  ; then
+		add_routing_for_ip "$_iface" "$_ip"
+	    fi
+	done
+    } || exit $?
 }
 
+######################################################################
+
 ctdb_check_args "$@"
 
 case "$1" in
-     #############################
-     # called when ctdbd starts up
-     startup)
-	# cleanup old rules
-	pref=$CTDB_PER_IP_ROUTING_RULE_PREF
-	rules=`ip rule show | grep "^$pref:" | sed -e 's/.*from \([^ ][^ ]*\) lookup \([^ ][^ ]*\)/\2;\1/' | xargs`
-	for r in $rules; do
-		table_id=`echo -n "$r" | cut -d ';' -f1`
-		ip=`echo -n "$r" | cut -d ';' -f2-`
-
-		echo "Removing ip rule for public address $ip for routing table $table_id"
-		cmd="ip rule del from $ip table $table_id pref $pref"
-		#echo $cmd
-		eval $cmd
-		cmd="ip route flush table $table_id"
-		#echo $cmd
-		eval $cmd 2>/dev/null
-	done
+    startup)
+	flush_rules_and_routes
 
-	# make sure that we only respond to ARP messages from the NIC where
-	# a particular ip address is associated.
-	[ -f /proc/sys/net/ipv4/conf/all/arp_filter ] && {
-	    echo 1 > /proc/sys/net/ipv4/conf/all/arp_filter
+	# make sure that we only respond to ARP messages from the NIC
+	# where a particular ip address is associated.
+	get_proc sys/net/ipv4/conf/all/arp_filter >/dev/null 2>&1 && {
+	    set_proc sys/net/ipv4/conf/all/arp_filter 1
 	}
-
-	mkdir -p $CTDB_PER_IP_ROUTING_STATE
-
 	;;
 
-     shutdown)
-
-	for s in $CTDB_PER_IP_ROUTING_STATE/ips/*/per_ip_routing_release.sh; do
-		run_release_script_once "$s"
-	done
-	rm -rf $CTDB_PER_IP_ROUTING_STATE
-
+    shutdown)
+	flush_rules_and_routes
+	clean_up_table_ids
 	;;
 
-     ################################################
-     # called when ctdbd wants to claim an IP address
-     takeip)
+    takeip)
 	iface=$2
 	ip=$3
 	maskbits=$4
 
-	ipv4_is_valid_addr $ip || {
-		echo "$0: $1 not an ipv4 address skipping IP:$ip"
-		exit 0;
-	}
-
-	[ ! -d "$CTDB_PER_IP_ROUTING_STATE" ] && {
-		echo "$0: $1 No state directory found, waiting for startup."
-		exit 0;
-	}
-
-	generate_per_ip_routing $ip $maskbits $iface "no" || {
-		echo "$0: $1: generate_per_ip_routing $ip $maskbits $iface no - failed"
-		exit 1;
-	}
-
-	setup_per_ip_routing $ip $iface $table_id $release_script $setup_script || {
-		echo "$0: $1: setup_per_ip_routing $ip $iface $table_id $release_script $setup_script - failed"
-		exit 1;
-	}
-
-	setup_iface_ip_readd_script $iface $ip $maskbits $setup_script || {
-		echo "$0: $1: setup_iface_ip_readd_script $iface $ip $maskbits $setup_script - failed"
-		exit 1;
-	}
+	ensure_ipv4_is_valid_addr "$1" "$ip"
+	add_routing_for_ip "$iface" "$ip"
 
 	# flush our route cache
-	echo 1 > /proc/sys/net/ipv4/route/flush
-	ctdb gratiousarp $ip $iface
+	set_proc sys/net/ipv4/route/flush 1
 
+	ctdb gratiousarp "$ip" "$iface"
 	;;
 
-     ################################################
-     # called when ctdbd wants to claim an IP address
-     updateip)
+    updateip)
 	oiface=$2
 	niface=$3
 	ip=$4
 	maskbits=$5
 
-	ipv4_is_valid_addr $ip || {
-		echo "$0: $1 not an ipv4 address skipping IP:$ip"
-		exit 0;
-	}
-
-	[ ! -d "$CTDB_PER_IP_ROUTING_STATE" ] && {
-		echo "$0: $1 No state directory found, waiting for startup."
-		exit 0;
-	}
-
-	generate_per_ip_routing $ip $maskbits $niface "no" || {
-		echo "$0: $1: generate_per_ip_routing $ip $maskbits $niface no - failed"
-		exit 1;
-	}
-
-	setup_per_ip_routing $ip $niface $table_id $release_script $setup_script || {
-		echo "$0: $1: setup_per_ip_routing $ip $niface $table_id $release_script $setup_script - failed"
-		exit 1;
-	}
-
-	setup_iface_ip_readd_script $niface $ip $maskbits $setup_script || {
-		echo "$0: $1: setup_iface_ip_readd_script $niface $ip $maskbits $setup_script - failed"
-		exit 1;
-	}
+	ensure_ipv4_is_valid_addr "$1" "$ip"
+	add_routing_for_ip "$niface" "$ip"
 
 	# flush our route cache
-	echo 1 > /proc/sys/net/ipv4/route/flush
-
-	ctdb gratiousarp $ip $niface
-	tickle_tcp_connections $ip
+	set_proc sys/net/ipv4/route/flush 1
 
+	ctdb gratiousarp "$ip" "$niface"
+	tickle_tcp_connections "$ip"
 	;;
 
-     ##################################################
-     # called when ctdbd wants to release an IP address
-     releaseip)
+    releaseip)
 	iface=$2
 	ip=$3
 	maskbits=$4
 
-	ipv4_is_valid_addr $ip || {
-		echo "$0: $1 not an ipv4 address skipping IP:$ip"
-		exit 0;
-	}
-
-	[ ! -d "$CTDB_PER_IP_ROUTING_STATE" ] && {
-		echo "$0: $1 No state directory found, waiting for startup."
-		exit 0;
-	}
-
-	generate_per_ip_routing $ip $maskbits $iface "yes" || {
-		echo "$0: $1: generate_per_ip_routing $ip $maskbits $iface yes - failed"
-		exit 1;
-	}
-
-	run_release_script_once "$release_script"
-
+	ensure_ipv4_is_valid_addr "$1" "$ip"
+	del_routing_for_ip "$ip"
 	;;
 
-
-     ###########################################
-     # called when ctdbd has finished a recovery
-     recovered)
-	;;
-
-     ####################################
-     # called when ctdbd is shutting down
-     shutdown)
+    ipreallocated)
+	add_missing_routes
 	;;
 
-     monitor)
-	;;
     *)
 	ctdb_standard_event_handler "$@"
 	;;
 esac
 
 exit 0
-
diff --git a/ctdb/config/functions b/ctdb/config/functions
index 7c5c1c245d..ca89c38379 100755
--- a/ctdb/config/functions
+++ b/ctdb/config/functions
@@ -86,6 +86,15 @@ debug ()
     fi
 }
 
+die ()
+{
+    _msg="$1"
+    _rc="${2:-1}"
+
+    echo "$_msg"
+    exit $_rc
+}
+
 ##############################################################
 # check number of args for different events
 ctdb_check_args ()
@@ -770,72 +779,88 @@ startstop_nfslock() {
 
 add_ip_to_iface()
 {
-	local _iface=$1
-	local _ip=$2
-	local _maskbits=$3
-	local _state_dir="$CTDB_VARDIR/state/interface_modify"
-	local _lockfile="$_state_dir/$_iface.flock"
-	local _readd_base="$_state_dir/$_iface.readd.d"
-
-	mkdir -p $_state_dir || {
-		ret=$?
-		echo "Failed to mkdir -p $_state_dir - $ret"
-		return $ret
-	}
+    _iface=$1
+    _ip=$2
+    _maskbits=$3
 
-	test -f $_lockfile || {
-		touch $_lockfile
-	}
+    _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
+    [ -f "$_lockfile" ] || touch "$_lockfile"
 
-	flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh add "$_iface" "$_ip" "$_maskbits" "$_readd_base"
-	return $?
+    (
+	# Note: use of return/exit/die() below only gets us out of the
+	# sub-shell, which is actually what we want.  That is, the
+	# function should just return non-zero.
+
+	flock --timeout 30 0 || \
+	    die "add_ip_to_iface: unable to get lock for ${_iface}"
+
+	# Ensure interface is up
+	ip link set "$_iface" up || \
+	    die "Failed to bringup interface $_iface"
+
+	ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || \
+	    die "Failed to add $_ip/$_maskbits on dev $_iface"
+    ) <"$_lockfile"
+
+    # Do nothing here - return above only gets us out of the subshell
+    # and doing anything here will affect the return code.
 }
 
 delete_ip_from_iface()
 {
-	local _iface=$1
-	local _ip=$2
-	local _maskbits=$3
-	local _state_dir="$CTDB_VARDIR/state/interface_modify"
-	local _lockfile="$_state_dir/$_iface.flock"
-	local _readd_base="$_state_dir/$_iface.readd.d"
-
-	mkdir -p $_state_dir || {
-		ret=$?
-		echo "Failed to mkdir -p $_state_dir - $ret"
-		return $ret
-	}
+    _iface=$1
+    _ip=$2
+    _maskbits=$3
 
-	test -f $_lockfile || {
-		touch $_lockfile
-	}
+    _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
+    [ -f "$_lockfile" ] || touch "$_lockfile"
 
-	flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh delete "$_iface" "$_ip" "$_maskbits" "$_readd_base"
-	return $?
-}
+    (
+	# Note: use of return/exit/die() below only gets us out of the
+	# sub-shell, which is actually what we want.  That is, the
+	# function should just return non-zero.
 
-setup_iface_ip_readd_script()
-{
-	local _iface=$1
-	local _ip=$2
-	local _maskbits=$3
-	local _readd_script=$4
-	local _state_dir="$CTDB_VARDIR/state/interface_modify"
-	local _lockfile="$_state_dir/$_iface.flock"
-	local _readd_base="$_state_dir/$_iface.readd.d"
-
-	mkdir -p $_state_dir || {
-		ret=$?
-		echo "Failed to mkdir -p $_state_dir - $ret"
-		return $ret
-	}
+	flock --timeout 30 0 || \
+	    die "delete_ip_from_iface: unable to get lock for ${_iface}"
 
-	test -f $_lockfile || {
-		touch $_lockfile
+	_im="$_ip/$_maskbits"  # shorthand for readability
+
+	# "ip addr del" will delete all secondary IPs if this is the
+	# primary.  To work around this _very_ annoying behaviour we
+	# have to keep a record of the secondaries and re-add them
+	# afterwards.  Yuck!
+
+	_secondaries=""
+	if ip addr list dev "$_iface" primary | grep -Fq "inet $_im " ; then
+	    _secondaries=$(ip addr list dev "$_iface" secondary | \
+		awk '$1 == "inet" { print $2 }')
+	fi
+
+	local _rc=0
+	ip addr del "$_im" dev "$_iface" || {
+	    echo "Failed to del $_ip on dev $_iface"
+	    _rc=1
 	}
 
-	flock --timeout 30 $_lockfile $CTDB_BASE/interface_modify.sh readd_script "$_iface" "$_ip" "$_maskbits" "$_readd_base" "$_readd_script"
-	return $?
+	if [ -n "$_secondaries" ] ; then
+	    for _i in $_secondaries; do
+		if ip addr list dev "$_iface" | grep -Fq "inet $_i" ; then
+		    echo "Kept secondary $_i on dev $_iface"
+		else
+		    echo "Re-adding secondary address $_i to dev $_iface"
+		    ip addr add $_i brd + dev $_iface || {
+			echo "Failed to re-add address $_i to dev $_iface"
+			_rc=1
+		    }
+		fi
+	    done
+	fi
+
+	return $_rc
+    ) <"$_lockfile"
+
+    # Do nothing here - return above only gets us out of the subshell
+    # and doing anything here will affect the return code.
 }
 
 ########################################################
@@ -1278,70 +1303,6 @@ ctdb_standard_event_handler ()
     esac
 }
 
-ipv4_host_addr_to_net_addr()
-{
-	local HOST=$1
-	local MASKBITS=$2
-
-	local HOST0=$(echo $HOST | awk -F . '{print $4}')
-	local HOST1=$(echo $HOST | awk -F . '{print $3}')
-	local HOST2=$(echo $HOST | awk -F . '{print $2}')
-	local HOST3=$(echo $HOST | awk -F . '{print $1}')
-
-	local HOST_NUM=$(( $HOST0 + $HOST1 * 256 + $HOST2 * (256 ** 2) + $HOST3 * (256 ** 3) ))
-
-	local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
-
-	local NET_NUM=$(( $HOST_NUM & $MASK_NUM))
-
-	local NET0=$(( $NET_NUM & 255 ))
-	local NET1=$(( ($NET_NUM & (255 * 256)) / 256 ))
-	local NET2=$(( ($NET_NUM & (255 * 256**2)) / 256**2 ))
-	local NET3=$(( ($NET_NUM & (255 * 256**3)) / 256**3 ))
-
-	echo "$NET3.$NET2.$NET1.$NET0"
-}
-
-ipv4_maskbits_to_net_mask()
-{
-	local MASKBITS=$1
-
-	local MASK_NUM=$(( ( (2**32 - 1) * (2**(32 - $MASKBITS)) ) & (2**32 - 1) ))
-
-	local MASK0=$(( $MASK_NUM & 255 ))
-	local MASK1=$(( ($MASK_NUM & (255 * 256)) / 256 ))
-	local MASK2=$(( ($MASK_NUM & (255 * 256**2)) / 256**2 ))
-	local MASK3=$(( ($MASK_NUM & (255 * 256**3)) / 256**3 ))
-
-	echo "$MASK3.$MASK2.$MASK1.$MASK0"
-}
-
-ipv4_is_valid_addr()
-{
-	local ADDR=$1
-	local fail=0
-
-	local N=`echo $ADDR | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//'`
-	test -n "$N" && fail=1
-
-	local ADDR0=$(echo $ADDR | awk -F . '{print $4}')
-	local ADDR1=$(echo $ADDR | awk -F . '{print $3}')
-	local ADDR2=$(echo $ADDR | awk -F . '{print $2}')
-	local ADDR3=$(echo $ADDR | awk -F . '{print $1}')
-
-	test "$ADDR0" -gt 255 && fail=1
-	test "$ADDR1" -gt 255 && fail=1
-	test "$ADDR2" -gt 255 && fail=1
-	test "$ADDR3" -gt 255 && fail=1
-
-	test x"$fail" != x"0" && {
-		#echo "IPv4: '$ADDR' is not a valid address"
-		return 1;
-	}
-
-	return 0;
-}
-
 # iptables doesn't like being re-entered, so flock-wrap it.
 iptables()
 {
diff --git a/ctdb/config/interface_modify.sh b/ctdb/config/interface_modify.sh
deleted file mode 100755
index 041637650f..0000000000
--- a/ctdb/config/interface_modify.sh
+++ /dev/null
@@ -1,142 +0,0 @@
-#!/bin/sh
-#
-
-OP=$1
-IFACE=$2
-IP=$3
-MASKBITS=$4
-READD_BASE=$5
-READD_SCRIPT=$6
-
-add_ip_to_iface()
-{
-	local _iface=$1
-	local _ip=$2
-	local _maskbits=$3
-	local _readd_base=$4
-	local _script_dir="$_readd_base/$_ip.$_maskbits"
-
-	# we make sure the interface is up first
-	ip link set $_iface up || {
-		echo "Failed to bringup interface $_iface"
-		return 1;
-	}
-	ip addr add $_ip/$_maskbits brd + dev $_iface || {
-		echo "Failed to add $_ip/$_maskbits on dev $_iface"
-		return 1;
-	}
-
-	mkdir -p $_script_dir || {
-		echo "Failed to mkdir -p $_script_dir"
-		return 1;
-	}
-
-	rm -f $_script_dir/*
-
-	return 0;
-}
-
-delete_ip_from_iface()
-{
-	local _iface=$1
-	local _ip=$2
-	local _maskbits=$3
-	local _readd_base=$4
-	local _script_dir="$_readd_base/$_ip.$_maskbits"
-
-	# the ip tool will delete all secondary IPs if this is the primary. To work around
-	# this _very_ annoying behaviour we have to keep a record of the secondaries and re-add
-	# them afterwards. yuck
-	local _secondaries=""
-	if ip addr list dev $_iface primary | grep -q "inet $_ip/$_maskbits " ; then
-	    _secondaries=`ip addr list dev $_iface secondary | grep " inet " | awk '{print $2}'`
-	fi
-	local _failed=0
-	ip addr del $_ip/$_maskbits dev $_iface || _failed=1
-	[ -z "$_secondaries" ] || {
-	    local _i=""
-	    for _i in $_secondaries; do
-		if ip addr list dev $_iface | grep -q "inet $_i" ; then
-		    echo "kept secondary $_i on dev $_iface"
-		else
-		    echo "re-adding secondary address $_i to dev $_iface"
-		    ip addr add $_i brd + dev $_iface || _failed=1
-		fi
-		local _s_ip=`echo "$_i" | cut -d '/' -f1`
-		local _s_maskbits=`echo "$_i" | cut -d '/' -f2`
-		local _s_script_dir="$_readd_base/$_s_ip.$_s_maskbits"
-
-		local _s_script=""
-		for _s_script in $_s_script_dir/*; do
-			test -x "$_s_script" || {
-				continue
-			}
-			echo "call $_s_script '$_iface' '$_s_ip' '$_s_maskbits'"
-			$_s_script "$_iface" "$_s_ip" "$_s_maskbits" || {
-				ret=$?
-				echo "$_s_script '$_iface' '$_s_ip' '$_s_maskbits' - failed - $ret"
-				_failed=1
-			}
-		done
-
-	    done
-	}
-
-	test -d $_script_dir && {
-		rm -f $_script_dir/*
-	}
-
-	[ $_failed = 0 ] || {
-		echo "Failed to del $_ip on dev $_iface"
-		return 1;
-	}
-	return 0;
-}
-
-setup_iface_ip_readd_script()
-{
-	local _iface=$1
-	local _ip=$2
-	local _maskbits=$3
-	local _readd_base=$4
-	local _readd_script=$5
-	local _script_dir="$_readd_base/$_ip.$_maskbits"
-
-	test -x "$_readd_script" || {
-		echo "Script '$_readd_script' isn't executable"
-		return 1;
-	}
-
-	local _readd_basename=`basename $_readd_script`
-	local _readd_final="$_script_dir/$_readd_basename"
-
-	mkdir -p $_script_dir || {
-		echo "Failed to mkdir -p $_script_dir"
-		return 1;
-	}
-
-	cp -a $_readd_script $_readd_final || {
-		echo "Failed to - cp -a $_readd_script $_readd_final"
-		return 1;
-	}
-
-	return 0
-}
-
-case "$OP" in
-	add)
-		add_ip_to_iface $IFACE $IP $MASKBITS $READD_BASE
-		exit $?
-		;;
-	delete)
-		delete_ip_from_iface $IFACE $IP $MASKBITS $READD_BASE
-		exit $?
-		;;
-	readd_script)
-		setup_iface_ip_readd_script $IFACE $IP $MASKBITS $READD_BASE $READD_SCRIPT
-		exit $?
-		;;
-esac
-
-echo "$0: unknown operation[$OP]"
-exit 1
diff --git a/ctdb/packaging/RPM/ctdb.spec.in b/ctdb/packaging/RPM/ctdb.spec.in
index 1d6f5702b7..f9a5cefb93 100644
--- a/ctdb/packaging/RPM/ctdb.spec.in
+++ b/ctdb/packaging/RPM/ctdb.spec.in
@@ -112,7 +112,6 @@ rm -rf $RPM_BUILD_ROOT
 %{_sysconfdir}/ctdb/events.d/70.iscsi
 %{_sysconfdir}/ctdb/events.d/91.lvs
 %{_sysconfdir}/ctdb/statd-callout
-%{_sysconfdir}/ctdb/interface_modify.sh
 %{_sbindir}/ctdbd
 %{_bindir}/ctdb
 %{_bindir}/smnotify
diff --git a/ctdb/tests/eventscripts/common.sh b/ctdb/tests/eventscripts/common.sh
index df9656b831..44c975e034 100644
--- a/ctdb/tests/eventscripts/common.sh
+++ b/ctdb/tests/eventscripts/common.sh
@@ -183,6 +183,50 @@ setup_nmap_output_filter ()
     OUT_FILTER="-e 's@^(DEBUG: # Nmap 5.21 scan initiated) .+ (as:)@\1 DATE \2@' -e 's@^(DEBUG: # Nmap done at) .+ (--)@\1 DATE \2@'"
 }
 
+dump_routes ()
+{
+    echo "# ip rule show"
+    ip rule show
+
+    ip rule show |
+    while read _p _x _i _x _t ; do
+	# Remove trailing colon after priority/preference.
+	_p="${_p%:}"
+	# Only remove rules that match our priority/preference.
+	[ "$CTDB_PER_IP_ROUTING_RULE_PREF" = "$_p" ] || continue
+
+	echo "# ip route show table $_t"
+	ip route show table "$_t"
+    done
+}
+
+# Copied from 13.per_ip_routing for now... so this is lazy testing  :-(
+ipv4_host_addr_to_net ()
+{
+    _host="$1"
+    _maskbits="$2"
+
+    # Convert the host address to an unsigned long by splitting out
+    # the octets and doing the math.
+    _host_ul=0
+    for _o in $(export IFS="." ; echo $_host) ; do
+	_host_ul=$(( ($_host_ul << 8) + $_o)) # work around Emacs color bug
+    done
+
+    # Calculate the mask and apply it.
+    _mask_ul=$(( 0xffffffff << (32 - $_maskbits) ))
+    _net_ul=$(( $_host_ul & $_mask_ul ))
+
+    # Now convert to a network address one byte at a time.
+    _net=""
+    for _o in $(seq 1 4) ; do
+	_net="$(($_net_ul & 255))${_net:+.}${_net}"
+	_net_ul=$(($_net_ul >> 8))
+    done
+
+    echo "${_net}/${_maskbits}"
+}
+
 ######################################################################
 
 # CTDB fakery
@@ -231,35 +275,17 @@ setup_ctdb ()
 	eventscripts_test_add_cleanup "rm -f $CTDB_PUBLIC_ADDRESSES"
     fi
 
-    export FAKE_CTDB_IFACES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb/ifaces-down"
+    export FAKE_CTDB_STATE="$EVENTSCRIPTS_TESTS_VAR_DIR/fake-ctdb"
+
+    export FAKE_CTDB_IFACES_DOWN="$FAKE_CTDB_STATE/ifaces-down"
     mkdir -p "$FAKE_CTDB_IFACES_DOWN"
     rm -f "$FAKE_CTDB_IFACES_DOWN"/*
 
-    export FAKE_CTDB_NODES_DOWN="$EVENTSCRIPTS_TESTS_VAR_DIR/nodes-down"
-    mkdir -p "$FAKE_CTDB_NODES_DOWN"
-    rm -f "$FAKE_CTDB_NODES_DOWN"/*
-
-    export FAKE_CTDB_SCRIPTSTATUS="$EVENTSCRIPTS_TESTS_VAR_DIR/scriptstatus"
+    export FAKE_CTDB_SCRIPTSTATUS="$FAKE_CTDB_STATE/scriptstatus"
     mkdir -p "$FAKE_CTDB_SCRIPTSTATUS"
     rm -f "$FAKE_CTDB_SCRIPTSTATUS"/*
 }
 
-
-
-ctdb_nodes_up ()
-{
-    for _i ; do
-	rm -f "${FAKE_CTDB_NODES_DOWN}/${_i}"
-    done
-}
-
-ctdb_nodes_down ()
-{
-    for _i ; do
-	touch "${FAKE_CTDB_NODES_DOWN}/${_i}"
-    done
-}
-
 ctdb_get_interfaces ()
 {
     # The echo/subshell forces all the output onto 1 line.
@@ -272,22 +298,40 @@ ctdb_get_1_interface ()
     echo ${_t%% *}
 }
 
-ctdb_get_public_addresses ()
+# Print all public addresses as: interface IP maskbits
+# Each line is suitable for passing to takeip/releaseip
+ctdb_get_all_public_addresses ()
 {
     _f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
-    echo $(if [ -r "$_f" ] ; then 
-	while read _ip _iface ; do
-	    echo "${_ip}@${_iface}"
-	done <"$_f"
-	fi)
+    while IFS="/$IFS" read _ip _maskbits _ifaces ; do
+	echo "$_ifaces $_ip $_maskbits"
+    done <"$_f"
+}
+
+# Print public addresses on this node as: interface IP maskbits
+# Each line is suitable for passing to takeip/releaseip
+ctdb_get_my_public_addresses ()
+{
+    ctdb ip -v -Y | {
+	read _x # skip header line
+
+	while IFS=":" read _x _ip _x _iface _x ; do
+	    [ -n "$_iface" ] || continue
+	    while IFS="/$IFS" read _i _maskbits _x ; do
+		if [ "$_ip" = "$_i" ] ; then
+		    echo $_iface $_ip $_maskbits
+		    break
+		fi
+	    done <"${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
+	done
+    }
 }
 
 # Prints the 1st public address as: interface IP maskbits
-# This is suitable for passing to 10.interfaces takeip/releaseip
+# This is suitable for passing to takeip/releaseip
 ctdb_get_1_public_address ()
 {
-    _addrs=$(ctdb_get_public_addresses)
-    echo "${_addrs%% *}" | sed -r -e 's#(.*)/(.*)@(.*)#\3 \1 \2#g'
+    ctdb_get_my_public_addresses | head -n 1
 }
 
 ctdb_not_implemented ()
@@ -309,6 +353,17 @@ ctdb_fake_scriptstatus ()
     echo "$_code $_status $_err_out" >"$FAKE_CTDB_SCRIPTSTATUS/$script"
 }
 
+setup_ctdb_policy_routing ()
+{
+    export CTDB_PER_IP_ROUTING_CONF="$CTDB_BASE/policy_routing"
+    export CTDB_PER_IP_ROUTING_RULE_PREF=100
+    export CTDB_PER_IP_ROUTING_TABLE_ID_LOW=1000
+    export CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=2000
+
+    # Tests need to create and populate this file
+    rm -f "$CTDB_PER_IP_ROUTING_CONF"
+}
+
 ######################################################################
 
 # Samba fakery
@@ -633,7 +688,7 @@ define_test ()
 	    die "Internal error - unknown testcase filename format"
     esac
 
-    printf "%-14s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
+    printf "%-17s %-10s %-4s - %s\n\n" "$script" "$event" "$_num" "$desc"
 }
 
 # Set the required result for a test.
@@ -690,8 +745,8 @@ cat <<EOF
 --------------------------------------------------
 Output (Exit status: ${_rc}):
 --------------------------------------------------
-$_out
 EOF
+	echo "$_out" | cat $EVENTSCRIPT_TESTS_CAT_RESULTS_OPTS
     fi
 
     if ! $_passed ; then
@@ -699,8 +754,24 @@ EOF
 --------------------------------------------------
 Required output (Exit status: ${required_rc}):
 --------------------------------------------------
-$required_output
 EOF
+	echo "$required_output" | cat $EVENTSCRIPT_TESTS_CAT_RESULTS_OPTS
+
+	if $EVENTSCRIPT_TESTS_DIFF_RESULTS ; then
+	    _outr=$(mktemp)
+	    echo "$required_output" >"$_outr"
+
+	    _outf=$(mktemp)
+	    echo "$_out" >"$_outf"
+
+	    cat <<EOF
+--------------------------------------------------
+Diff:
+--------------------------------------------------
+EOF
+	    diff -u "$_outr" "$_outf" | cat -A
+	    rm "$_outr" "$_outf"
+	fi
     fi
 }
 
@@ -729,22 +800,8 @@ EOF
     fi
 }
 
-# Run an eventscript once.  The test passes if the return code and
-# output match those required.
-
-# Any args are passed to the eventscript.
-
-# Eventscript tracing can be done by setting:
-#   EVENTSCRIPTS_TESTS_TRACE="sh -x"
-
-# or similar.  This will almost certainly make a test fail but is
-# useful for debugging.
-simple_test ()
+result_check ()
 {
-    [ -n "$event" ] || die 'simple_test: $event not set'
-
-    echo "Running \"$script $event${1:+ }$*\""
-    _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
     _rc=$?
 
     if [ -n "$OUT_FILTER" ] ; then
@@ -763,6 +820,26 @@ simple_test ()
     result_footer "$_passed"
 }
 
+# Run an eventscript once.  The test passes if the return code and
+# output match those required.
+
+# Any args are passed to the eventscript.
+
+# Eventscript tracing can be done by setting:
+#   EVENTSCRIPTS_TESTS_TRACE="sh -x"
+
+# or similar.  This will almost certainly make a test fail but is
+# useful for debugging.
+simple_test ()
+{
+    [ -n "$event" ] || die 'simple_test: $event not set'
+
+    echo "Running eventscript \"$script $event${1:+ }$*\""
+    _out=$($EVENTSCRIPTS_TESTS_TRACE "${CTDB_BASE}/events.d/$script" "$event" "$@" 2>&1)
+
+    result_check
+}
+
 simple_test_event ()
 {
     # If something has previously failed then don't continue.
@@ -774,6 +851,19 @@ simple_test_event ()
     simple_test "$@"
 }
 
+simple_test_command ()
+{
+    # If something has previously failed then don't continue.
+    : ${_passed:=true}
+    $_passed || return 1
+
+    echo "##################################################"
+    echo "Running command \"$*\""
+    _out=$("$@" 2>&1)
+
+    result_check
+}
+
 # Run an eventscript iteratively.
 # - 1st argument is the number of iterations.
 # - 2nd argument is something to eval to do setup for every iteration.
diff --git a/ctdb/tests/eventscripts/etc-ctdb/public_addresses b/ctdb/tests/eventscripts/etc-ctdb/public_addresses
index 5421cd781d..cd2f6be4e1 100644
--- a/ctdb/tests/eventscripts/etc-ctdb/public_addresses
+++ b/ctdb/tests/eventscripts/etc-ctdb/public_addresses
@@ -1,4 +1,9 @@
 10.0.0.1/24 dev123
 10.0.0.2/24 dev123
+10.0.0.3/24 dev123
+10.0.0.4/24 dev123
+10.0.0.5/24 dev123
+10.0.0.6/24 dev123
 10.0.1.1/24 dev456
 10.0.1.2/24 dev456
+10.0.1.3/24 dev456
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh
new file mode 100755
index 0000000000..0bcf8e3055
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.001.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "not configured"
+
+setup_ctdb
+
+ok <<EOF
+# ip rule show
+0:	from all lookup local 
+32766:	from all lookup main 
+32767:	from all lookup default 
+EOF
+
+simple_test_command dump_routes
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.002.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.002.sh
new file mode 100755
index 0000000000..9a32df09f1
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.002.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "missing config, no takeip, ipreallocated"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+required_result 1 <<EOF
+error: CTDB_PER_IP_ROUTING_CONF=/home/martins/samba/ctdb/tests/eventscripts/etc-ctdb/policy_routing file not found
+EOF
+
+simple_test_event "ipreallocated"
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.003.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.003.sh
new file mode 100755
index 0000000000..832101f91d
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.003.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "empty config, ipreallocated"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+touch "$CTDB_PER_IP_ROUTING_CONF"
+
+ok_null
+
+# ipreallocated should add any missing routes
+simple_test_event "ipreallocated"
+
+ok <<EOF
+# ip rule show
+0:	from all lookup local 
+32766:	from all lookup main 
+32767:	from all lookup default 
+EOF
+
+simple_test_command dump_routes
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.004.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.004.sh
new file mode 100755
index 0000000000..49d2b4efa1
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.004.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "empty config, takeip"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+touch "$CTDB_PER_IP_ROUTING_CONF"
+
+public_address=$(ctdb_get_1_public_address)
+
+ok_null
+
+simple_test_event "takeip" $public_address
+
+ok <<EOF
+# ip rule show
+0:	from all lookup local 
+32766:	from all lookup main 
+32767:	from all lookup default 
+EOF
+
+simple_test_command dump_routes
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.005.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.005.sh
new file mode 100755
index 0000000000..3ebacbedbb
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.005.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 IP configured, takeip"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+ctdb_get_1_public_address |
+{
+    read dev ip bits
+
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat >"$CTDB_PER_IP_ROUTING_CONF" <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+
+    ok_null
+
+    simple_test_event "takeip" $dev $ip $bits
+
+    ok <<EOF
+# ip rule show
+0:	from all lookup local 
+${CTDB_PER_IP_ROUTING_RULE_PREF}:	from $ip lookup ctdb.$ip 
+32766:	from all lookup main 
+32767:	from all lookup default 
+# ip route show table ctdb.$ip
+$net dev $dev  scope link 
+default via $gw dev $dev 
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.006.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.006.sh
new file mode 100755
index 0000000000..d458556372
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.006.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 IP configured, takeip, releaseip"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+ctdb_get_1_public_address |
+{
+    read dev ip bits
+
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat >"$CTDB_PER_IP_ROUTING_CONF" <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+
+    ok_null
+
+    simple_test_event "takeip" $dev $ip $bits
+
+    ok_null
+
+    simple_test_event "releaseip" $dev $ip $bits
+
+    ok <<EOF
+# ip rule show
+0:	from all lookup local 
+32766:	from all lookup main 
+32767:	from all lookup default 
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.007.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.007.sh
new file mode 100755
index 0000000000..53e9d3866a
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.007.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 IP configured, ipreallocated"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+ctdb_get_1_public_address |
+{
+    read dev ip bits
+
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat >"$CTDB_PER_IP_ROUTING_CONF" <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+
+    ok_null
+
+    # ipreallocated should add any missing routes
+    simple_test_event "ipreallocated"
+
+    ok <<EOF
+# ip rule show
+0:	from all lookup local 
+${CTDB_PER_IP_ROUTING_RULE_PREF}:	from $ip lookup ctdb.$ip 
+32766:	from all lookup main 
+32767:	from all lookup default 
+# ip route show table ctdb.$ip
+$net dev $dev  scope link 
+default via $gw dev $dev 
+EOF
+
+    simple_test_command dump_routes
+}
+
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.008.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.008.sh
new file mode 100755
index 0000000000..282ec07364
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.008.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 IP configured, takeip twice"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+ctdb_get_1_public_address |
+{
+    read dev ip bits
+
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat >"$CTDB_PER_IP_ROUTING_CONF" <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+
+    ok_null
+
+    simple_test_event "takeip" $dev $ip $bits
+
+    ok_null
+
+    simple_test_event "takeip" $dev $ip $bits
+
+    ok <<EOF
+# ip rule show
+0:	from all lookup local 
+${CTDB_PER_IP_ROUTING_RULE_PREF}:	from $ip lookup ctdb.$ip 
+32766:	from all lookup main 
+32767:	from all lookup default 
+# ip route show table ctdb.$ip
+$net dev $dev  scope link 
+default via $gw dev $dev 
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.009.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.009.sh
new file mode 100755
index 0000000000..771651e94a
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.009.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "All IPs configured, takeip"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+ctdb_get_all_public_addresses |
+while read dev ip bits ; do
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+done >"$CTDB_PER_IP_ROUTING_CONF"
+
+ctdb_get_1_public_address |
+{
+    read dev ip bits
+
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    ok_null
+
+    simple_test_event "takeip" $dev $ip $bits
+
+    ok <<EOF
+# ip rule show
+0:	from all lookup local 
+${CTDB_PER_IP_ROUTING_RULE_PREF}:	from $ip lookup ctdb.$ip 
+32766:	from all lookup main 
+32767:	from all lookup default 
+# ip route show table ctdb.$ip
+$net dev $dev  scope link 
+default via $gw dev $dev 
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.010.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.010.sh
new file mode 100755
index 0000000000..f7caf1c98d
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.010.sh
@@ -0,0 +1,52 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "All IPs configured, takeip all on node"
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+# First setup the policy routing config for all possible IPs
+ctdb_get_all_public_addresses |
+while read dev ip bits ; do
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+done >"$CTDB_PER_IP_ROUTING_CONF"
+
+# Now do a takeip for each IP on the "current" node
+ctdb_get_my_public_addresses |
+{
+    policy_rules=""
+    policy_routes=""
+    while read dev ip bits ; do
+
+	net=$(ipv4_host_addr_to_net "$ip" "$bits")
+	gw="${net%.*}.1" # a dumb, calculated default
+
+	ok_null
+
+	simple_test_event "takeip" $dev $ip $bits
+
+	policy_rules="${policy_rules}
+${CTDB_PER_IP_ROUTING_RULE_PREF}:	from $ip lookup ctdb.$ip "
+	policy_routes="${policy_routes}
+# ip route show table ctdb.$ip
+$net dev $dev  scope link 
+default via $gw dev $dev "
+    done
+
+	ok <<EOF
+# ip rule show
+0:	from all lookup local ${policy_rules}
+32766:	from all lookup main 
+32767:	from all lookup default ${policy_routes}
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.011.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.011.sh
new file mode 100755
index 0000000000..18b51d4b78
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.011.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "__auto_link_local__, takeip all on node"
+
+setup_ctdb
+setup_ctdb_policy_routing
+# Override to do link local fu
+CTDB_PER_IP_ROUTING_CONF="__auto_link_local__"
+
+# Do a takeip for each IP on the "current" node
+ctdb_get_my_public_addresses |
+{
+    policy_rules=""
+    policy_routes=""
+    while read dev ip bits ; do
+
+	net=$(ipv4_host_addr_to_net "$ip" "$bits")
+	gw="${net%.*}.1" # a dumb, calculated default
+
+	ok_null
+
+	simple_test_event "takeip" $dev $ip $bits
+
+	policy_rules="${policy_rules}
+${CTDB_PER_IP_ROUTING_RULE_PREF}:	from $ip lookup ctdb.$ip "
+	policy_routes="${policy_routes}
+# ip route show table ctdb.$ip
+$net dev $dev  scope link "
+    done
+
+	ok <<EOF
+# ip rule show
+0:	from all lookup local ${policy_rules}
+32766:	from all lookup main 
+32767:	from all lookup default ${policy_routes}
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/multievent/13.per_ip_routing.012.sh b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.012.sh
new file mode 100755
index 0000000000..c2720646d6
--- /dev/null
+++ b/ctdb/tests/eventscripts/multievent/13.per_ip_routing.012.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+. "${EVENTSCRIPTS_TESTS_DIR}/common.sh"
+
+define_test "1 IP configured, takeip, releaseip, ipreallocated"
+
+# This partly tests the test infrastructure.  If the (stub) "ctdb
+# moveip" doesn't do anything then the IP being released will still be
+# on the node and the ipreallocated event will add the routes back.
+
+setup_ctdb
+setup_ctdb_policy_routing
+
+ctdb_get_1_public_address |
+{
+    read dev ip bits
+
+    net=$(ipv4_host_addr_to_net "$ip" "$bits")
+    gw="${net%.*}.1" # a dumb, calculated default
+
+    cat >"$CTDB_PER_IP_ROUTING_CONF" <<EOF
+$ip $net
+$ip 0.0.0.0/0 $gw
+EOF
+
+    ok_null
+
+    simple_test_event "takeip" $dev $ip $bits
+
+    ok_null
+
+    ctdb moveip $ip 1
+    simple_test_event "releaseip" $dev $ip $bits
+
+    ok_null
+
+    # This will cause any
+    simple_test_event "ipreallocated"
+
+    ok <<EOF
+# ip rule show
+0:	from all lookup local 
+32766:	from all lookup main 
+32767:	from all lookup default 
+EOF
+
+    simple_test_command dump_routes
+}
diff --git a/ctdb/tests/eventscripts/run_tests.sh b/ctdb/tests/eventscripts/run_tests.sh
index ec17b5f7a7..406689a0d6 100755
--- a/ctdb/tests/eventscripts/run_tests.sh
+++ b/ctdb/tests/eventscripts/run_tests.sh
@@ -7,6 +7,9 @@ export EVENTSCRIPTS_TESTS_DIR=$(pwd)
 
 test_dir=$(dirname "$EVENTSCRIPTS_TESTS_DIR")
 
+export EVENTSCRIPT_TESTS_CAT_RESULTS_OPTS=""
+export EVENTSCRIPT_TESTS_DIFF_RESULTS=false
+
 opts="-d"
 
 for i ; do
@@ -21,6 +24,16 @@ for i ; do
 	    export EVENTSCRIPTS_TESTS_TRACE="sh -x"
 	    shift
 	    ;;
+	-A)
+	    # Useful for detecting whitespace differences in results
+	    export EVENTSCRIPT_TESTS_CAT_RESULTS_OPTS="-A"
+	    shift
+	    ;;
+	-D)
+	    # Useful for detecting whitespace differences in results
+	    export EVENTSCRIPT_TESTS_DIFF_RESULTS=true
+	    shift
+	    ;;
 	-*)
 	    opts="$opts $i"
 	    shift
diff --git a/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh b/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh
index 88ce593b15..6810733b54 100755
--- a/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh
+++ b/ctdb/tests/eventscripts/simple/10.interface.monitor.015.sh
@@ -15,7 +15,7 @@ export CTDB_DELETE_UNEXPECTED_IPS="yes"
 
 ok <<EOF
 WARNING: Removing unmanaged IP address 192.168.253.253/24 from interface dev123
-re-adding secondary address 192.168.254.254/24 to dev dev123
+Re-adding secondary address 192.168.254.254/24 to dev dev123
 WARNING: Removing unmanaged IP address 192.168.254.254/24 from interface dev123
 EOF
 
diff --git a/ctdb/tests/eventscripts/simple/10.interface.takeip.003.sh b/ctdb/tests/eventscripts/simple/10.interface.takeip.003.sh
index 8997d71d76..ce78be90b0 100755
--- a/ctdb/tests/eventscripts/simple/10.interface.takeip.003.sh
+++ b/ctdb/tests/eventscripts/simple/10.interface.takeip.003.sh
@@ -7,6 +7,10 @@ define_test "error - add same IP twice"
 setup_ctdb
 
 public_address=$(ctdb_get_1_public_address)
+dev="${public_address%% *}"
+t="${public_address#* }"
+ip="${t% *}"
+bits="${t#* }"
 
 # This is a bit gross and contrived.  The method of quoting the error
 # message so it makes it to required_result() is horrible.  Hopefully
@@ -14,7 +18,7 @@ public_address=$(ctdb_get_1_public_address)
 
 err2="\
 RTNETLINK answers: File exists
-Failed to add 10.0.0.1/24 on dev dev123"
+Failed to add $ip/$bits on dev $dev"
 
 #EVENTSCRIPTS_TESTS_TRACE="sh -x"
 iterate_test -- $public_address -- 2 "ok_null" \
diff --git a/ctdb/tests/eventscripts/stubs/ctdb b/ctdb/tests/eventscripts/stubs/ctdb
index 155f518ddc..81d275af70 100755
--- a/ctdb/tests/eventscripts/stubs/ctdb
+++ b/ctdb/tests/eventscripts/stubs/ctdb
@@ -22,18 +22,22 @@ not_implemented ()
 }
 
 # Don't set $POSIXLY_CORRECT here.
-_temp=$(getopt -n "$prog" -o "Yvh" -l help -- "$@") || \
+_temp=$(getopt -n "$prog" -o "Yvhn:" -l help -- "$@") || \
     usage
 
 eval set -- "$_temp"
 
 verbose=false
 machine_readable=false
+nodespec=""
+
+args="$*"
 
 while true ; do
     case "$1" in
 	-Y) machine_readable=true ; shift ;;
 	-v) verbose=true ; shift ;;
+	-n) nodespec="$2" ; shift 2 ;;
 	--) shift ; break ;;
 	-h|--help|*) usage ;; # * shouldn't happen, so this is reasonable.
     esac
@@ -55,6 +59,18 @@ setup_pstore ()
     mkdir -p "$pstore_dir"
 }
 
+parse_nodespec ()
+{
+    if [ "$nodespec" = "all" ] ; then
+	nodes="$(seq 0 $((FAKE_CTDB_NUMNODES - 1)) )"
+    elif [ -n "$nodespec" ] ; then
+	nodes="$(echo $nodespec | sed -e 's@,@ @g')"
+    else
+	_t=$(ctdb_pnn)
+	nodes="${_t#PNN:}"
+    fi
+}
+
 # For testing backward compatibility...
 for i in $CTDB_NOT_IMPLEMENTED ; do
     if [ "$i" = "$1" ] ; then
@@ -62,58 +78,145 @@ for i in $CTDB_NOT_IMPLEMENTED ; do
     fi
 done
 
-case "$1" in
-    ip)
-	# NOTE: all nodes share the same public addresses file.
+ctdb_pnn ()
+{
+    # Defaults to 0
+    echo "PNN:${FAKE_CTDB_PNN:-0}"
+}
 
-	# This is completely stateless and IPs move unnecessarily.
-	_f="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
-	if [ -f "$_f" ] ; then
-	    if $verbose ; then
-		echo ":Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:"
+######################################################################
+
+FAKE_CTDB_NODE_STATE="$FAKE_CTDB_STATE/node-state"
+FAKE_CTDB_NODES_DISABLED="$FAKE_CTDB_NODE_STATE/0x4"
+
+######################################################################
+
+# NOTE: all nodes share $CTDB_PUBLIC_ADDRESSES
+
+FAKE_CTDB_IP_LAYOUT="$FAKE_CTDB_STATE/ip-layout"
+
+ip_reallocate ()
+{
+    touch "$FAKE_CTDB_IP_LAYOUT"
 
+    (
+	flock 0
+
+	_pa="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
+
+	if [ ! -s "$FAKE_CTDB_IP_LAYOUT" ] ; then
+	    sed -n -e 's@^\([^#][^/]*\)/.*@\1 -1@p' \
+		"$_pa" >"$FAKE_CTDB_IP_LAYOUT"
+	fi
+
+	_t="${FAKE_CTDB_IP_LAYOUT}.new"
+
+	_flags=""
+	for _i in $(seq 0 $((FAKE_CTDB_NUMNODES - 1)) ) ; do
+	    if ls "$FAKE_CTDB_STATE/node-state/"*"/$_i" >/dev/null 2>&1 ; then
+		# Have non-zero flags
+		_this=0
+		for _j in "$FAKE_CTDB_STATE/node-state/"*"/$_i" ; do
+		    _t="${_j%/*}" # dirname
+		    _f="${_t%/*}" # basename
+		    _this=$(( $_this | $_f ))
+		done
 	    else
-		echo ":Public IP:Node:"
+		_this="0"
 	    fi
-	    # Here IPs are distributed across nodes in a stupid way...
-	    _n=0
-	    while read _ip _ifaces ; do
-		case "_$ip" in
-		    \#*) : ;;
-		    *)
-			# Find a node that is up... but don't loop forever.
-			_orig=$_n
-			while [ -f "${FAKE_CTDB_NODES_DOWN}/${_n}" ] ; do
-			    _n=$(($_n + 1))
-			    if [ _n -eq $FAKE_CTDB_NUMNODES ] ; then
-				_n=0
-			    fi
-			    if [ $_n -eq $_orig ] ; then
-				_n=-1 # Never down!  :-)
-			    fi
-			done
-			if $verbose ; then
-			    # If more than 1 interface, assume all
-			    # addresses are on the 1st.
-			    _first_iface="${_ifaces%%,*}"
-			    # Only show interface if address is on
-			    # this node.
-			    _my_iface=""
-			    if [ "PNN:$_n" = $(ctdb -Y pnn) ]; then
-				_my_iface="$_first_iface"
-			    fi
-			    echo ":${_ip}:${_n}:${_my_iface}:${_first_iface}:${_ifaces}:"
-			else
-			    echo ":${_ip}:${_n}:"
-			fi
-		esac
-	    done <"$_f"
+	    _flags="${_flags}${_flags:+,}${_this}"
+	done
+	"$(dirname ${EVENTSCRIPTS_TESTS_DIR})/bin/ctdb_takeover_tests" \
+	    "ctdb_takeover_run_core" "$_flags" <"$FAKE_CTDB_IP_LAYOUT" |
+	    sort >"$_t"
+	mv "$_t" "$FAKE_CTDB_IP_LAYOUT"
+    ) <"$FAKE_CTDB_IP_LAYOUT"
+}
+
+ctdb_ip ()
+{
+    # If nobody has done any IP-fu then generate a layout.
+    [ -f "$FAKE_CTDB_IP_LAYOUT" ] || ip_reallocate
+
+    if $verbose ; then
+	echo ":Public IP:Node:ActiveInterface:AvailableInterfaces:ConfiguredInterfaces:"
+    else
+	echo ":Public IP:Node:"
+    fi
+
+    _mypnn=$(ctdb_pnn | sed -e 's@PNN:@@')
+
+    # Join public addresses file with $FAKE_CTDB_IP_LAYOUT, and
+    # process output line by line...
+    _pa="${CTDB_PUBLIC_ADDRESSES:-${CTDB_BASE}/public_addresses}"
+    sed -e 's@/@ @' "$_pa" | sort | join - "$FAKE_CTDB_IP_LAYOUT" |
+    while read _ip _bit _ifaces _pnn ; do
+	if $verbose ; then
+	    # If more than 1 interface, assume all addresses are on the 1st.
+	    _first_iface="${_ifaces%%,*}"
+	    # Only show interface if address is on this node.
+	    _my_iface=""
+	    if [ "$_pnn" = "$_mypnn" ]; then
+		_my_iface="$_first_iface"
+	    fi
+	    echo ":${_ip}:${_pnn}:${_my_iface}:${_first_iface}:${_ifaces}:"
+	else
+	    echo ":${_ip}:${_pnn}:"
 	fi
-	;;
-    pnn|xpnn)
-	# Defaults to 0
-	echo "PNN:${FAKE_CTDB_PNN:-0}"
-	;;
+    done
+}
+
+ctdb_moveip ()
+{
+    _ip="$1"
+    _target="$2"
+
+    ip_reallocate  # should be harmless and ensures we have good state
+
+    (
+	flock 0
+
+	_t="${FAKE_CTDB_IP_LAYOUT}.new"
+
+	while read _i _pnn ; do
+	    if [ "$_ip" = "$_i" ] ; then
+		echo "$_ip $_target"
+	    else
+		echo "$_ip $_pnn"
+	    fi
+	done | sort >"$_t"
+	mv "$_t" "$FAKE_CTDB_IP_LAYOUT"
+    ) <"$FAKE_CTDB_IP_LAYOUT"
+}
+
+######################################################################
+
+ctdb_enable ()
+{
+    parse_nodespec
+    
+    for _i in $nodes ; do
+	rm -f "${FAKE_CTDB_NODES_DISABLED}/${_i}"
+    done
+
+    ip_reallocate
+}
+
+ctdb_disable ()
+{
+    parse_nodespec
+
+    for _i in $nodes ; do
+	mkdir -p "$FAKE_CTDB_NODES_DISABLED"
+	touch "${FAKE_CTDB_NODES_DISABLED}/${_i}"
+    done
+
+    ip_reallocate
+}
+
+######################################################################
+
+case "$1" in
     gettickles)
 	setup_tickles
 	echo ":source ip:port:destination ip:port:"
@@ -222,6 +325,11 @@ EOF
 	    echo ":${2:-monitor}:${_b}:${_code}:${_status}:${_d1}:${_d2}:${_err_out}:"
 	done
 	;;
-    *)
-	not_implemented "$1"
+    gratiousarp) : ;;  # Do nothing for now
+    ip)          ctdb_ip "$@" ;;
+    pnn|xpnn)    ctdb_pnn ;;
+    enable)      ctdb_enable "$@";;
+    disable)     ctdb_disable "$@";;
+    moveip)      ctdb_moveip "$@";;
+    *) not_implemented "$1" ;;
 esac
diff --git a/ctdb/tests/eventscripts/stubs/ip b/ctdb/tests/eventscripts/stubs/ip
index d14b06a530..c9b8eb11c7 100755
--- a/ctdb/tests/eventscripts/stubs/ip
+++ b/ctdb/tests/eventscripts/stubs/ip
@@ -8,198 +8,448 @@ not_implemented ()
     exit 127
 }
 
+######################################################################
 
-case "$1" in
-    link)
-	case "$2" in
-	    set)
-		iface="$3"
-		case "$4" in
-		    up)
-			rm -f "${FAKE_IP_STATE}/interfaces-down/${iface}"
-			;;
-		    down)
-			mkdir -p "${FAKE_IP_STATE}/interfaces-down"
-			touch "${FAKE_IP_STATE}/interfaces-down/${iface}"
-			;;
-		    *)
-			not_implemented "$*"
-		esac
+ip_link ()
+{
+    case "$2" in
+	set)
+	    # iface="$3"
+	    case "$4" in
+		up)   ip_link_set_up "$@"  ;;
+		down) ip_link_down_up "$@" ;;
+		*)    not_implemented "$*" ;;
+	    esac
+	    ;;
+	*) not_implemented "$*" ;;
+    esac
+}
+
+ip_link_set_up ()
+{
+    rm -f "${FAKE_IP_STATE}/interfaces-down/$3"
+}
+
+ip_link_set_down ()
+{
+    mkdir -p "${FAKE_IP_STATE}/interfaces-down"
+    touch "${FAKE_IP_STATE}/interfaces-down/$3"
+}
+
+######################################################################
+
+ip_addr ()
+{
+    case "$2" in
+	show|list|"") ip_addr_show "$@" ;;
+	add*)      ip_addr_add  "$@" ;;
+	del*)      ip_addr_del  "$@" ;;
+	*) not_implemented "$*" ;;
+    esac
+}
+
+ip_addr_show ()
+{
+    _args="$*"
+
+    shift 2
+    dev=""
+    primary=true
+    secondary=true
+    while [ -n "$1" ] ; do
+	case "$1" in
+	    dev)
+		dev="$2" ; shift 2
+		;;
+            # Do stupid things and stupid things will happen!
+	    primary)
+		primary=true ; secondary=false ; shift
+		;;
+	    secondary)
+		secondary=true ; primary=false ; shift
 		;;
 	    *)
-		not_implemented "$1 $2"
+	        # Assume an interface name
+		dev="$1" ; shift 1
 	esac
-	
-	;;
-    addr)
-	case "$2" in
-	    add)
-		shift 2
-		local=""
-		dev=""
-		brd=""
-		while [ -n "$1" ] ; do
-		    case "$1" in
-			*.*.*.*/*)
-			    local="$1" ; shift
-			    ;;
-			local)
-			    local="$2" ; shift 2
-			    ;;
-			broadcast|brd)
-			    # For now assume this is always '+'.
-			    if [ "$2" != "+" ] ; then
-				not_implemented "addr add ... brd $2 ..."
-			    fi
-			    shift 2
-			    ;;
-			dev)
-			    dev="$2" ; shift 2
-			    ;;
-			*)
-			    not_implemented "addr add ... $1 ..."
-		    esac
-		done
-		if [ -z "$dev" ] ; then
-		    not_implemented "addr add (without dev)"
-		fi
-		mkdir -p "${FAKE_IP_STATE}/addresses"
-		pf="${FAKE_IP_STATE}/addresses/${dev}-primary"
-		sf="${FAKE_IP_STATE}/addresses/${dev}-secondary"
-		# We could lock here... but we should be the only ones
-		# playing around here with these stubs.
-		if [ ! -f "$pf" ] ; then
-		    echo "$local" >"$pf"
-		elif grep -Fq "$local" "$pf" ; then 
-		    echo "RTNETLINK answers: File exists" >&2
-		    exit 254
-		elif [ -f "$sf" ] && grep -Fq "$local" "$sf" ; then 
-		    echo "RTNETLINK answers: File exists" >&2
-		    exit 254
-		else
-		    echo "$local" >>"$sf"
-		fi
-		;;
-	    delete|del)
-		shift 2
-		local=""
-		dev=""
-		while [ -n "$1" ] ; do
-		    case "$1" in
-			*.*.*.*/*)
-			    local="$1" ; shift
-			    ;;
-			local)
-			    local="$2" ; shift 2
-			    ;;
-			dev)
-			    dev="$2" ; shift 2
-			    ;;
-			*)
-			    not_implemented "addr del ... $1 ..."
-		    esac
-		done
-		if [ -z "$dev" ] ; then
-		    not_implemented "addr del (without dev)"
-		fi
-		mkdir -p "${FAKE_IP_STATE}/addresses"
-		pf="${FAKE_IP_STATE}/addresses/${dev}-primary"
-		sf="${FAKE_IP_STATE}/addresses/${dev}-secondary"
-		# We could lock here... but we should be the only ones
-		# playing around here with these stubs.
-		if [ ! -f "$pf" ] ; then
-		    echo "RTNETLINK answers: Cannot assign requested address" >&2
-		    exit 254
-		elif grep -Fq "$local" "$pf" ; then
-		    # Remove primaries AND SECONDARIES.
-		    rm -f "$pf" "$sf"
-		elif [ -f "$sf" ] && grep -Fq "$local" "$sf" ; then 
-		    grep -Fv "$local" "$sf" >"${sf}.new"
-		    mv "${sf}.new" "$sf"
-		else
-		    echo "RTNETLINK answers: Cannot assign requested address" >&2
-		    exit 254
-		fi
+    done
+    devices="$dev"
+    if [ -z "$devices" ] ; then
+	# No device specified?  Get all the primaries...
+	devices=$(ls "${FAKE_IP_STATE}/addresses/"*-primary 2>/dev/null | \
+	    sed -e 's@.*/@@' -e 's@-primary$@@')
+    fi
+    calc_brd ()
+    {
+	case "${local#*/}" in
+	    24)
+		brd="${local%.*}.255"
 		;;
-	    show|list)
-		shift 2
-		dev=""
-		primary=true
-		secondary=true
-		while [ -n "$1" ] ; do
-		    case "$1" in
-			dev)
-			    dev="$2" ; shift 2
-			    ;;
-			# Do stupid things and stupid things will happen!
-			primary)
-			    primary=true ; secondary=false ; shift
-			    ;;
-			secondary)
-			    secondary=true ; primary=false ; shift
-			    ;;
-			*)
-			    # Assume an interface name
-			    dev="$1" ; shift 1
-		    esac
-		done
-		devices="$dev"
-		if [ -z "$devices" ] ; then
-		    # No device specified?  Get all the primaries...
-		    devices=$(ls "${FAKE_IP_STATE}/addresses/"*-primary 2>/dev/null | \
-			sed -e 's@.*/@@' -e 's@-primary$@@')
-		fi
-		calc_brd ()
-		{
-		    case "${local#*/}" in
-			24)
-			    brd="${local%.*}.255"
-			    ;;
-			*)
-			   not_implemented "list ... fake bits other than 24: ${local#*/}"
-		    esac
-		}
-		show_iface()
-		{
-		    pf="${FAKE_IP_STATE}/addresses/${dev}-primary"
-		    sf="${FAKE_IP_STATE}/addresses/${dev}-secondary"
-		    mac=$(echo $dev | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
-		    cat <<EOF
+	    *)
+		not_implemented "list ... fake bits other than 24: ${local#*/}"
+	esac
+    }
+    show_iface()
+    {
+	pf="${FAKE_IP_STATE}/addresses/${dev}-primary"
+	sf="${FAKE_IP_STATE}/addresses/${dev}-secondary"
+	mac=$(echo $dev | md5sum | sed -r -e 's@(..)(..)(..)(..)(..)(..).*@\1:\2:\3:\4:\5:\6@')
+	cat <<EOF
 ${n}: ${dev}: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
     link/ether ${mac} brd ff:ff:ff:ff:ff:ff
 EOF
-		    if $primary && [ -r "$pf" ] ; then
-			read local <"$pf"
-			calc_brd
-cat <<EOF
+	if $primary && [ -r "$pf" ] ; then
+	    read local <"$pf"
+	    calc_brd
+	    cat <<EOF
     inet ${local} brd ${brd} scope global ${dev}
 EOF
-		    fi
-		    if $secondary && [ -r "$sf" ] ; then
-			while read local ; do
-			    calc_brd
-cat <<EOF
+	fi
+	if $secondary && [ -r "$sf" ] ; then
+	    while read local ; do
+		calc_brd
+		cat <<EOF
     inet ${local} brd ${brd} scope global secondary ${dev}
 EOF
-			done <"$sf"
-		    fi
-cat <<EOF
+	    done <"$sf"
+	fi
+	cat <<EOF
        valid_lft forever preferred_lft forever
 EOF
+    }
+    n=1
+    for dev in $devices ; do
+	show_iface
+	n=$(($n + 1))
+    done
+}
+
+ip_addr_add ()
+{
+    _args="$*"
 
-		}
-		n=1
-		for dev in $devices ; do
-		    show_iface
-		    n=$(($n + 1))
-		done
+    shift 2
+    local=""
+    dev=""
+    brd=""
+    while [ -n "$1" ] ; do
+	case "$1" in
+	    *.*.*.*/*)
+		local="$1" ; shift
+		;;
+	    local)
+		local="$2" ; shift 2
+		;;
+	    broadcast|brd)
+		# For now assume this is always '+'.
+		if [ "$2" != "+" ] ; then
+		    not_implemented "addr add ... brd $2 ..."
+		fi
+		shift 2
+		;;
+	    dev)
+		dev="$2" ; shift 2
 		;;
 	    *)
-		not_implemented "$1 $2"
+		not_implemented "$@"
 	esac
-	;;
-    *)
-	not_implemented "$1"
+    done
+    if [ -z "$dev" ] ; then
+	not_implemented "addr add (without dev)"
+    fi
+    mkdir -p "${FAKE_IP_STATE}/addresses"
+    pf="${FAKE_IP_STATE}/addresses/${dev}-primary"
+    sf="${FAKE_IP_STATE}/addresses/${dev}-secondary"
+    # We could lock here... but we should be the only ones playing
+    # around here with these stubs.
+    if [ ! -f "$pf" ] ; then
+	echo "$local" >"$pf"
+    elif grep -Fq "$local" "$pf" ; then 
+	echo "RTNETLINK answers: File exists" >&2
+	exit 254
+    elif [ -f "$sf" ] && grep -Fq "$local" "$sf" ; then 
+	echo "RTNETLINK answers: File exists" >&2
+	exit 254
+    else
+	echo "$local" >>"$sf"
+    fi
+}
+
+ip_addr_del ()
+{
+    shift 2
+    local=""
+    dev=""
+    while [ -n "$1" ] ; do
+	case "$1" in
+	    *.*.*.*/*)
+		local="$1" ; shift
+		;;
+	    local)
+		local="$2" ; shift 2
+		;;
+	    dev)
+		dev="$2" ; shift 2
+		;;
+	    *)
+		not_implemented "addr del ... $1 ..."
+	esac
+    done
+    if [ -z "$dev" ] ; then
+	not_implemented "addr del (without dev)"
+    fi
+    mkdir -p "${FAKE_IP_STATE}/addresses"
+    pf="${FAKE_IP_STATE}/addresses/${dev}-primary"
+    sf="${FAKE_IP_STATE}/addresses/${dev}-secondary"
+    # We could lock here... but we should be the only ones playing
+    # around here with these stubs.
+    if [ ! -f "$pf" ] ; then
+	echo "RTNETLINK answers: Cannot assign requested address" >&2
+	exit 254
+    elif grep -Fq "$local" "$pf" ; then
+	# Remove primaries AND SECONDARIES.
+	rm -f "$pf" "$sf"
+    elif [ -f "$sf" ] && grep -Fq "$local" "$sf" ; then 
+	grep -Fv "$local" "$sf" >"${sf}.new"
+	mv "${sf}.new" "$sf"
+    else
+	echo "RTNETLINK answers: Cannot assign requested address" >&2
+	exit 254
+    fi
+}
+
+######################################################################
+
+ip_rule ()
+{
+    case "$2" in
+	show|list|"") ip_rule_show "$@" ;;
+	add*)         ip_rule_add  "$@" ;;
+	del*)         ip_rule_del  "$@" ;;
+	*) not_implemented "$2 in \"$*\"" ;;
+    esac
+
+}
+
+# All non-default rules are in $FAKE_IP_STATE_RULES/rules.  As with
+# the real version, rules can be repeated.  Deleting just deletes the
+# 1st match.
+
+ip_rule_show ()
+{
+    ip_rule_show_1 ()
+    {
+	_pre="$1"
+	_table="$2"
+	_selectors="$3"
+	# potentially more options
+
+	printf "%d:\t%s lookup %s \n" $_pre "$_selectors" "$_table"
+    }
+
+    ip_rule_show_some ()
+    {
+	_min="$1"
+	_max="$2"
+
+	[ -f "${FAKE_IP_STATE}/rules" ] || return
+
+	while read _pre _table _selectors ; do
+	    # Only print those in range
+	    [ $_min -le $_pre -a $_pre -le $_max ] || continue
+
+	    ip_rule_show_1 $_pre "$_table" "$_selectors"
+	done <"${FAKE_IP_STATE}/rules"
+    }
+
+    ip_rule_show_1 0 "local" "from all"
+
+    ip_rule_show_some 1 32765
+
+    ip_rule_show_1 32766 "main" "from all"
+    ip_rule_show_1 32767 "default" "from all"
+
+    ip_rule_show_some 32768 2147483648
+}
+
+ip_rule_common ()
+{
+    _args="$*"
+
+    shift 2
+    _from=""
+    _pre=""
+    _table=""
+    while [ -n "$1" ] ; do
+	case "$1" in
+	    from)  _from="$2"  ; shift 2 ;;
+	    pref)  _pre="$2"   ; shift 2 ;;
+	    table) _table="$2" ; shift 2 ;;
+	    *) not_implemented "$1 in \"$_args\"" ;;
+	esac
+    done
+
+    [ -n "$_pre" ]   || not_implemented "ip rule without \"pref\""
+    [ -n "$_table" ] || not_implemented "ip rule without \"table\""
+    # Relax this if more selectors added later...
+    [ -n "$_from" ]  || not_implemented "ip rule without \"from\""
+}
+
+ip_rule_add ()
+{
+    ip_rule_common "$@"
+
+    _f="${FAKE_IP_STATE}/rules"
+    touch "$_f"
+    (
+	flock 0
+	# Filter order must be consistent with the comparison in ip_rule_del()
+	echo "$_pre $_table${_from:+ from }$_from" >>"$_f"
+    ) <"$_f"
+}
+
+ip_rule_del ()
+{
+    ip_rule_common "$@"
+
+    _f="${FAKE_IP_STATE}/rules"
+    touch "$_f"
+    (
+	flock 0
+	_tmp="$(mktemp)"
+	_found=false
+	while read _p _t _s ; do
+	    if ! $_found && \
+		[ "$_p" = "$_pre" -a "$_t" = "$_table" -a \
+		"$_s" = "${_from:+from }$_from" ] ; then
+		# Found.  Skip this one but not future ones.
+		_found=true
+	    else
+		echo "$_p $_t $_s" >>"$_tmp"
+	    fi
+	done
+	if cmp -s "$_tmp" "$_f" ; then
+	    # No changes, must not have found what we wanted to delete
+	    echo "RTNETLINK answers: No such file or directory" >&2
+	    rm -f "$_tmp"
+	    exit 2
+	else
+	    mv "$_tmp" "$_f"
+	fi
+    ) <"$_f" || exit $?
+}
+
+######################################################################
+
+ip_route ()
+{
+    case "$2" in
+	show|list)    ip_route_show  "$@" ;;
+	flush)        ip_route_flush "$@" ;;
+	add)          ip_route_add   "$@" ;;
+	*) not_implemented "$2 in \"$*\"" ;;
+    esac
+}
+
+ip_route_check_table ()
+{
+    [ -n "$_table" ] || not_implemented "ip rule without \"table\""
+
+    # Only allow tables names from 13.per_ip_routing.  This is a cheap
+    # way of avoiding implementing the default/main/local tables.
+    case "$_table" in
+	ctdb.*) : ;;
+	*) not_implemented "table=${_table} in ${_args}" ;;
+    esac
+}
+
+ip_route_common ()
+{
+    _args="$*"
+    shift 2
+    [ "$1" = table ] || not_implemented "$1 in \"$_args\""
+    _table="$2"
+
+    ip_route_check_table
+}
+
+# Routes are in a file per table in the directory
+# $FAKE_IP_STATE/routes.  These routes just use the table ID
+# that is passed and don't do any lookup.  This could be "improved" if
+# necessary.
+
+ip_route_show ()
+{
+    ip_route_common "$@"
+
+    # Missing file is just an empty table
+    cat "$FAKE_IP_STATE/routes/${_table}" 2>/dev/null || true
+}
+
+ip_route_flush ()
+{
+    ip_route_common "$@"
+
+    rm -f "$FAKE_IP_STATE/routes/${_table}"
+}
+
+ip_route_add ()
+{
+    _args="$*"
+
+    shift 2
+
+    _prefix=""
+    _dev=""
+    _gw=""
+    _table=""
+
+    while [ -n "$1" ] ; do
+	case "$1" in
+	    *.*.*.*/*|*.*.*.*) _prefix="$1" ; shift 1 ;;
+	    local) _prefix="$2" ; shift 2 ;;
+	    dev)   _dev="$2"   ; shift 2 ;;
+	    via)   _gw="$2"    ; shift 2 ;;
+	    table) _table="$2" ; shift 2 ;;
+	    *) not_implemented "$1 in \"$_args\"" ;;
+	esac
+    done
+
+    ip_route_check_table
+    [ -n "$_prefix" ] || not_implemented "ip route without inet prefix in \"$_args\""
+    [ -n "$_dev" ] || not_implemented "ip route without \"dev\" in \"$_args\""
+
+    # Alias or add missing bits
+    case "$_prefix" in
+	0.0.0.0/0) _prefix="default" ;;
+	*/*) : ;;
+	*) _prefix="${_prefix}/32" ;;
+    esac
+
+    _f="$FAKE_IP_STATE/routes/${_table}"
+    mkdir -p "$FAKE_IP_STATE/routes"
+    touch "$_f"
+
+    (
+	flock 0
+
+	if [ -n "$_gw" ] ; then
+	    echo "${_prefix} via ${_gw} dev ${_dev} "
+	else
+	    echo "${_prefix} dev ${_dev}  scope link "
+	fi >>"$_f"
+    ) <"$_f"
+}
+
+
+######################################################################
+
+case "$1" in
+    link)   ip_link  "$@" ;;
+    addr*)  ip_addr  "$@" ;;
+    rule)   ip_rule  "$@" ;;
+    route)  ip_route "$@" ;;
+    *) not_implemented "$1" ;;
 esac
 
 exit 0