Tests: add initial onnode tests

Add some simple tests for the onnode command.  These use fake ssh and
ctdb commands that are added to $PATH.  The infrastructure used is
quite flexible and would allow more complex tests to be written.

As-is, these tests expose some bugs in the an older version of onnode
that is included so it can be used to validate some of the tests.

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit f7f9d0943474cb2de7832d7ca95210ea9e9c772b)

21 files changed, 873 insertions, 0 deletions

diff --git a/ctdb/tests/onnode/0001.sh b/ctdb/tests/onnode/0001.sh
new file mode 100755
index 0000000000..6c86ac1212
--- /dev/null
+++ b/ctdb/tests/onnode/0001.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE all hostname"
+
+echo "$cmd - all nodes OK"
+
+required_result <<EOF
+
+>> NODE: 192.168.1.101 <<
+-n 192.168.1.101 hostname
+
+>> NODE: 192.168.1.102 <<
+-n 192.168.1.102 hostname
+
+>> NODE: 192.168.1.103 <<
+-n 192.168.1.103 hostname
+
+>> NODE: 192.168.1.104 <<
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0002.sh b/ctdb/tests/onnode/0002.sh
new file mode 100755
index 0000000000..e8e082daec
--- /dev/null
+++ b/ctdb/tests/onnode/0002.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE -q all hostname"
+
+echo "$cmd - all nodes OK"
+
+required_result <<EOF
+-n 192.168.1.101 hostname
+-n 192.168.1.102 hostname
+-n 192.168.1.103 hostname
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0003.sh b/ctdb/tests/onnode/0003.sh
new file mode 100755
index 0000000000..7e55c6015f
--- /dev/null
+++ b/ctdb/tests/onnode/0003.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE -p all hostname"
+
+echo "$cmd - all nodes OK"
+
+required_result <<EOF
+[192.168.1.101] -n 192.168.1.101 hostname
+[192.168.1.102] -n 192.168.1.102 hostname
+[192.168.1.103] -n 192.168.1.103 hostname
+[192.168.1.104] -n 192.168.1.104 hostname
+EOF
+
+simple_test -s $cmd
diff --git a/ctdb/tests/onnode/0004.sh b/ctdb/tests/onnode/0004.sh
new file mode 100755
index 0000000000..daa6794811
--- /dev/null
+++ b/ctdb/tests/onnode/0004.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE -pq all hostname"
+
+echo "$cmd - all nodes OK"
+
+required_result <<EOF
+-n 192.168.1.101 hostname
+-n 192.168.1.102 hostname
+-n 192.168.1.103 hostname
+-n 192.168.1.104 hostname
+EOF
+
+simple_test -s $cmd
diff --git a/ctdb/tests/onnode/0005.sh b/ctdb/tests/onnode/0005.sh
new file mode 100755
index 0000000000..bb82166f3c
--- /dev/null
+++ b/ctdb/tests/onnode/0005.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE 3 hostname"
+
+echo "$cmd - all nodes OK"
+
+required_result <<EOF
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0006.sh b/ctdb/tests/onnode/0006.sh
new file mode 100755
index 0000000000..2960061d46
--- /dev/null
+++ b/ctdb/tests/onnode/0006.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE -v 3 hostname"
+
+echo "$cmd - all nodes OK"
+
+required_result <<EOF
+
+>> NODE: 192.168.1.104 <<
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0070.sh b/ctdb/tests/onnode/0070.sh
new file mode 100755
index 0000000000..758298124f
--- /dev/null
+++ b/ctdb/tests/onnode/0070.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE ok hostname"
+
+echo "$cmd - all nodes OK"
+
+ctdb_set_output <<EOF
+:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:
+:0:192.168.1.101:0:0:0:0:0:0:
+:1:192.168.1.102:0:0:0:0:0:0:
+:2:192.168.1.103:0:0:0:0:0:0:
+:3:192.168.1.104:0:0:0:0:0:0:
+EOF
+
+required_result <<EOF
+
+>> NODE: 192.168.1.101 <<
+-n 192.168.1.101 hostname
+
+>> NODE: 192.168.1.102 <<
+-n 192.168.1.102 hostname
+
+>> NODE: 192.168.1.103 <<
+-n 192.168.1.103 hostname
+
+>> NODE: 192.168.1.104 <<
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0071.sh b/ctdb/tests/onnode/0071.sh
new file mode 100755
index 0000000000..3c0a7748e9
--- /dev/null
+++ b/ctdb/tests/onnode/0071.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE ok hostname"
+
+echo "$cmd - 2nd node disconnected"
+
+ctdb_set_output  <<EOF 
+ctdb_set_output <<EOF
+:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:
+:0:192.168.1.101:0:0:0:0:0:0:
+:1:192.168.1.102:1:0:0:0:0:0:
+:2:192.168.1.103:0:0:0:0:0:0:
+:3:192.168.1.104:0:0:0:0:0:0:
+EOF
+
+required_result <<EOF
+
+>> NODE: 192.168.1.101 <<
+-n 192.168.1.101 hostname
+
+>> NODE: 192.168.1.103 <<
+-n 192.168.1.103 hostname
+
+>> NODE: 192.168.1.104 <<
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0072.sh b/ctdb/tests/onnode/0072.sh
new file mode 100755
index 0000000000..a8c7d07329
--- /dev/null
+++ b/ctdb/tests/onnode/0072.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE ok hostname"
+
+echo "$cmd - 2nd node disconnected, extra status columns"
+
+ctdb_set_output <<EOF
+:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:X1:X2:X3:X4:
+:0:192.168.1.101:0:0:0:0:0:0:0:0:0:0:
+:1:192.168.1.102:1:0:0:0:0:0:0:0:0:0:
+:2:192.168.1.103:0:0:0:0:0:0:0:0:0:0:
+:3:192.168.1.104:0:0:0:0:0:0:0:0:0:0:
+EOF
+
+required_result <<EOF
+
+>> NODE: 192.168.1.101 <<
+-n 192.168.1.101 hostname
+
+>> NODE: 192.168.1.103 <<
+-n 192.168.1.103 hostname
+
+>> NODE: 192.168.1.104 <<
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0075.sh b/ctdb/tests/onnode/0075.sh
new file mode 100755
index 0000000000..8315f78dbf
--- /dev/null
+++ b/ctdb/tests/onnode/0075.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE con hostname"
+
+echo "$cmd - 1st node disconnected"
+
+ctdb_set_output <<EOF
+:Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:
+:0:192.168.1.101:1:0:0:0:0:0:
+:1:192.168.1.102:0:0:0:0:0:0:
+:2:192.168.1.103:0:0:0:0:0:0:
+:3:192.168.1.104:0:0:0:0:0:0:
+EOF
+
+required_result <<EOF
+
+>> NODE: 192.168.1.102 <<
+-n 192.168.1.102 hostname
+
+>> NODE: 192.168.1.103 <<
+-n 192.168.1.103 hostname
+
+>> NODE: 192.168.1.104 <<
+-n 192.168.1.104 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0080.sh b/ctdb/tests/onnode/0080.sh
new file mode 100755
index 0000000000..f04155455d
--- /dev/null
+++ b/ctdb/tests/onnode/0080.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE recmaster hostname"
+
+echo "$cmd - node 1 (192.168.1.102) is recmaster"
+
+ctdb_set_output <<EOF
+1
+EOF
+
+required_result <<EOF
+-n 192.168.1.102 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0081.sh b/ctdb/tests/onnode/0081.sh
new file mode 100755
index 0000000000..c31af289d6
--- /dev/null
+++ b/ctdb/tests/onnode/0081.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE lvsmaster hostname"
+
+echo "$cmd - no lvsmaster"
+
+ctdb_set_output 255 <<EOF
+There is no LVS master
+EOF
+
+required_result 1 <<EOF
+onnode: No lvsmaster available
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0090.sh b/ctdb/tests/onnode/0090.sh
new file mode 100755
index 0000000000..c64c1587c9
--- /dev/null
+++ b/ctdb/tests/onnode/0090.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE natgw hostname"
+
+echo "$cmd - no natgw"
+
+ctdb_set_output <<EOF
+-1 0.0.0.0
+:0:192.168.1.101:0:0:0:0:0:
+:1:192.168.1.102:0:0:0:0:0:
+:2:192.168.1.103:0:0:0:0:0:
+:3:192.168.1.104:0:0:0:0:0:
+EOF
+
+required_result 1 <<EOF
+onnode: No natgwlist available
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/0091.sh b/ctdb/tests/onnode/0091.sh
new file mode 100755
index 0000000000..947760647a
--- /dev/null
+++ b/ctdb/tests/onnode/0091.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+. "${ONNODE_TESTS_DIR}/common.sh"
+
+cmd="$ONNODE natgw hostname"
+
+echo "$cmd - node 2 (192.168.1.103) is natgw"
+
+ctdb_set_output <<EOF
+2 192.168.1.103
+:0:192.168.1.101:0:0:0:0:0:
+:1:192.168.1.102:0:0:0:0:0:
+:2:192.168.1.103:0:0:0:0:0:
+:3:192.168.1.104:0:0:0:0:0:
+EOF
+
+required_result <<EOF
+-n 192.168.1.103 hostname
+EOF
+
+simple_test $cmd
diff --git a/ctdb/tests/onnode/README b/ctdb/tests/onnode/README
new file mode 100644
index 0000000000..1bbf7af5a5
--- /dev/null
+++ b/ctdb/tests/onnode/README
@@ -0,0 +1,38 @@
+onnode unit tests
+=================
+
+Examples:
+
+* ./run_tests.sh
+
+  Run all tests, displaying output.
+
+* ./run_tests.sh -s
+
+  Run all tests, displaying output and a summary.
+
+* ./run_tests.sh -sq
+
+  Run all tests, displaying only a summary.
+
+* ONNODE=onnode-buggy-001 ./run_tests.sh -s
+
+  Run against bin/onnode-buggy-001 instead of default onnode version.
+
+  Add more buggy versions of onnode to this directory as bugs are
+  fixed to enable test validation using this feature.
+
+* ./run_tests.sh ./009*.sh
+
+  Run only the specified tests.
+
+* ONNODE="bash -x bin/onnode-buggy-001" ./run_tests.sh ./0090.sh
+  ONNODE="bash -x ../../tools/onnode" ./run_tests.sh ./0090.sh
+
+  Debug the specified test or test failure.  The test will fail
+  because the bash trace output will be included in the test output.
+  However, this at least makes it easy to trace onnode while running
+  the test...
+
+  To see if the test pases, the -x can be dropped... so command-line
+  editing can be kept to a minimum.
diff --git a/ctdb/tests/onnode/bin/ctdb b/ctdb/tests/onnode/bin/ctdb
new file mode 100755
index 0000000000..e420d25e02
--- /dev/null
+++ b/ctdb/tests/onnode/bin/ctdb
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# Fake ctdb client for onnode tests.
+
+cmd=$(echo "$*" | sed -r -e 's@[[:space:]]+@_@g')
+
+out="${ONNODE_TESTS_VAR_DIR}/ctdb.out"
+if [ -r "$out" ] ; then
+    cat "$out"
+
+    rc="${ONNODE_TESTS_VAR_DIR}/ctdb.rc"
+    if [ -r "$rc" ] ; then
+	exit $(cat "$rc")
+    fi
+
+    exit 0
+fi
+
+f="${ONNODE_TESTCASE_DIR}/ctdb.d/${cmd}.sh"
+if [ -x "$f" ] ; then
+    "$f"
+    exit $?
+fi
+
+f="${ONNODE_TESTCASE_DIR}/ctdb.d/${cmd}.out"
+if [ -r "$f" ] ; then
+    cat "$f"
+    exit 0
+fi
+
+echo "fake ctdb: no implementation for \"$*\""
+
+exit 1
diff --git a/ctdb/tests/onnode/bin/onnode-buggy-001 b/ctdb/tests/onnode/bin/onnode-buggy-001
new file mode 100755
index 0000000000..77a1207d6c
--- /dev/null
+++ b/ctdb/tests/onnode/bin/onnode-buggy-001
@@ -0,0 +1,376 @@
+#!/bin/bash
+
+# Run commands on CTDB nodes.
+
+# See http://ctdb.samba.org/ for more information about CTDB.
+
+# Copyright (C) Martin Schwenke  2008
+
+# Based on an earlier script by Andrew Tridgell and Ronnie Sahlberg.
+
+# Copyright (C) Andrew Tridgell  2007
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+   
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+   
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+prog=$(basename $0)
+
+usage ()
+{
+    cat >&2 <<EOF
+Usage: onnode [OPTION] ... <NODES> <COMMAND> ...
+  options:
+    -c          Run in current working directory on specified nodes.
+    -o <prefix> Save standard output from each node to file <prefix>.<ip>
+    -p          Run command in parallel on specified nodes.
+    -q          Do not print node addresses (overrides -v).
+    -n          Allow nodes to be specified by name.
+    -f          Specify nodes file, overrides CTDB_NODES_FILE.
+    -v          Print node address even for a single node.
+  <NODES>       "all", "any", "ok" (or "healthy"), "con" (or "connected"),
+                "rm" (or "recmaster"), "lvs" (or "lvsmaster"),
+                "natgw" (or "natgwlist"); or
+                a node number (0 base); or
+                a hostname (if -n is specified); or
+                list (comma separated) of <NODES>; or
+                range (hyphen separated) of node numbers.
+EOF
+    exit 1
+
+}
+
+invalid_nodespec ()
+{
+    echo "Invalid <nodespec>" >&2 ; echo >&2
+    usage
+}
+
+# Defaults.
+current=false
+parallel=false
+verbose=false
+quiet=false
+prefix=""
+names_ok=false
+
+ctdb_base="${CTDB_BASE:-/etc/ctdb}"
+
+parse_options ()
+{
+    # $POSIXLY_CORRECT means that the command passed to onnode can
+    # take options and getopt won't reorder things to make them
+    # options ot onnode.
+    local temp
+    # Not on the previous line - local returns 0!
+    temp=$(POSIXLY_CORRECT=1 getopt -n "$prog" -o "cf:hno:pqv" -l help -- "$@")
+
+    [ $? != 0 ] && usage
+
+    eval set -- "$temp"
+
+    while true ; do
+	case "$1" in
+	    -c) current=true ; shift ;;
+	    -f) CTDB_NODES_FILE="$2" ; shift 2 ;;
+	    -n) names_ok=true ; shift ;;
+	    -o) prefix="$2" ; shift 2 ;;
+	    -p) parallel=true ; shift ;;
+	    -q) quiet=true ; shift ;;
+	    -v) verbose=true ; shift ;;
+	    --) shift ; break ;;
+	    -h|--help|*) usage ;; # Shouldn't happen, so this is reasonable.
+	esac
+    done
+
+    [ $# -lt 2 ] && usage
+
+    nodespec="$1" ; shift
+    command="$@"
+}
+
+echo_nth ()
+{
+    local n="$1" ; shift
+
+    shift $n
+    local node="$1"
+
+    if [ -n "$node" -a "$node" != "#DEAD" ] ; then
+	echo $node
+    else
+	echo "${prog}: \"node ${n}\" does not exist" >&2
+	exit 1
+    fi
+}
+
+parse_nodespec ()
+{
+    # Subshell avoids hacks to restore $IFS.
+    (
+	IFS=","
+	for i in $1 ; do
+	    case "$i" in
+		*-*) seq "${i%-*}" "${i#*-}" 2>/dev/null || invalid_nodespec ;;
+		# Separate lines for readability.
+		all|any|ok|healthy|con|connected) echo "$i" ;;
+		rm|recmaster|lvs|lvsmaster|natgw|natgwlist) echo "$i" ;;
+		*)
+		    [ $i -gt -1 ] 2>/dev/null || $names_ok || invalid_nodespec
+		    echo $i
+	    esac
+	done
+    )
+}
+
+ctdb_status_output="" # cache
+get_nodes_with_status ()
+{
+    local all_nodes="$1"
+    local status="$2"
+
+    local bits
+    case "$status" in
+	healthy)
+	    bits="0:0:0:0:0:0"
+	    ;;
+	connected)
+	    bits="0:[0-1]:[0-1]:[0-1]:[0-1]:[0-1]"
+	    ;;
+	*)
+	    invalid_nodespec
+    esac
+
+    if [ -z "$ctdb_status_output" ] ; then
+	# FIXME: need to do something if $CTDB_NODES_SOCKETS is set.
+	ctdb_status_output=$(ctdb -Y status 2>/dev/null)
+	if [ $? -ne 0 ] ; then
+	    echo "${prog}: unable to get status of CTDB nodes" >&2
+	    exit 1
+	fi
+	ctdb_status_output="${ctdb_status_output#* }"
+    fi
+
+    local nodes=""
+    local i
+    for i in $ctdb_status_output ; do
+	# Try removing bits from end.
+	local t="${i%:${bits}:}"
+	if [ "$t" != "$i" ] ; then
+	    # Succeeded.  Get address.  NOTE: this is an optimisation.
+	    # It might be better to get the node number and then get
+	    # the nth node to get the address.  This would make things
+	    # more consistent if $ctdb_base/nodes actually contained
+	    # hostnames.
+	    nodes="${nodes} ${t#:*:}"
+	fi
+    done
+
+    echo $nodes
+}
+
+ctdb_props="" # cache
+get_node_with_property ()
+{
+    local all_nodes="$1"
+    local prop="$2"
+
+    local prop_node=""
+    if [ "${ctdb_props##:${prop}:}" = "$ctdb_props" ] ; then
+	prop_node=$(ctdb "$prop" -Y 2>/dev/null)
+	# We only want the first line.
+	local nl="
+"
+	prop_node="${prop_node%%${nl}*}"
+	if [ $? -eq 0 ] ; then
+	    ctdb_props="${ctdb_props}${ctdb_props:+ }:${prop}:${prop_node}"
+	else
+	    prop_node=""
+	fi
+    else
+	prop_node="${ctdb_props##:${prop}:}"
+	prop_node="${prop_node%% *}"
+    fi
+    if [ -n "$prop_node" ] ; then
+	echo_nth "$prop_node" $all_nodes
+    else
+	echo "${prog}: No ${prop} available" >&2
+	exit 1
+    fi
+}
+
+get_any_available_node ()
+{
+    local all_nodes="$1"
+
+    # We do a recursive onnode to find which nodes are up and running.
+    local out=$($0 -pq all ctdb pnn 2>&1)
+    local line
+    while read line ; do 
+	local pnn="${line#PNN:}"
+	if [ "$pnn" != "$line" ] ; then
+	    echo_nth "$pnn" $all_nodes
+	    return 0
+	fi
+	# Else must be an error message from a down node.
+    done <<<"$out"
+    return 1
+}
+
+get_nodes ()
+{
+    local all_nodes
+
+    if [ -n "$CTDB_NODES_SOCKETS" ] ; then 
+	all_nodes="$CTDB_NODES_SOCKETS"
+    else
+	local f="${ctdb_base}/nodes"
+	if [ -n "$CTDB_NODES_FILE" ] ; then
+	    f="$CTDB_NODES_FILE"
+	    if [ ! -e "$f" -a "${f#/}" = "$f" ] ; then
+		# $f is relative, try in $ctdb_base
+		f="${ctdb_base}/${f}"
+	    fi
+	fi
+
+	if [ ! -r "$f" ] ; then
+	    echo "${prog}: unable to open nodes file  \"${f}\"" >&2
+	    exit 1
+	fi
+
+	all_nodes=$(sed -e 's@#.*@@g' -e 's@ *@@g' -e 's@^$@#DEAD@' "$f")
+    fi
+
+    local nodes=""
+    local n
+    for n in $(parse_nodespec "$1") ; do
+	[ $? != 0 ] && exit 1  # Required to catch exit in above subshell.
+	case "$n" in
+	    all)
+		echo "${all_nodes//#DEAD/}"
+		;;
+	    any)
+		get_any_available_node "$all_nodes" || exit 1
+		;;
+	    ok|healthy) 
+		get_nodes_with_status "$all_nodes" "healthy" || exit 1
+		;;
+	    con|connected) 
+		get_nodes_with_status "$all_nodes" "connected" || exit 1
+		;;
+	    rm|recmaster)
+		get_node_with_property "$all_nodes" "recmaster" || exit 1
+		;;
+	    lvs|lvsmaster)
+		get_node_with_property "$all_nodes" "lvsmaster" || exit 1
+		;;
+	    natgw|natgwlist)
+		get_node_with_property "$all_nodes" "natgwlist" || exit 1
+		;;
+	    [0-9]|[0-9][0-9]|[0-9][0-9][0-9])
+		echo_nth $n $all_nodes
+		;;
+	    *)
+		$names_ok || invalid_nodespec
+		echo $n
+	esac
+    done
+}
+
+fakessh ()
+{
+    CTDB_SOCKET="$1" sh -c "$2" 3>/dev/null
+}
+
+stdout_filter ()
+{
+    if [ -n "$prefix" ] ; then
+	cat >"${prefix}.${n//\//_}"
+    elif $verbose && $parallel ; then
+	sed -e "s@^@[$n] @"
+    else
+	cat
+    fi
+}
+
+stderr_filter ()
+{
+    if $verbose && $parallel ; then
+	sed -e "s@^@[$n] @"
+    else
+	cat
+    fi
+}
+
+######################################################################
+
+parse_options "$@"
+
+$current && command="cd $PWD && $command"
+
+ssh_opts=
+if [ -n "$CTDB_NODES_SOCKETS" ] ; then
+    SSH=fakessh
+else 
+    # Could "2>/dev/null || true" but want to see errors from typos in file.
+    [ -r "${ctdb_base}/onnode.conf" ] && . "${ctdb_base}/onnode.conf"
+    [ -n "$SSH" ] || SSH=ssh
+    if [ "$SSH" = "ssh" ] ; then
+	ssh_opts="-n"
+    else
+	: # rsh? All bets are off!
+    fi
+fi
+
+######################################################################
+
+nodes=$(get_nodes "$nodespec")
+[ $? != 0 ] && exit 1   # Required to catch exit in above subshell.
+
+if $quiet ; then
+    verbose=false
+else
+    # If $nodes contains a space or a newline then assume multiple nodes.
+    nl="
+"
+    [ "$nodes" != "${nodes%[ ${nl}]*}" ] && verbose=true
+fi
+
+pids=""
+trap 'kill -TERM $pids 2>/dev/null' INT TERM
+# There's a small race here where the kill can fail if no processes
+# have been added to $pids and the script is interrupted.  However,
+# the part of the window where it matter is very small.
+retcode=0
+for n in $nodes ; do
+    set -o pipefail 2>/dev/null
+    if $parallel ; then
+	{ exec 3>&1 ; { $SSH $ssh_opts $EXTRA_SSH_OPTS $n "$command" | stdout_filter >&3 ; } 2>&1 | stderr_filter ; } &
+	pids="${pids} $!"
+    else
+	if $verbose ; then
+	    echo >&2 ; echo ">> NODE: $n <<" >&2
+	fi
+
+	{ exec 3>&1 ; { $SSH $ssh_opts $EXTRA_SSH_OPTS $n "$command" | stdout_filter >&3 ; } 2>&1 | stderr_filter ; }
+	[ $? = 0 ] || retcode=$?
+    fi
+done
+
+$parallel && {
+    for p in $pids; do
+	wait $p
+	[ $? = 0 ] || retcode=$?
+    done
+}
+
+exit $retcode
diff --git a/ctdb/tests/onnode/bin/ssh b/ctdb/tests/onnode/bin/ssh
new file mode 100755
index 0000000000..7be778f1b9
--- /dev/null
+++ b/ctdb/tests/onnode/bin/ssh
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "$*"
diff --git a/ctdb/tests/onnode/common.sh b/ctdb/tests/onnode/common.sh
new file mode 100644
index 0000000000..bbdbce4777
--- /dev/null
+++ b/ctdb/tests/onnode/common.sh
@@ -0,0 +1,93 @@
+# Hey Emacs, this is a -*- shell-script -*- !!!  :-)
+
+# Set indirectly by run_tests at top level.
+unset CTDB_NODES_SOCKETS
+
+# Default to just "onnode".
+: ${ONNODE:=onnode}
+
+# Augment PATH with relevant bin/ directories.
+
+if [ -d "${ONNODE_TESTS_DIR}/bin" ] ; then
+    PATH="${ONNODE_TESTS_DIR}/bin:$PATH"
+fi
+
+export ONNODE_TESTCASE_DIR=$(dirname "$0")
+if [ $(basename "$ONNODE_TESTCASE_DIR") = "onnode" ] ; then
+    # Just a test script, no testcase subdirectory.
+    ONNODE_TESTCASE_DIR="$ONNODE_TESTS_DIR"
+else
+    if [ -d "${ONNODE_TESTCASE_DIR}/bin" ] ; then
+	PATH="${ONNODE_TESTCASE_DIR}/bin:$PATH"
+    fi
+fi
+
+# Find CTDB nodes file.
+if [ -z "$CTDB_NODES_FILE" ] ; then
+    if [ -r "${ONNODE_TESTCASE_DIR}/nodes" ] ; then
+	CTDB_NODES_FILE="${ONNODE_TESTCASE_DIR}/nodes"
+    elif [ -r "${ONNODE_TESTS_DIR}/nodes" ] ; then
+	CTDB_NODES_FILE="${ONNODE_TESTS_DIR}/nodes"
+    else
+	CTDB_NODES_FILE="${CTDB_BASE:-/etc/ctdb}/nodes"
+    fi
+fi
+
+export CTDB_NODES_FILE
+
+export ONNODE_TESTS_VAR_DIR="${ONNODE_TESTS_DIR}/var"
+mkdir -p "$ONNODE_TESTS_VAR_DIR"
+
+if [ -z "$CTDB_BASE" ] ; then
+    export CTDB_BASE=$(dirname "$CTDB_NODES_FILE")
+fi
+
+# Set output for ctdb command.  Option 1st argument is return code.
+ctdb_set_output ()
+{
+    _out="$ONNODE_TESTS_VAR_DIR/ctdb.out"
+    cat >"$_out"
+
+    _rc="$ONNODE_TESTS_VAR_DIR/ctdb.rc"
+    echo "${1:-0}" >"$_rc"
+
+    trap "rm -f $_out $_rc" 0
+}
+
+required_result ()
+{
+    required_rc="${1:-0}"
+    required_output=$(cat)
+}
+
+simple_test ()
+{
+    _sort="cat"
+    if [ "$1" = "-s" ] ; then
+	shift
+	_sort="sort"
+    fi
+    _out=$("$@" 2>&1)
+    _rc=$?
+    _out=$(echo "$_out" | $_sort )
+
+    if [ "$_out" = "$required_output" -a $_rc = $required_rc ] ; then
+	echo "PASSED"
+    else
+	cat <<EOF
+CTDB_NODES_FILE="${CTDB_NODES_FILE}"
+CTDB_BASE="$CTDB_BASE"
+$(which ctdb)
+
+##################################################
+Required output (Exit status: ${required_rc}):
+##################################################
+$required_output
+##################################################
+Actual output (Exit status: ${_rc}):
+##################################################
+$_out
+EOF
+	return 1
+    fi
+}
diff --git a/ctdb/tests/onnode/nodes b/ctdb/tests/onnode/nodes
new file mode 100644
index 0000000000..e2fe268e8d
--- /dev/null
+++ b/ctdb/tests/onnode/nodes
@@ -0,0 +1,4 @@
+192.168.1.101
+192.168.1.102
+192.168.1.103
+192.168.1.104
diff --git a/ctdb/tests/onnode/run_tests.sh b/ctdb/tests/onnode/run_tests.sh
new file mode 100755
index 0000000000..e5fa7e28c2
--- /dev/null
+++ b/ctdb/tests/onnode/run_tests.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Run some onnode unit tests.
+
+cd $(dirname "$0")
+export ONNODE_TESTS_DIR=$(pwd)
+
+test_dir=$(dirname "$ONNODE_TESTS_DIR")
+
+opts="-d"
+
+for i ; do
+    case "$i" in
+	-*)
+	    opts="$opts $i"
+	    shift
+	    ;;
+	*)
+	    break
+    esac
+done
+
+tests=""
+if [ -z "$*" ] ; then
+    tests=$(ls ./[0-9][0-9][0-9][0-9].sh ./[0-9][0-9][0-9][0-9]/run_test.sh 2>/dev/null)
+fi
+
+"$test_dir/scripts/run_tests" $opts "$@" $tests || exit 1
+
+echo "All OK"
+exit 0