ctdb-tests: Unit tests for statd-callout

With improvements to unit test infrastructure to support.  This
includes linking the real statd-callout into etc-ctdb/ in place of the
placeholder script.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

11 files changed, 196 insertions, 13 deletions

diff --git a/ctdb/tests/eventscripts/etc-ctdb/statd-callout b/ctdb/tests/eventscripts/etc-ctdb/statd-callout
index 51779bd7ba..dbd3c98791 100755..120000
--- a/ctdb/tests/eventscripts/etc-ctdb/statd-callout
+++ b/ctdb/tests/eventscripts/etc-ctdb/statd-callout
@@ -1,5 +1 @@
-#!/bin/sh
-
-# For now, always succeed.
-
-exit 0
+../../../config/statd-callout
\ No newline at end of file
diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index 163a8fb3ac..fe4ae1eec3 100644
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -538,6 +538,65 @@ EOF
 
 ######################################################################
 
+ctdb_catdb_format_pairs ()
+{
+    _count=0
+
+    while read _k _v ; do
+	_kn=$(echo -n "$_k" | wc -c)
+	_vn=$(echo -n "$_v" | wc -c)
+	cat <<EOF
+key(${_kn}) = "${_k}"
+dmaster: 0
+rsn: 1
+data(${_vn}) = "${_v}"
+
+EOF
+	_count=$(($_count + 1))
+    done
+
+    echo "Dumped ${_count} records"
+}
+
+check_ctdb_tdb_statd_state ()
+{
+    ctdb_get_my_public_addresses |
+    while read _x _sip _x ; do
+	for _cip ; do
+	    echo "statd-state@${_sip}@${_cip}" "$FAKE_DATE_OUTPUT"
+	done
+    done |
+    ctdb_catdb_format_pairs | {
+	ok
+	simple_test_command ctdb catdb ctdb.tdb
+    }
+}
+
+check_statd_callout_smnotify ()
+{
+    _state_even=$(( $(date '+%s') / 2 * 2))
+    _state_odd=$(($_state_even + 1))
+
+    nfs_load_config
+
+    ctdb_get_my_public_addresses |
+    while read _x _sip _x ; do
+	for _cip ; do
+	    cat <<EOF
+--client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_even}
+--client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_even}
+--client=${_cip} --ip=${_sip} --server=${_sip} --stateval=${_state_odd}
+--client=${_cip} --ip=${_sip} --server=${NFS_HOSTNAME} --stateval=${_state_odd}
+EOF
+	done
+    done | {
+	ok
+	simple_test_event "notify"
+    }
+}
+
+######################################################################
+
 setup_ctdb_natgw ()
 {
     debug "Setting up NAT gateway"
@@ -810,6 +869,19 @@ rpc_services_up ()
     done
 }
 
+
+nfs_load_config ()
+{
+    _etc="$CTDB_ETCDIR" # shortcut for readability
+    for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
+	if [ -r "$_c" ] ; then
+	    . "$_c"
+	    break
+	fi
+    done
+}
+
+
 # Set the required result for a particular RPC program having failed
 # for a certain number of iterations.  This is probably still a work
 # in progress.  Note that we could hook aggressively
@@ -826,13 +898,7 @@ rpc_set_service_failure_response ()
     # the flexibility to set the number of failures.
     _numfails="${2:-${iteration}}"
 
-    _etc="$CTDB_ETCDIR" # shortcut for readability
-    for _c in "$_etc/sysconfig/nfs" "$_etc/default/nfs" "$_etc/ctdb/sysconfig/nfs" ; do
-	if [ -r "$_c" ] ; then
-	    . "$_c"
-	    break
-	fi
-    done
+    nfs_load_config
 
     # A handy newline.  :-)
     _nl="
diff --git a/ctdb/tests/eventscripts/statd-callout.001.sh b/ctdb/tests/eventscripts/statd-callout.001.sh
new file mode 100755
index 0000000000..5f7b7e281b
--- /dev/null
+++ b/ctdb/tests/eventscripts/statd-callout.001.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "single add-client"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+check_ctdb_tdb_statd_state "192.168.123.45"
diff --git a/ctdb/tests/eventscripts/statd-callout.002.sh b/ctdb/tests/eventscripts/statd-callout.002.sh
new file mode 100755
index 0000000000..f8778f7e98
--- /dev/null
+++ b/ctdb/tests/eventscripts/statd-callout.002.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "2 x add-client"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+simple_test_event "add-client" "192.168.123.46"
+
+check_ctdb_tdb_statd_state "192.168.123.45" "192.168.123.46"
diff --git a/ctdb/tests/eventscripts/statd-callout.003.sh b/ctdb/tests/eventscripts/statd-callout.003.sh
new file mode 100755
index 0000000000..1319ee4099
--- /dev/null
+++ b/ctdb/tests/eventscripts/statd-callout.003.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "add-client, del-client"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+simple_test_event "del-client" "192.168.123.45"
+
+check_ctdb_tdb_statd_state
diff --git a/ctdb/tests/eventscripts/statd-callout.004.sh b/ctdb/tests/eventscripts/statd-callout.004.sh
new file mode 100755
index 0000000000..5702b85290
--- /dev/null
+++ b/ctdb/tests/eventscripts/statd-callout.004.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "single add-client, notify"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+check_ctdb_tdb_statd_state "192.168.123.45"
+
+check_statd_callout_smnotify "192.168.123.45"
+
+check_ctdb_tdb_statd_state
diff --git a/ctdb/tests/eventscripts/statd-callout.005.sh b/ctdb/tests/eventscripts/statd-callout.005.sh
new file mode 100755
index 0000000000..65e291d088
--- /dev/null
+++ b/ctdb/tests/eventscripts/statd-callout.005.sh
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "2 x add-client to different nodes, notify on 1"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+ok_null
+simple_test_event "add-client" "192.168.123.46"
+
+FAKE_CTDB_PNN=0
+
+check_statd_callout_smnotify "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+check_ctdb_tdb_statd_state "192.168.123.46"
diff --git a/ctdb/tests/eventscripts/statd-callout.006.sh b/ctdb/tests/eventscripts/statd-callout.006.sh
new file mode 100755
index 0000000000..df8af8876b
--- /dev/null
+++ b/ctdb/tests/eventscripts/statd-callout.006.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "2 x add-client to different nodes, notify on both"
+
+setup_ctdb
+
+FAKE_DATE_OUTPUT="1234565789"
+
+ok_null
+simple_test_event "add-client" "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+ok_null
+simple_test_event "add-client" "192.168.123.46"
+
+FAKE_CTDB_PNN=0
+
+check_statd_callout_smnotify "192.168.123.45"
+
+FAKE_CTDB_PNN=1
+
+check_statd_callout_smnotify "192.168.123.46"
+
+check_ctdb_tdb_statd_state
diff --git a/ctdb/tests/eventscripts/stubs/id b/ctdb/tests/eventscripts/stubs/id
new file mode 100755
index 0000000000..1ecd2f80bf
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/id
@@ -0,0 +1,3 @@
+#!/bin/sh
+# Make statd-callout happy
+echo 0
diff --git a/ctdb/tests/eventscripts/stubs/smnotify b/ctdb/tests/eventscripts/stubs/smnotify
new file mode 100755
index 0000000000..414fa99a89
--- /dev/null
+++ b/ctdb/tests/eventscripts/stubs/smnotify
@@ -0,0 +1,3 @@
+#!/bin/sh
+# echo args
+echo "$*"
diff --git a/ctdb/wscript b/ctdb/wscript
index 1400685404..07651f1c7c 100755
--- a/ctdb/wscript
+++ b/ctdb/wscript
@@ -628,7 +628,8 @@ def build(bld):
     test_eventscript_links = [
         'events.d',
         'functions',
-        'nfs-rpc-checks.d'
+        'nfs-rpc-checks.d',
+        'statd-callout'
     ]
 
     test_link_dir = os.path.join(bld.env.CTDB_TEST_DATADIR,