Move tests/*.c to tests/src/*.c and adjust Makefile.in accordingly.

Move setting of $CTDB_NODES_SOCKETS to tests/scripts/run_tests and
make it only happen if $CTDB_TEST_REAL_CLUSTER is not set.  Bugfix in
function ips_are_on_nodeglob.  New/proper implementations of functions
stop_daemons and start_daemons, now called by function restart_ctdb.
In start_daemons.sh, add public addresses file generation/usage, use
new option --nopublicipcheck to ctdbd to avoid crazy behaviour and
kill ctdbd more carefully to avoid killing real daemons on a real
cluster - this should be able to coexist on a node of a real cluster.
start_daemons.sh is temporarily incompatible with start_daemons
function, but expecting to replace that script with function calls
very soon anyway...

Signed-off-by: Martin Schwenke <martin@meltin.net>

(This used to be ctdb commit 4c54772c5c2fa7d2a25963379b5b96caf0c2521c)

15 files changed, 102 insertions, 44 deletions

diff --git a/ctdb/.gitignore b/ctdb/.gitignore
index 91124db066..b84c676909 100644
--- a/ctdb/.gitignore
+++ b/ctdb/.gitignore
@@ -18,4 +18,4 @@ nodes.txt
 public_addresses.txt
 rec.lock
 test.db
-var
+tests/var
diff --git a/ctdb/Makefile.in b/ctdb/Makefile.in
index 779bd8b4b0..212fa69b15 100755
--- a/ctdb/Makefile.in
+++ b/ctdb/Makefile.in
@@ -114,37 +114,37 @@ utils/smnotify/gen_smnotify.c: utils/smnotify/smnotify.x utils/smnotify/smnotify
 	@echo Generating $@
 	rpcgen -l utils/smnotify/smnotify.x > utils/smnotify/gen_smnotify.c 
 
-bin/rb_test: $(CTDB_CLIENT_OBJ) tests/rb_test.o 
+bin/rb_test: $(CTDB_CLIENT_OBJ) tests/src/rb_test.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/rb_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/rb_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/ctdb_bench.o 
+bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/src/ctdb_bench.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/ctdb_fetch.o 
+bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/src/ctdb_fetch.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/ctdb_store.o 
+bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/src/ctdb_store.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_traverse: $(CTDB_CLIENT_OBJ) tests/ctdb_traverse.o 
+bin/ctdb_traverse: $(CTDB_CLIENT_OBJ) tests/src/ctdb_traverse.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_traverse.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_traverse.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_randrec: $(CTDB_CLIENT_OBJ) tests/ctdb_randrec.o 
+bin/ctdb_randrec: $(CTDB_CLIENT_OBJ) tests/src/ctdb_randrec.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_randrec.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_randrec.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_persistent: $(CTDB_CLIENT_OBJ) tests/ctdb_persistent.o 
+bin/ctdb_persistent: $(CTDB_CLIENT_OBJ) tests/src/ctdb_persistent.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_persistent.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_persistent.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_transaction: $(CTDB_CLIENT_OBJ) tests/ctdb_transaction.o 
+bin/ctdb_transaction: $(CTDB_CLIENT_OBJ) tests/src/ctdb_transaction.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_transaction.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/src/ctdb_transaction.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
 bin/ibwrapper_test: $(CTDB_CLIENT_OBJ) ib/ibwrapper_test.o
 	@echo Linking $@
diff --git a/ctdb/tests/scripts/ctdb_test_functions.bash b/ctdb/tests/scripts/ctdb_test_functions.bash
index d1886b8ebd..3713c98e49 100644
--- a/ctdb/tests/scripts/ctdb_test_functions.bash
+++ b/ctdb/tests/scripts/ctdb_test_functions.bash
@@ -1,15 +1,5 @@
 # Hey Emacs, this is a -*- shell-script -*- !!!  :-)
 
-numnodes=3
-
-export CTDB_NODES_SOCKETS=""
-for i in $(seq 1 $numnodes) ; do
-    CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${PWD}/sock.${i}"
-done
-
-
-######################################################################
-
 fail ()
 {
     echo "$*"
@@ -55,7 +45,7 @@ test_end ()
     testduration=$(($(date +%s)-$teststarttime))
 
     echo "=========================================================================="
-    echo "TEST ${interp}: ${name}${statstr}, duration: $testduration sec."
+    echo "TEST ${interp}: ${name}${statstr} (duration: ${testduration}s)"
     echo "=========================================================================="
 
 }
@@ -258,7 +248,7 @@ ips_are_on_nodeglob ()
 
     local out
 
-    try_command_on_node 1 ctdb ip
+    try_command_on_node 1 ctdb ip -n all
 
     while read ip pnn ; do
 	for check in $ips ; do
@@ -283,10 +273,68 @@ wait_until_ips_are_on_nodeglob ()
     wait_until 60 ips_are_on_nodeglob "$@"
 }
 
+stop_daemons ()
+{
+    echo "Attempting to politely shutdown daemons..."
+    onnode 1 ctdb shutdown -n all || true
+
+    echo "Sleeping for a while..."
+    sleep_for 1
+
+    if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then
+	echo "Killing remaining daemons..."
+	killall $CTDB_DIR/bin/ctdbd
+
+	if pidof $CTDB_DIR/bin/ctdbd >/dev/null ; then
+	    echo "Once more with feeling.."
+	    killall -9 $CTDB_DIR/bin/ctdbd
+	fi
+    fi
+
+    var_dir=$CTDB_DIR/tests/var
+    rm -rf $var_dir/test.db
+}
 
 start_daemons ()
 {
-    $CTDB_DIR/tests/start_daemons.sh $numnodes >$CTDB_DIR/var/daemons.log
+    local num_nodes="${1:-2}" # default is 2 nodes
+    shift # "$@" gets passed to ctdbd
+
+    local var_dir=$CTDB_DIR/tests/var
+
+    mkdir -p $var_dir/test.db/persistent
+
+    local nodes=$var_dir/nodes.txt
+    local public_addresses=$var_dir/public_addresses.txt
+    rm -f $nodes $public_addresses
+
+    local i
+    for i in $(seq 1 $num_nodes) ; do
+	if [ "${CTDB_USE_IPV6}x" != "x" ]; then
+	    echo ::$i >> $nodes
+	    ip addr add ::$i/128 dev lo
+	else
+	    echo 127.0.0.$i >> $nodes
+	    # 2 public addresses per node, just to make things interesting.
+	    echo "192.0.2.$i/24 lo" >> $public_addresses
+	    echo "192.0.2.$(($i + $num_nodes))/24 lo" >> $public_addresses
+	fi
+    done
+
+    local ctdb_options="--reclock=$var_dir/rec.lock --nlist $nodes --public-addresses $public_addresses --nopublicipcheck --event-script-dir=tests/events.d --logfile=$var_dir/daemons.log -d 0 --dbdir=$var_dir/test.db --dbdir-persistent=$var_dir/test.db/persistent"
+
+    echo "Starting $num_nodes ctdb daemons..."
+    for i in $(seq 1 $num_nodes) ; do
+	if [ $(id -u) -eq 0 ]; then
+            ctdb_options="$ctdb_options --public-interface=lo"
+	fi
+
+	$VALGRIND bin/ctdbd --socket=$var_dir/sock.$i $ctdb_options "$@" || return 1
+    done
+
+    if [ -L /tmp/ctdb.socket -o ! -S /tmp/ctdb.socket ] ; then 
+	ln -sf $var_dir/sock.1 /tmp/ctdb.socket || return 1
+    fi
 }
 
 _restart_ctdb ()
@@ -301,8 +349,8 @@ _restart_ctdb ()
 restart_ctdb ()
 {
     if [ -n "$CTDB_NODES_SOCKETS" ] ; then
-	onnode all ctdb shutdown
-	start_daemons
+	stop_daemons
+	start_daemons $CTDB_NUM_NODES
     else
 	onnode -pq all $TEST_WRAP _restart_ctdb 
     fi || return 1
@@ -327,7 +375,3 @@ ctdb_test_exit ()
 
     test_exit
 }
-
-########################################
-
-export PATH=/usr/local/autocluster:$PATH
diff --git a/ctdb/tests/scripts/run_tests b/ctdb/tests/scripts/run_tests
index eab28f06ea..9827ebed63 100755
--- a/ctdb/tests/scripts/run_tests
+++ b/ctdb/tests/scripts/run_tests
@@ -1,9 +1,23 @@
 #!/bin/bash
 
 export CTDB_DIR=$(cd $(dirname $(dirname $(dirname $0))) ; pwd)
+var_dir=$CTDB_DIR/tests/var
 
-ctdb_bin_dir="${CTDB_DIR}/bin"
-ctdb_tools_dir="${CTDB_DIR}/tools"
+export CTDB_NUM_NODES=3
+
+######################################################################
+
+if [ ! -n "$CTDB_TEST_REAL_CLUSTER" ] ; then
+    export CTDB_NODES_SOCKETS=""
+    for i in $(seq 1 $CTDB_NUM_NODES) ; do
+	CTDB_NODES_SOCKETS="${CTDB_NODES_SOCKETS}${CTDB_NODES_SOCKETS:+ }${var_dir}/sock.${i}"
+    done
+fi
+
+######################################################################
+
+ctdb_bin_dir=$CTDB_DIR/bin
+ctdb_tools_dir=$CTDB_DIR/tools
 ctdb_test_scripts_dir=$(cd $(dirname $0) ; pwd)
 
 PATH="${ctdb_test_scripts_dir}:${ctdb_bin_dir}:${ctdb_tools_dir}:${PATH}"
diff --git a/ctdb/tests/simple/31_ctdb_disable_simple.sh b/ctdb/tests/simple/31_ctdb_disable_simple.sh
index fdb7daed87..f49af88396 100755
--- a/ctdb/tests/simple/31_ctdb_disable_simple.sh
+++ b/ctdb/tests/simple/31_ctdb_disable_simple.sh
@@ -36,4 +36,6 @@ else
     testfailures=1
 fi
 
+echo "Expect a restart here..."
+
 ctdb_test_exit
diff --git a/ctdb/tests/ctdb_bench.c b/ctdb/tests/src/ctdb_bench.c
index 2d6b3ab298..2d6b3ab298 100644
--- a/ctdb/tests/ctdb_bench.c
+++ b/ctdb/tests/src/ctdb_bench.c
diff --git a/ctdb/tests/ctdb_fetch.c b/ctdb/tests/src/ctdb_fetch.c
index 2cc51d51bc..2cc51d51bc 100644
--- a/ctdb/tests/ctdb_fetch.c
+++ b/ctdb/tests/src/ctdb_fetch.c
diff --git a/ctdb/tests/ctdb_persistent.c b/ctdb/tests/src/ctdb_persistent.c
index 8f0452abad..8f0452abad 100644
--- a/ctdb/tests/ctdb_persistent.c
+++ b/ctdb/tests/src/ctdb_persistent.c
diff --git a/ctdb/tests/ctdb_randrec.c b/ctdb/tests/src/ctdb_randrec.c
index 4b9b2bc93e..4b9b2bc93e 100644
--- a/ctdb/tests/ctdb_randrec.c
+++ b/ctdb/tests/src/ctdb_randrec.c
diff --git a/ctdb/tests/ctdb_store.c b/ctdb/tests/src/ctdb_store.c
index ce4195c65f..ce4195c65f 100644
--- a/ctdb/tests/ctdb_store.c
+++ b/ctdb/tests/src/ctdb_store.c
diff --git a/ctdb/tests/ctdb_transaction.c b/ctdb/tests/src/ctdb_transaction.c
index 783fa123fd..783fa123fd 100644
--- a/ctdb/tests/ctdb_transaction.c
+++ b/ctdb/tests/src/ctdb_transaction.c
diff --git a/ctdb/tests/ctdb_traverse.c b/ctdb/tests/src/ctdb_traverse.c
index 1726b235d4..1726b235d4 100644
--- a/ctdb/tests/ctdb_traverse.c
+++ b/ctdb/tests/src/ctdb_traverse.c
diff --git a/ctdb/tests/rb_perftest.c b/ctdb/tests/src/rb_perftest.c
index 1760cd1149..1760cd1149 100644
--- a/ctdb/tests/rb_perftest.c
+++ b/ctdb/tests/src/rb_perftest.c
diff --git a/ctdb/tests/rb_test.c b/ctdb/tests/src/rb_test.c
index 553a0f6c48..553a0f6c48 100644
--- a/ctdb/tests/rb_test.c
+++ b/ctdb/tests/src/rb_test.c
diff --git a/ctdb/tests/start_daemons.sh b/ctdb/tests/start_daemons.sh
index 424d892061..60c0a66a9e 100755
--- a/ctdb/tests/start_daemons.sh
+++ b/ctdb/tests/start_daemons.sh
@@ -7,26 +7,24 @@ fi
 shift
 
 NODES="./tests/nodes.txt"
-rm -f $NODES
 PUBLIC_ADDRESSES=./tests/public_addresses.txt
-rm -f $PUBLIC_ADDRESSES
+rm -f $NODES $PUBLIC_ADDRESSES
 for i in `seq 1 $NUMNODES`; do
   if [ "${CTDB_USE_IPV6}x" != "x" ]; then
     echo ::$i >> $NODES
     ip addr add ::$i/128 dev lo
   else
     echo 127.0.0.$i >> $NODES
-    #echo "127.0.1.$i/24 lo" >> $PUBLIC_ADDRESSES
-    #echo "127.0.1.$(($i + $NUMNODES))/24 lo" >> $PUBLIC_ADDRESSES
+    # 2 public addresses per node, just to make things interesting.
     echo "192.0.2.$i/24 lo" >> $PUBLIC_ADDRESSES
     echo "192.0.2.$(($i + $NUMNODES))/24 lo" >> $PUBLIC_ADDRESSES
   fi
 done
 
-killall -q ctdbd
+killall -q $PWD/bin/ctdbd
 rm -rf test.db/persistent/*
  
-CTDB_OPTIONS="--reclock=rec.lock --nlist $NODES --public-addresses $PUBLIC_ADDRESSES --event-script-dir=tests/events.d --logfile=- -d 0 --dbdir=test.db --dbdir-persistent=test.db/persistent $*"
+CTDB_OPTIONS="--reclock=rec.lock --nlist $NODES --public-addresses $PUBLIC_ADDRESSES --nopublicipcheck --event-script-dir=tests/events.d --logfile=- -d 0 --dbdir=test.db --dbdir-persistent=test.db/persistent $*"
 
 echo "Starting $NUMNODES ctdb daemons"
 for i in `seq 1 $NUMNODES`; do