adapted testbench to new capabilities

... and now make check fails again, we have obviously found some new
bugs thanks to the additional cases

12 files changed, 86 insertions, 101 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 279158cf..71daa3bc 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -2,7 +2,8 @@ TESTRUNS = rt_init rscript
 check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq
 TESTS = $(TESTRUNS) cfg.sh arrayqueue.sh linkedlistqueue.sh da-mainmsg-q.sh \
 	diskqueue.sh manytcp.sh \
-	queue-persist.sh
+	queue-persist.sh \
+	daqueue-persist.sh
 
 if ENABLE_OMSTDOUT
 TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh
@@ -65,6 +66,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/1.inputname_imtcp_12516 \
 	   omod-if-array.sh \
 	   diag.sh \
+	   daqueue-persist.sh \
+	   daqueue-persist-drvr.sh \
 	   queue-persist.sh \
 	   queue-persist-drvr.sh \
 	   testsuites/queue-persist.conf \
diff --git a/tests/arrayqueue.sh b/tests/arrayqueue.sh
index 7791ed57..01fc133b 100755
--- a/tests/arrayqueue.sh
+++ b/tests/arrayqueue.sh
@@ -2,25 +2,13 @@
 # added 2009-05-20 by rgerhards
 # This file is part of the rsyslog project, released  under GPLv3
 echo testing queue fixedArray queue mode
-rm -f work rsyslog.out.log
-# enable this, if you need debug output: export RSYSLOG_DEBUG="debug"
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/arrayqueue.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup arrayqueue.conf
+
 # 40000 messages should be enough
-./tcpflood 127.0.0.1 13514 1 40000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-sleep 4 # we need this so that rsyslogd can receive all outstanding messages
+source $srcdir/diag.sh injectmsg  0 40000
+
+# terminate *now* (don't wait for queue to drain!)
 kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq -fwork -e 39999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 39999
+source $srcdir/diag.sh exit
diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh
index fde9e06e..1c947de2 100755
--- a/tests/da-mainmsg-q.sh
+++ b/tests/da-mainmsg-q.sh
@@ -8,55 +8,25 @@
 # added 2009-04-22 by Rgerhards
 # This file is part of the rsyslog project, released  under GPLv3
 echo "testing main message queue in DA mode (going to disk)"
-rm -f work rsyslog.out.log
-rm -rf test-spool
-mkdir test-spool
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/da-mainmsg-q.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
-#
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup da-mainmsg-q.conf
+
 # part1: send first 50 messages (in memory, only)
-#
-./tcpflood 127.0.0.1 13514 1 50
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-ls -l test-spool
-sleep 2 # we need this so that rsyslogd can receive all outstanding messages
-#
+#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 50
+source $srcdir/diag.sh injectmsg 0 50
+source $srcdir/diag.sh wait-queueempty # let queue drain for this test case
+
 # part 2: send bunch of messages. This should trigger DA mode
-#
-# 20000 messages should be enough - the disk test is slow enough ;)
-./tcpflood 127.0.0.1 13514 2 20000 50
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-ls -l test-spool
-sleep 8 # we need this so that rsyslogd can receive all outstanding messages
-#
+source $srcdir/diag.sh injectmsg 50 20000
+ls -l test-spool	 # for manual review
+
 # send another handful
-#
-ls -l test-spool
-./tcpflood 127.0.0.1 13514 1 50 20050
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-sleep 1 # we need this so that rsyslogd can receive all outstanding messages
-# 
+source $srcdir/diag.sh injectmsg 20050 50
+#sleep 1 # we need this so that rsyslogd can receive all outstanding messages
+
 # clean up and check test result
-#
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq -fwork -e20099
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
-rm -rf test-spool
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+### currently, we get a stable abort if we use the former kill logic. With shutdown-when-empty, it hangs (but that still tells us there is a bug ;)) ###
+#kill `cat rsyslog.pid`
+source $srcdir/diag.sh seq-check 0 20099
+source $srcdir/diag.sh exit
diff --git a/tests/daqueue-persist-drvr.sh b/tests/daqueue-persist-drvr.sh
new file mode 100755
index 00000000..11058110
--- /dev/null
+++ b/tests/daqueue-persist-drvr.sh
@@ -0,0 +1,28 @@
+# Test for queue data persisting at shutdown. The
+# plan is to start an instance, emit some data, do a relatively
+# fast shutdown and then re-start the engine to process the 
+# remaining data.
+# added 2009-05-27 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+# uncomment for debugging support:
+echo testing memory daqueue persisting to disk, mode $1
+source $srcdir/diag.sh init
+
+# prepare config
+echo \$MainMsgQueueType $1 > work-queuemode.conf
+echo "*.*     :omtesting:sleep 0 1000" > work-delay.conf
+
+# inject 10000 msgs, so that DO hit the high watermark
+source $srcdir/diag.sh startup queue-persist.conf
+source $srcdir/diag.sh injectmsg 0 10000
+$srcdir/diag.sh shutdown-immediate
+$srcdir/diag.sh wait-shutdown
+source $srcdir/diag.sh check-mainq-spool
+
+# restart engine and have rest processed
+#remove delay
+echo "#" > work-delay.conf
+source $srcdir/diag.sh startup queue-persist.conf
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/daqueue-persist.sh b/tests/daqueue-persist.sh
new file mode 100755
index 00000000..157b8f7a
--- /dev/null
+++ b/tests/daqueue-persist.sh
@@ -0,0 +1,11 @@
+# Test for queue data persisting at shutdown. We use the actual driver
+# to carry out multiple tests with different queue modes
+# added 2009-05-27 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+source $srcdir/daqueue-persist-drvr.sh LinkedList
+#source $srcdir/daqueue-persist-drvr.sh FixedArray
+# the disk test should not fail, however, the config is extreme and using
+# it more or less is a config error
+#source $srcdir/daqueue-persist-drvr.sh Disk
+# we do not test Direct mode because this absolute can not work in direct mode
+# (maybe we should do a fail-type of test?)
diff --git a/tests/diag.sh b/tests/diag.sh
index a8f9b1b4..01b335ee 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -6,10 +6,11 @@
 # begun 2009-05-27 by rgerhards
 # This file is part of the rsyslog project, released under GPLv3
 #set -o xtrace
-#export RSYSLOG_DEBUG="debug nostdout"
-#export RSYSLOG_DEBUGLOG="log"
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="log"
 case $1 in
    'init')	$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
+		rm -f core.* vgcore.* # do NOT delete them at exit ;)
 		rm -f rsyslogd.started work-*.conf
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
 		rm -rf test-spool
@@ -21,7 +22,7 @@ case $1 in
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
    		# returns only after successful startup
-		../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 &
+		valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 &
    		$srcdir/diag.sh wait-startup
 		;;
    'wait-startup') # wait for rsyslogd startup
@@ -73,7 +74,6 @@ case $1 in
 		sort < rsyslog.out.log > work
 		./chkseq -fwork -e$2 $3
 		if [ "$?" -ne "0" ]; then
-		  rm -f work rsyslog.out.log
 		  echo "sequence error detected"
 		  exit 1
 		fi
diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh
index bf1a46fd..2fe31db9 100755
--- a/tests/diskqueue.sh
+++ b/tests/diskqueue.sh
@@ -5,9 +5,6 @@
 # added 2009-04-17 by Rgerhards
 # This file is part of the rsyslog project, released  under GPLv3
 # uncomment for debugging support:
-#set -o xtrace
-#export RSYSLOG_DEBUG="debug nostdout"
-#export RSYSLOG_DEBUGLOG="tmp"
 echo testing queue disk-only mode
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup diskqueue.conf
diff --git a/tests/linkedlistqueue.sh b/tests/linkedlistqueue.sh
index aa574bd1..9570ed2b 100755
--- a/tests/linkedlistqueue.sh
+++ b/tests/linkedlistqueue.sh
@@ -2,25 +2,13 @@
 # added 2009-05-20 by rgerhards
 # This file is part of the rsyslog project, released  under GPLv3
 echo testing queue Linkedlist queue mode
-rm -f work rsyslog.out.log
-# enable this, if you need debug output: export RSYSLOG_DEBUG="debug"
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/arrayqueue.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup linkedlistqueue.conf
+
 # 40000 messages should be enough
-./tcpflood 127.0.0.1 13514 1 40000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-sleep 4 # we need this so that rsyslogd can receive all outstanding messages
+source $srcdir/diag.sh injectmsg  0 40000
+
+# terminate *now* (don't wait for queue to drain)
 kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq -fwork -e39999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 39999
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/arrayqueue.conf b/tests/testsuites/arrayqueue.conf
index 7fdbc79b..44b9920b 100644
--- a/tests/testsuites/arrayqueue.conf
+++ b/tests/testsuites/arrayqueue.conf
@@ -1,11 +1,11 @@
 # Test for queue fixedArray mode (see .sh file for details)
 # rgerhards, 2009-04-17
+$IncludeConfig testsuites/diag-common.conf
+
 $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 10000
 $InputTCPServerRun 13514
 
-$ErrorMessagesToStderr off
-
 # set spool locations and switch queue to disk-only mode
 $MainMsgQueueType FixedArray
 
diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf
index fbda27d4..c1e8cb01 100644
--- a/tests/testsuites/da-mainmsg-q.conf
+++ b/tests/testsuites/da-mainmsg-q.conf
@@ -4,7 +4,7 @@ $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 10000
 $InputTCPServerRun 13514
 
-$ErrorMessagesToStderr off
+$IncludeConfig testsuites/diag-common.conf
 
 # set spool locations and switch queue to disk assisted mode
 $WorkDirectory test-spool
diff --git a/tests/testsuites/linkedlistqueue.conf b/tests/testsuites/linkedlistqueue.conf
index ecfc7a26..321678a5 100644
--- a/tests/testsuites/linkedlistqueue.conf
+++ b/tests/testsuites/linkedlistqueue.conf
@@ -1,5 +1,7 @@
 # Test for queue LinkedList mode (see .sh file for details)
 # rgerhards, 2009-04-17
+$IncludeConfig testsuites/diag-common.conf
+
 $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 10000
 $InputTCPServerRun 13514
diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf
index 80f8ba30..4396c7bd 100644
--- a/tests/testsuites/queue-persist.conf
+++ b/tests/testsuites/queue-persist.conf
@@ -9,8 +9,6 @@ $InputTCPServerRun 13514
 
 $ModLoad ../plugins/omtesting/.libs/omtesting
 
-$ErrorMessagesToStderr off
-
 # set spool locations and switch queue to disk-only mode
 $WorkDirectory test-spool
 $MainMsgQueueFilename mainq