greatly enhanced testbench

The imdiag module now can very effectively inject messages, which also
frees us from uncertainties of tcp reception and processing. All shell
script based tests have been modularized, what makes it far easier to
create new tests. Also, the test bench now executes more reliable and
much faster, because we can now rely on actual engine information where
we previously did just a dumb sleep.

20 files changed, 319 insertions, 217 deletions

diff --git a/ChangeLog b/ChangeLog
index 2fc760c3..d8fae145 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 ---------------------------------------------------------------------------
 Version 4.3.?  [DEVEL] (rgerhards), 2009-??-??
 - bugfix: imdiag/imtcp had a race condition
+- improved testbench (now much better code design and reuse)
 ---------------------------------------------------------------------------
 Version 4.3.1  [DEVEL] (rgerhards), 2009-05-25
 - added capability to run multiple tcp listeners (on different ports)
diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c
index 40f94692..c700cab7 100644
--- a/plugins/imdiag/imdiag.c
+++ b/plugins/imdiag/imdiag.c
@@ -52,6 +52,8 @@
 #include "errmsg.h"
 #include "tcpsrv.h"
 #include "srUtils.h"
+#include "msg.h"
+#include "datetime.h"
 #include "net.h" /* for permittedPeers, may be removed when this is removed */
 
 MODULE_TYPE_INPUT
@@ -63,6 +65,7 @@ DEFobjCurrIf(tcps_sess)
 DEFobjCurrIf(net)
 DEFobjCurrIf(netstrm)
 DEFobjCurrIf(errmsg)
+DEFobjCurrIf(datetime)
 
 /* Module static data */
 static tcpsrv_t *pOurTcpsrv = NULL;  /* our TCP server(listener) TODO: change for multiple instances */
@@ -134,10 +137,123 @@ onErrClose(tcps_sess_t *pSess)
 /* ------------------------------ end callbacks ------------------------------ */
 
 
+/* get the first word delimited by space from a given string. The pointer is
+ * advanced to after the word. Any leading spaces are discarded. If the
+ * output buffer is too small, parsing ends on buffer full condition.
+ * An empty buffer is returned if there is no more data inside the string.
+ * rgerhards, 2009-05-27
+ */
+#define TO_LOWERCASE	1
+#define NO_MODIFY	0
+static void
+getFirstWord(uchar **ppszSrc, uchar *pszBuf, size_t lenBuf, int options)
+{
+	uchar c;
+	uchar *pszSrc = *ppszSrc;
+
+	while(*pszSrc && *pszSrc == ' ')
+		++pszSrc; /* skip to first non-space */
+
+	while(*pszSrc && *pszSrc != ' ' && lenBuf > 1) {
+		c = *pszSrc++;
+		if(options & TO_LOWERCASE)
+			c = tolower(c);
+		*pszBuf++ = c;
+		lenBuf--;
+	}
+
+	*pszBuf = '\0';
+	*ppszSrc = pszSrc;
+}
+
+
+/* send a response back to the originator
+ * rgerhards, 2009-05-27
+ */
+static rsRetVal __attribute__((format(printf, 2, 3)))
+sendResponse(tcps_sess_t *pSess, char *fmt, ...)
+{
+	va_list ap;
+	ssize_t len;
+	uchar buf[1024];
+	DEFiRet;
+
+	va_start(ap, fmt);
+	len = vsnprintf((char*)buf, sizeof(buf), fmt, ap);
+	va_end(ap);
+	CHKiRet(netstrm.Send(pSess->pStrm, buf, &len));
+
+finalize_it:
+	RETiRet;
+}
+
+
+/* actually submit a message to the rsyslog core
+ */
+static rsRetVal
+doInjectMsg(int iNum)
+{
+	uchar szMsg[1024];
+	msg_t *pMsg;
+	struct syslogTime stTime;
+	time_t ttGenTime;
+	DEFiRet;
+
+	snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar),
+		 "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum);
+
+	datetime.getCurrTime(&stTime, &ttGenTime);
+	/* we now create our own message object and submit it to the queue */
+	CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime));
+	CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg));
+	pMsg->iLenRawMsg = ustrlen(szMsg);
+	MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"));
+	MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY);
+	pMsg->msgFlags  = NEEDS_PARSING | PARSE_HOSTNAME;
+	pMsg->bParseHOSTNAME = 1;
+	MsgSetRcvFrom(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* TODO: way may use the real sender here... */
+	CHKiRet(MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")));
+	CHKiRet(submitMsg(pMsg));
+
+finalize_it:
+	RETiRet;
+}
+
+
+/* This function injects messages. Command format:
+ * injectmsg <fromnbr> <number-of-messages>
+ * rgerhards, 2009-05-27
+ */
+static rsRetVal
+injectMsg(uchar *pszCmd, tcps_sess_t *pSess)
+{
+	uchar wordBuf[1024];
+	int iFrom;
+	int nMsgs;
+	int i;
+	DEFiRet;
+
+	/* we do not check errors here! */
+	getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE);
+	iFrom = atoi((char*)wordBuf);
+	getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE);
+	nMsgs = atoi((char*)wordBuf);
+
+	for(i = 0 ; i < nMsgs ; ++i) {
+		doInjectMsg(i + iFrom);
+	}
+
+	CHKiRet(sendResponse(pSess, "messages injected\n"));
+
+finalize_it:
+	RETiRet;
+}
+
+
 /* This function waits until the main queue is drained (size = 0)
  */
 static rsRetVal
-waitMainQEmpty(void)
+waitMainQEmpty(tcps_sess_t *pSess)
 {
 	int iMsgQueueSize;
 	DEFiRet;
@@ -148,21 +264,21 @@ waitMainQEmpty(void)
 		CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize));
 	}
 
+	CHKiRet(sendResponse(pSess, "mainqueue empty\n"));
+
 finalize_it:
 	RETiRet;
 }
 
-
 /* Function to handle received messages. This is our core function!
  * rgerhards, 2009-05-24
  */
 static rsRetVal
 OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg)
 {
-	ssize_t len;
 	int iMsgQueueSize;
 	uchar *pszMsg;
-	uchar buf[1024];
+	uchar cmdBuf[1024];
 	DEFiRet;
 
 	assert(pSess != NULL);
@@ -176,17 +292,18 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg)
 	memcpy(pszMsg, pRcv, iLenMsg);
 	pszMsg[iLenMsg] = '\0';
 
-	if(!ustrcmp(pszMsg, UCHAR_CONSTANT("GetMainMsgQueueSize"))) {
+	getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE);
+
+	if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) {
 		CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize));
-		len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "%d\n", iMsgQueueSize);
-		CHKiRet(netstrm.Send(pSess->pStrm, buf, &len));
-	} else if(!ustrcmp(pszMsg, UCHAR_CONSTANT("WaitMainQueueEmpty"))) {
-		CHKiRet(waitMainQEmpty());
-		len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "mainqueue empty\n");
-		CHKiRet(netstrm.Send(pSess->pStrm, buf, &len));
+		CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize));
+	} else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) {
+		CHKiRet(waitMainQEmpty(pSess));
+	} else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) {
+		CHKiRet(injectMsg(pszMsg, pSess));
 	} else {
-		len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "unkown command '%s'\n", pszMsg);
-		CHKiRet(netstrm.Send(pSess->pStrm, buf, &len));
+		dbgprintf("imdiag unkown command '%s'\n", cmdBuf);
+		CHKiRet(sendResponse(pSess, "unkown command '%s'\n", cmdBuf));
 	}
 
 finalize_it:
@@ -285,6 +402,7 @@ CODESTARTmodExit
 	objRelease(tcps_sess, LM_TCPSRV_FILENAME);
 	objRelease(tcpsrv, LM_TCPSRV_FILENAME);
 	objRelease(errmsg, CORE_COMPONENT);
+	objRelease(datetime, CORE_COMPONENT);
 ENDmodExit
 
 
@@ -321,6 +439,7 @@ CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME));
 	CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME));
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	CHKiRet(objUse(datetime, CORE_COMPONENT));
 
 	/* register config file handlers */
 	CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord,
diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c
index f0efe1ee..84e660bc 100644
--- a/plugins/imtcp/imtcp.c
+++ b/plugins/imtcp/imtcp.c
@@ -180,7 +180,6 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa
 		}
 	}
 
-dbgprintf("XXX: try add listen port %s\n", pNewVal);
 	/* initialized, now add socket */
 	CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ?
 						UCHAR_CONSTANT("imtcp") : pszInputName));
diff --git a/runtime/netstrm.c b/runtime/netstrm.c
index f6a8de7f..05bb25c0 100644
--- a/runtime/netstrm.c
+++ b/runtime/netstrm.c
@@ -114,7 +114,6 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew)
 	ISOBJ_TYPE_assert(pThis, netstrm);
 	assert(ppNew != NULL);
 
-RUNLOG_STR("XXX: accept conn reqeust");
 	/* accept the new connection */
 	CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd));
 	/* construct our object so that we can use it... */
diff --git a/tcpsrv.c b/tcpsrv.c
index 249eeecf..02eee88e 100644
--- a/tcpsrv.c
+++ b/tcpsrv.c
@@ -97,7 +97,6 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort)
 
 	ISOBJ_TYPE_assert(pThis, tcpsrv);
 
-dbgprintf("XXX: tcpsrv.c add port %s, name '%s'\n", pszPort, pThis->pszInputName);
 	/* create entry */
 	CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t)));
 	pEntry->pszPort = pszPort;
@@ -267,7 +266,6 @@ addTcpLstn(void *pUsr, netstrm_t *pLstn)
 	tcpsrv_t *pThis = pPortList->pSrv;
 	DEFiRet;
 
-dbgprintf("XXX: addTcpLst name %s\n", pPortList->pszInputName);
 	ISOBJ_TYPE_assert(pThis, tcpsrv);
 	ISOBJ_TYPE_assert(pLstn, netstrm);
 
@@ -326,7 +324,6 @@ create_tcp_socket(tcpsrv_t *pThis)
 	/* init all configured ports */
 	pEntry = pThis->pLstnPorts;
 	while(pEntry != NULL) {
-dbgprintf("XXX: tcpsrv.c create_tcp_socket do port %s\n", pEntry->pszPort);
 		CHKiRet(initTCPListener(pThis, pEntry));
 		pEntry = pEntry->pNext;
 	}
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 92401511..a2e6c34c 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,6 @@
 TESTRUNS = rt_init rscript
 check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq
-TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh memq-persist.sh
+TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh
 if ENABLE_OMSTDOUT
 TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh
 endif
@@ -9,7 +9,6 @@ DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class
 test_files = testbench.h runtime-dummy.c
 
 check_JAVA = DiagTalker.java
-#dist_java_JAVA = DiagTalker.java
 
 EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   cfg1.cfgtest \
@@ -34,6 +33,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/1.omod-if-array \
 	   killrsyslog.sh \
 	   parsertest.sh \
+	   queue-persist.sh \
+	   testsuites/queue-persist.conf \
 	   diskqueue.sh \
 	   testsuites/diskqueue.conf \
 	   imtcp-multiport.sh \
@@ -49,10 +50,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/1.inputname_imtcp_12515 \
 	   testsuites/1.inputname_imtcp_12516 \
 	   omod-if-array.sh \
-	   waitqueueempty.sh \
-	   memq-persist.sh \
-	   testsuites/memq-persist1.sh \
-	   testsuites/memq-persist2.sh \
+	   diag.sh \
+	   queue-persist.sh \
+	   queue-persist-drvr.sh \
+	   testsuites/queue-persist.sh \
 	   DiagTalker.java \
 	   cfg.sh
 
diff --git a/tests/diag.sh b/tests/diag.sh
new file mode 100755
index 00000000..270b30d6
--- /dev/null
+++ b/tests/diag.sh
@@ -0,0 +1,82 @@
+# this shell script provides commands to the common diag system. It enables
+# test scripts to wait for certain conditions and initiate certain actions.
+# needs support in config file.
+# NOTE: this file should be included with "source diag.sh", as it otherwise is
+# not always able to convey back states to the upper-level test driver
+# begun 2009-05-27 by rgerhards
+# This file is part of the rsyslog project, released under GPLv3
+#set -o xtrace
+#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="tmp"
+case $1 in
+   'init')	$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
+		rm -f rsyslogd.started work-*.conf
+		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
+		rm -rf test-spool
+		mkdir test-spool
+		;;
+   'exit')	rm -f rsyslogd.started work-*.conf
+		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
+		rm -rf test-spool
+		;;
+   'startup')   # start rsyslogd with default params. $2 is the config file name to use
+   		# returns only after successful startup
+		../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 &
+   		$srcdir/diag.sh wait-startup
+		;;
+   'wait-startup') # wait for rsyslogd startup
+		while test ! -f rsyslogd.started; do
+			true
+		done
+		echo "rsyslogd started with pid " `cat rsyslog.pid`
+		;;
+   'wait-shutdown')  # actually, we wait for rsyslog.pid to be deleted
+		while test -f rsyslog.pid; do
+			true
+		done
+		;;
+   'wait-queueempty') # wait for main message queue to be empty
+		echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker
+		;;
+   'shutdown-when-empty') # shut rsyslogd down when main queue is empty
+   		$srcdir/diag.sh wait-queueempty
+		kill `cat rsyslog.pid`
+		# note: we do not wait for the actual termination!
+		;;
+   'shutdown-immediate') # shut rsyslogd down without emptying the queue
+		kill `cat rsyslog.pid`
+		# note: we do not wait for the actual termination!
+		;;
+   'tcpflood') # do a tcpflood run and check if it worked params are passed to tcpflood
+		./tcpflood $2 $3 $4 $5 $6 $7 $8
+		if [ "$?" -ne "0" ]; then
+		  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
+		  cp rsyslog.out.log rsyslog.out.log.save
+		  exit 1
+		fi
+		;;
+   'injectmsg') # inject messages via our inject interface (imdiag)
+		echo injectmsg $2 $3 $4 $5 | java -classpath $abs_top_builddir DiagTalker
+		# TODO: some return state checking? (does it really make sense here?)
+		;;
+   'check-mainq-spool') # check if mainqueue spool files exist, if not abort (we just check .qi)
+		echo There must exist some files now:
+		ls -l test-spool
+		if test ! -f test-spool/mainq.qi; then
+		  echo "error: mainq.qi does not exist where expected to do so!"
+		  ls -l test-spool
+		  exit 1
+		fi
+		;;
+   'seq-check') # do the usual sequence check to see if everything was properly received
+		rm -f work
+		sort < rsyslog.out.log > work
+		./chkseq work $2 $3
+		if [ "$?" -ne "0" ]; then
+		  rm -f work rsyslog.out.log
+		  echo "sequence error detected"
+		  exit 1
+		fi
+		;;
+   *)		echo "invalid argument" $1
+esac
diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh
index eabfcf78..bf1a46fd 100755
--- a/tests/diskqueue.sh
+++ b/tests/diskqueue.sh
@@ -9,27 +9,10 @@
 #export RSYSLOG_DEBUG="debug nostdout"
 #export RSYSLOG_DEBUGLOG="tmp"
 echo testing queue disk-only mode
-rm -rf test-spool
-mkdir test-spool
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup diskqueue.conf
 # 20000 messages should be enough - the disk test is slow enough ;)
-./tcpflood 127.0.0.1 13514 1 20000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq work 0 19999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
-rm -rf test-spool
+source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 19999
+source $srcdir/diag.sh exit
diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh
index 17480dae..702f8834 100755
--- a/tests/imtcp-multiport.sh
+++ b/tests/imtcp-multiport.sh
@@ -8,73 +8,31 @@
 # added 2009-05-22 by Rgerhards
 # This file is part of the rsyslog project, released  under GPLv3
 echo testing imtcp multiple listeners
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
-./tcpflood 127.0.0.1 13514 1 10000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq work 0 9999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup imtcp-multiport.conf
+source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 9999
+source $srcdir/diag.sh exit
 #
 #
 # ### now complete new cycle with other port ###
 #
 #
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
-./tcpflood 127.0.0.1 13515 1 10000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq work 0 9999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup imtcp-multiport.conf
+source $srcdir/diag.sh tcpflood 127.0.0.1 13515 1 10000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 9999
+source $srcdir/diag.sh exit
 #
 #
 # ### now complete new cycle with other port ###
 #
 #
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
-./tcpflood 127.0.0.1 13516 1 10000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq work 0 9999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup imtcp-multiport.conf
+source $srcdir/diag.sh tcpflood 127.0.0.1 13516 1 10000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 9999
+source $srcdir/diag.sh exit
diff --git a/tests/manytcp.sh b/tests/manytcp.sh
index 06bd38b6..c55eb9c0 100755
--- a/tests/manytcp.sh
+++ b/tests/manytcp.sh
@@ -1,21 +1,8 @@
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
+# test many concurrent tcp connections
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup manytcp.conf
 # the config file specifies exactly 1100 connections
-./tcpflood 127.0.0.1 13514 1000 40000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq work 0 39999
-if [ "$?" -ne "0" ]; then
-  rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
+source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1000 40000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 39999
+source $srcdir/diag.sh exit
diff --git a/tests/memq-persist.sh b/tests/memq-persist.sh
deleted file mode 100755
index 108cba57..00000000
--- a/tests/memq-persist.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-# Test for memory queue which is persisted at shutdown. The
-# plan is to start an instance, emit some data, do a relatively
-# fast shutdown and then re-start the engine to process the 
-# remaining data.
-# added 2009-05-25 by Rgerhards
-# This file is part of the rsyslog project, released  under GPLv3
-# uncomment for debugging support:
-#set -o xtrace
-#export RSYSLOG_DEBUG="debug nostdout"
-#export RSYSLOG_DEBUGLOG="log"
-echo testing memory queue persisting to disk
-$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
-rm -rf test-spool
-mkdir test-spool
-rm -f work rsyslog.out.log rsyslog.out.log.save # work files
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf &
-sleep 1
-echo "rsyslogd started with pid " `cat rsyslog.pid`
-# 20000 messages should be enough
-./tcpflood 127.0.0.1 13514 1 10000
-if [ "$?" -ne "0" ]; then
-  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
-  cp rsyslog.out.log rsyslog.out.log.save
-fi
-sleep 3 # we need to wait to ensure everything is received (less 1 second would be better)
-kill `cat rsyslog.pid`
-sleep 5 # wait for engine to terminate
-echo There must exist some files now:
-ls -l test-spool
-# restart engine and have rest processed
-../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist2.conf &
-$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages
-kill `cat rsyslog.pid`
-rm -f work
-sort < rsyslog.out.log > work
-./chkseq work 0 9999
-if [ "$?" -ne "0" ]; then
- # rm -f work rsyslog.out.log
-  echo "sequence error detected"
-  exit 1
-fi
-rm -f work rsyslog.out.log
-rm -rf test-spool
diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh
new file mode 100755
index 00000000..ea5386a7
--- /dev/null
+++ b/tests/queue-persist-drvr.sh
@@ -0,0 +1,28 @@
+# Test for queue data persisting at shutdown. The
+# plan is to start an instance, emit some data, do a relatively
+# fast shutdown and then re-start the engine to process the 
+# remaining data.
+# added 2009-05-27 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+# uncomment for debugging support:
+echo testing memory queue persisting to disk, mode $1
+source $srcdir/diag.sh init
+
+# prepare config
+echo \$MainMsgQueueType $1 > work-queuemode.conf
+echo "*.*     :omtesting:sleep 0 1000" > work-delay.conf
+
+# inject 5000 msgs, so that we do not hit the high watermark
+source $srcdir/diag.sh startup queue-persist.conf
+source $srcdir/diag.sh injectmsg 0 5000
+$srcdir/diag.sh shutdown-immediate
+$srcdir/diag.sh wait-shutdown
+source $srcdir/diag.sh check-mainq-spool
+
+# restart engine and have rest processed
+#remove delay
+echo "#" > work-delay.conf
+source $srcdir/diag.sh startup queue-persist.conf
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/queue-persist.sh b/tests/queue-persist.sh
new file mode 100755
index 00000000..999655b1
--- /dev/null
+++ b/tests/queue-persist.sh
@@ -0,0 +1,11 @@
+# Test for queue data persisting at shutdown. We use the actual driver
+# to carry out multiple tests with different queue modes
+# added 2009-05-27 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+source $srcdir/queue-persist-drvr.sh LinkedList
+source $srcdir/queue-persist-drvr.sh FixedArray
+# the disk test should not fail, however, the config is extreme and using
+# it more or less is a config error
+source $srcdir/queue-persist-drvr.sh Disk
+# we do not test Direct mode because this absolute can not work in direct mode
+# (maybe we should do a fail-type of test?)
diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf
new file mode 100644
index 00000000..9e9e28fe
--- /dev/null
+++ b/tests/testsuites/diag-common.conf
@@ -0,0 +1,16 @@
+# This is a config include file. It sets up rsyslog so that the
+# diag system can successfully be used. Also, it generates a file
+# "rsyslogd.started" after rsyslogd is initialized. This config file
+# should be included in all tests that intend to use common code for
+# controlling the daemon.
+# NOTE: we assume that rsyslogd's current working directory is 
+# ./tests (or the distcheck equivalent), in particlular that this
+# config file resides in the testsuites subdirectory.
+# rgerhards, 2009-05-27
+$ModLoad ../plugins/imdiag/.libs/imdiag
+$IMDiagServerRun 13500
+
+$template startupfile,"rsyslogd.started" # trick to use relative path names!
+:syslogtag, contains, "rsyslogd"  ?startupfile
+
+$ErrorMessagesToStderr off
diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf
index 017ee96d..d7f323bc 100644
--- a/tests/testsuites/diskqueue.conf
+++ b/tests/testsuites/diskqueue.conf
@@ -1,14 +1,11 @@
 # Test for queue disk mode (see .sh file for details)
 # rgerhards, 2009-04-17
+$IncludeConfig testsuites/diag-common.conf
+
 $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 10000
 $InputTCPServerRun 13514
 
-$ModLoad ../plugins/imdiag/.libs/imdiag
-$IMDiagServerRun 13500
-
-$ErrorMessagesToStderr off
-
 # set spool locations and switch queue to disk-only mode
 $WorkDirectory test-spool
 $MainMsgQueueFilename mainq
diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf
index ec059fe4..9146f6e0 100644
--- a/tests/testsuites/imtcp-multiport.conf
+++ b/tests/testsuites/imtcp-multiport.conf
@@ -1,16 +1,13 @@
 # Test for queue disk mode (see .sh file for details)
 # rgerhards, 2009-05-22
+$IncludeConfig testsuites/diag-common.conf
+
 $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 10000
 $InputTCPServerRun 13514
 $InputTCPServerRun 13515
 $InputTCPServerRun 13516
 
-$ModLoad ../plugins/imdiag/.libs/imdiag
-$IMDiagServerRun 13500
-
-$ErrorMessagesToStderr off
-
 $template outfmt,"%msg:F,58:2%\n"
 $template dynfile,"rsyslog.out.log" # trick to use relative path names!
 :msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf
index 3867da46..772ec9ce 100644
--- a/tests/testsuites/manytcp.conf
+++ b/tests/testsuites/manytcp.conf
@@ -1,16 +1,13 @@
 # Test for tcp "flood" testing
 # rgerhards, 2009-04-08
+$IncludeConfig testsuites/diag-common.conf
+
 $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 10000
 $MaxOpenFiles 2000
 $InputTCPMaxSessions 1100
 $InputTCPServerRun 13514
 
-$ModLoad ../plugins/imdiag/.libs/imdiag
-$IMDiagServerRun 13500
-
-$ErrorMessagesToStderr off
-
 $template outfmt,"%msg:F,58:2%\n"
 $template dynfile,"rsyslog.out.log" # trick to use relative path names!
 :msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/memq-persist2.conf b/tests/testsuites/memq-persist2.conf
deleted file mode 100644
index 23e29e2f..00000000
--- a/tests/testsuites/memq-persist2.conf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Test for persisting messages to disk on shutdown
-# rgerhards, 2009-04-17
-$ModLoad ../plugins/imtcp/.libs/imtcp
-$MainMsgQueueTimeoutShutdown 10000
-$MainMsgQueueSaveOnShutdown on
-$InputTCPServerRun 13514
-
-$ModLoad ../plugins/imdiag/.libs/imdiag
-$IMDiagServerRun 13500
-
-$ErrorMessagesToStderr off
-
-# set spool locations and switch queue to disk-only mode
-$WorkDirectory test-spool
-$MainMsgQueueFilename mainq
-$MainMsgQueueType LinkedList
-
-$template outfmt,"%msg:F,58:2%\n"
-$template dynfile,"rsyslog.out.log" # trick to use relative path names!
-:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/memq-persist1.conf b/tests/testsuites/queue-persist.conf
index 5240090f..80f8ba30 100644
--- a/tests/testsuites/memq-persist1.conf
+++ b/tests/testsuites/queue-persist.conf
@@ -1,13 +1,12 @@
-# Test for persisting messages to disk on shutdown
+# Test for persisting messages on shutdown
 # rgerhards, 2009-04-17
+$IncludeConfig testsuites/diag-common.conf
+
 $ModLoad ../plugins/imtcp/.libs/imtcp
 $MainMsgQueueTimeoutShutdown 1
 $MainMsgQueueSaveOnShutdown on
 $InputTCPServerRun 13514
 
-$ModLoad ../plugins/imdiag/.libs/imdiag
-$IMDiagServerRun 13500
-
 $ModLoad ../plugins/omtesting/.libs/omtesting
 
 $ErrorMessagesToStderr off
@@ -15,11 +14,10 @@ $ErrorMessagesToStderr off
 # set spool locations and switch queue to disk-only mode
 $WorkDirectory test-spool
 $MainMsgQueueFilename mainq
-$MainMsgQueueType LinkedList
+$IncludeConfig work-queuemode.conf
 
 $template outfmt,"%msg:F,58:2%\n"
 $template dynfile,"rsyslog.out.log" # trick to use relative path names!
 :msg, contains, "msgnum:" ?dynfile;outfmt
 
-# delay execution so that a queue can build up:
-*.*	:omtesting:sleep 0 1000
+$IncludeConfig work-delay.conf
diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh
deleted file mode 100755
index 4825853a..00000000
--- a/tests/waitqueueempty.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-# wait until main message queue is empty. This is currently done in
-# a separate shell script so that we can change the implementation
-# at some later point. -- rgerhards, 2009-05-25
-#echo WaitMainQueueEmpty | nc 127.0.0.1 13500
-echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker