added test for DA queue mode (in main msg queue) - needs more work

The problem is that the rsyslog engine writes messages too quickly, so
that the queue never enters DA mode. We still have some look contemption.
One possible solution (hopefully) is to finally move the parser code out
of imtcp and onto the next thread. Need to address these issues first, then
come back to the test case.

4 files changed, 95 insertions, 7 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0f4cbce1..b4509dee 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,6 @@
 TESTRUNS = rt_init rscript
 check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq
-TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh
+TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh da-mainmsg-q.sh diskqueue.sh manytcp.sh
 TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/
 DISTCLEANFILES=rsyslog.pid
 test_files = testbench.h runtime-dummy.c
@@ -29,6 +29,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   parsertest.sh \
 	   diskqueue.sh \
 	   testsuites/diskqueue.conf \
+	   da-mainmsg-q.sh \
+	   testsuites/da-mainmsg-q.conf \
 	   manytcp.sh \
 	   testsuites/manytcp.conf \
 	   omod-if-array.sh \
diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh
new file mode 100755
index 00000000..dc091e49
--- /dev/null
+++ b/tests/da-mainmsg-q.sh
@@ -0,0 +1,63 @@
+# Test for DA mode on the main message queue
+# This test checks if DA mode operates correctly. To do so,
+# it uses a small in-memory queue size, so that DA mode is initiated
+# rather soon, and disk spooling used. There is some uncertainty (based
+# on machine speeds), but in general the test should work rather well.
+# We add a few messages after the initial run, just so that we can
+# check everything recovers from DA mode correctly.
+# added 2009-04-22 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo "THIS TEST DOES NOT YET WORK RELIABLY!"
+echo "testing main message queue in DA mode (going to disk)"
+rm -f work rsyslog.out.log
+rm -rf test-spool
+mkdir test-spool
+rm -f work rsyslog.out.log rsyslog.out.log.save # work files
+../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/da-mainmsg-q.conf &
+sleep 1
+echo "rsyslogd started with pid " `cat rsyslog.pid`
+#
+# part1: send first 50 messages (in memory, only)
+#
+./tcpflood 127.0.0.1 13514 1 50
+if [ "$?" -ne "0" ]; then
+  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
+  cp rsyslog.out.log rsyslog.out.log.save
+fi
+ls -l test-spool
+sleep 1 # we need this so that rsyslogd can receive all outstanding messages
+#
+# part 2: send bunch of messages. This should trigger DA mode
+#
+# 20000 messages should be enough - the disk test is slow enough ;)
+./tcpflood 127.0.0.1 13514 2 20000 50
+if [ "$?" -ne "0" ]; then
+  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
+  cp rsyslog.out.log rsyslog.out.log.save
+fi
+ls -l test-spool
+sleep 5 # we need this so that rsyslogd can receive all outstanding messages
+#
+# send another handful
+#
+ls -l test-spool
+./tcpflood 127.0.0.1 13514 1 50 20050
+if [ "$?" -ne "0" ]; then
+  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
+  cp rsyslog.out.log rsyslog.out.log.save
+fi
+sleep 1 # we need this so that rsyslogd can receive all outstanding messages
+# 
+# clean up and check test result
+#
+kill `cat rsyslog.pid`
+rm -f work
+sort < rsyslog.out.log > work
+./chkseq work 0 20099
+if [ "$?" -ne "0" ]; then
+ # rm -f work rsyslog.out.log
+  echo "sequence error detected"
+  exit 1
+fi
+rm -f work rsyslog.out.log
+rm -rf test-spool
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 8dbc201b..2ca796ca 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -6,6 +6,7 @@
  * argv[2]	target port
  * argv[3]	number of connections
  * argv[4]	number of messages to send (connection is random)
+ * argv[5]	initial message number (optional)
  *
  * Part of the testbench for rsyslog.
  *
@@ -51,6 +52,7 @@ static int targetPort;
 static int numMsgsToSend; /* number of messages to send */
 static int numConnections; /* number of connections to create */
 static int *sockArray;  /* array of sockets to use */
+static int msgNum = 0;	/* initial message number to start with */
 
 
 /* open a single tcp connection
@@ -154,8 +156,6 @@ int sendMessages(void)
 	int lenBuf;
 	int lenSend;
 	char buf[2048];
-	char msgBuf[128];
-	size_t lenMsg;
 
 	srand(time(NULL));	/* seed is good enough for our needs */
 
@@ -168,19 +168,20 @@ int sendMessages(void)
 			socknum = i - (numMsgsToSend - numConnections);
 		else
 			socknum = rand() % numConnections;
-		lenBuf = sprintf(buf, "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i);
+		lenBuf = sprintf(buf, "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum);
 		lenSend = send(sockArray[socknum], buf, lenBuf, 0);
 		if(lenSend != lenBuf) {
 			printf("\r%5.5d\n", i);
 			fflush(stdout);
 			perror("send test data");
-			printf("send() failed at socket %d, index %d\n", socknum, i);
+			printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum);
 			fflush(stderr);
 			return(1);
 		}
 		if(i % 100 == 0) {
 			printf("\r%5.5d", i);
 		}
+		++msgNum;
 	}
 	printf("\r%5.5d messages sent\n", i);
 
@@ -262,9 +263,9 @@ int main(int argc, char *argv[])
 
 	setvbuf(stdout, buf, _IONBF, 48);
 
-	if(argc != 5) {
+	if(argc != 5 && argc != 6) {
 		printf("Invalid call of tcpflood\n");
-		printf("Usage: tcpflood target-host target-port num-connections num-messages\n");
+		printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n");
 		exit(1);
 	}
 	
@@ -272,6 +273,8 @@ int main(int argc, char *argv[])
 	targetPort = atoi(argv[2]);
 	numConnections = atoi(argv[3]);
 	numMsgsToSend = atoi(argv[4]);
+	if(argc == 6)
+		msgNum = atoi(argv[5]);
 
 	if(openConnections() != 0) {
 		printf("error opening connections\n");
diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf
new file mode 100644
index 00000000..2d0e5b92
--- /dev/null
+++ b/tests/testsuites/da-mainmsg-q.conf
@@ -0,0 +1,20 @@
+# Test for DA mode in main message queue (see .sh file for details)
+# rgerhards, 2009-04-22
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$ErrorMessagesToStderr off
+
+# set spool locations and switch queue to disk assisted mode
+$WorkDirectory test-spool
+$MainMsgQueueSize 100 # this *should* trigger moving on to DA mode...
+$MainMsgQueueHighWatermark 80
+$MainMsgQueueLowWatermark 40
+$MainMsgQueueFilename mainq
+$MainMsgQueueType linkedlist
+$MainMsgQueueDeqzezeSlowdown 1000
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+:msg, contains, "msgnum:" ?dynfile;outfmt