fixed problem in testbench & added new test (not yet integrated)

The test is a setup scenario for this bug tracker:
http://bugzilla.adiscon.com/show_bug.cgi?id=241

6 files changed, 139 insertions, 5 deletions

diff --git a/tests/diag.sh b/tests/diag.sh
index 8fbe42ed..d1242fb1 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -84,15 +84,27 @@ case $1 in
 			exit 1
 		fi
 		;;
+   'get-mainqueuesize') # show the current main queue size
+		if [ "$2" == "2" ]
+		then
+			echo getmainmsgqueuesize | ./diagtalker -p13501
+		else
+			echo getmainmsgqueuesize | ./diagtalker
+		fi
+		;;
    'wait-queueempty') # wait for main message queue to be empty. $2 is the instance.
 		if [ "$2" == "2" ]
 		then
-			echo WaitMainQueueEmpty | ./diagtalker
+			echo WaitMainQueueEmpty | ./diagtalker -p13501
 		else
 			echo WaitMainQueueEmpty | ./diagtalker
 		fi
 		;;
    'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance.
+		if [ "$2" == "2" ]
+		then
+		   echo Shutting down instance 2
+		fi
    		$srcdir/diag.sh wait-queueempty $2
 		kill `cat rsyslog$2.pid`
 		# note: we do not wait for the actual termination!
diff --git a/tests/diagtalker.c b/tests/diagtalker.c
index 6a721e47..0d728bda 100644
--- a/tests/diagtalker.c
+++ b/tests/diagtalker.c
@@ -1,7 +1,7 @@
 /* A yet very simple tool to talk to imdiag (this replaces the
  * previous Java implementation in order to get fewer dependencies).
  *
- * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2010,2011 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -126,7 +126,7 @@ doProcessing()
 		len = strlen(line);
 		sendCmd(fd, line, len);
 		waitRsp(fd, line, sizeof(line));
-		printf("imdiag: %s", line);
+		printf("imdiag[%d]: %s", targetPort, line);
 	}
 }
 
@@ -139,7 +139,7 @@ int main(int argc, char *argv[])
 	int ret = 0;
 	int opt;
 
-	while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) {
+	while((opt = getopt(argc, argv, "t:p:")) != -1) {
 		switch (opt) {
 		case 't':	targetIP = optarg;
 				break;
diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh
new file mode 100755
index 00000000..e2f0a0d8
--- /dev/null
+++ b/tests/rcvr_fail_restore.sh
@@ -0,0 +1,96 @@
+# Copyright (C) 2011 by Rainer Gerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo \[rcvr_fail_restore.sh\]: test failed receiver restore case
+source $srcdir/diag.sh init
+#
+# STEP1: start both instances and send 1000 messages.
+# Note: receiver is instance 2, sender instance 1.
+#
+# start up the instances. Note that the envrionment settings can be changed to
+# set instance-specific debugging parameters!
+export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log2"
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+export RSYSLOG_DEBUGLOG="log"
+#valgrind="valgrind"
+source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf
+# re-set params so that new instances do not thrash it...
+unset RSYSLOG_DEBUG
+unset RSYSLOG_DEBUGLOG
+
+# now inject the messages into instance 2. It will connect to instance 1,
+# and that instance will record the data.
+source $srcdir/diag.sh injectmsg  1 1000
+source $srcdir/diag.sh wait-queueempty
+./msleep 3000 # make sure some retries happen (retry interval is set to 1 second)
+#
+# Step 2: shutdown receiver, then send some more data, which then
+# needs to go into the queue.
+#
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+source $srcdir/diag.sh injectmsg  1001 10000
+sleep 1 # we need to wait, otherwise we may be so fast that the receiver
+# comes up before we have finally suspended the action
+source $srcdir/diag.sh get-mainqueuesize
+ls -l test-spool
+
+#
+# Step 3: restart receiver, wait that the sender drains its queue
+#
+#export RSYSLOG_DEBUGLOG="log2"
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+echo waiting for sender to drain queue [may need a short while]
+source $srcdir/diag.sh wait-queueempty
+ls -l test-spool
+OLDFILESIZE=$(stat -c%s test-spool/mainq.00000001)
+echo file size to expect is $OLDFILESIZE
+
+
+#
+# Step 4: send new data. Queue files are not permitted to grow now
+# (but one file continous to exist).
+#
+source $srcdir/diag.sh injectmsg  11001 10
+source $srcdir/diag.sh wait-queueempty
+
+# at this point, the queue file shall not have grown. Note
+# that we MUST NOT shut down the instance right now, because it
+# would clean up the queue files! So we need to do our checks
+# first (here!).
+ls -l test-spool
+NEWFILESIZE=$(stat -c%s test-spool/mainq.00000001)
+if [ $NEWFILESIZE != $OLDFILESIZE ]
+then
+   echo file sizes do not match, expected $OLDFILESIZE, actual $NEWFILESIZE
+   echo this means that data has been written to the queue file where it
+   echo no longer should be written.
+   # abort will happen below, because we must ensure proper system shutdown
+   # HOWEVER, during actual testing it may be useful to do an exit here (so
+   # that e.g. the debug log is pointed right at the correct spot).
+   # exit 1
+fi
+
+
+#
+# Queue file size checks done. Now it is time to terminate the system
+# and see if everything could be received (the usual check, done here
+# for completeness, more or less as a bonus).
+#
+source $srcdir/diag.sh shutdown-when-empty
+source $srcdir/diag.sh wait-shutdown
+
+# now it is time to stop the receiver as well
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+# now abort test if we need to (due to filesize predicate)
+if [ $NEWFILESIZE != $OLDFILESIZE ]
+then
+   exit 1
+fi
+# do the final check
+source $srcdir/diag.sh seq-check 1 11010
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf
index 9e9e28fe..d76e2d15 100644
--- a/tests/testsuites/diag-common.conf
+++ b/tests/testsuites/diag-common.conf
@@ -13,4 +13,7 @@ $IMDiagServerRun 13500
 $template startupfile,"rsyslogd.started" # trick to use relative path names!
 :syslogtag, contains, "rsyslogd"  ?startupfile
 
-$ErrorMessagesToStderr off
+# I have disabled the directive below, so that we see errors in testcase
+# creation. I am not sure why it was present in the first place, so for
+# now I just leave it commented out -- rgerhards, 2011-03-30
+#$ErrorMessagesToStderr off
diff --git a/tests/testsuites/rcvr_fail_restore_rcvr.conf b/tests/testsuites/rcvr_fail_restore_rcvr.conf
new file mode 100644
index 00000000..38ebad29
--- /dev/null
+++ b/tests/testsuites/rcvr_fail_restore_rcvr.conf
@@ -0,0 +1,8 @@
+$IncludeConfig diag-common2.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# then SENDER sends to this port (not tcpflood!)
+$InputTCPServerRun 13515
+
+$template outfmt,"%msg:F,58:2%\n"
+:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/rcvr_fail_restore_sender.conf b/tests/testsuites/rcvr_fail_restore_sender.conf
new file mode 100644
index 00000000..6b11aa4a
--- /dev/null
+++ b/tests/testsuites/rcvr_fail_restore_sender.conf
@@ -0,0 +1,15 @@
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# this listener is for message generation by the test framework!
+$InputTCPServerRun 13514
+
+$WorkDirectory test-spool
+$MainMsgQueueMaxFileSize 1g
+$MainMsgQueueFileName mainq
+
+# we use the shortest resume interval a) to let the test not run too long 
+# and b) make sure some retries happen before the reconnect
+$ActionResumeInterval 1
+$ActionResumeRetryCount -1
+*.*	@@127.0.0.1:13515