Merge branch 'v4-stable' into v5-stable

Conflicts:
	ChangeLog
	configure.ac
	doc/manual.html
	runtime/debug.c
	runtime/stream.c
	tests/Makefile.am
	tests/diskqueue.sh
	tests/nettester.c
	tools/omfile.c

52 files changed, 1182 insertions, 64 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index a31a6f42..747b10cf 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -17,6 +17,21 @@ TESTS = $(TESTRUNS) cfg.sh \
 	sndrcv_gzip.sh \
 	sndrcv_udp.sh \
 	sndrcv_udp_nonstdpt.sh \
+	asynwr_simple.sh \
+	asynwr_timeout.sh \
+	asynwr_small.sh \
+	asynwr_tinybuf.sh \
+	wr_large_async.sh \
+	wr_large_sync.sh \
+	asynwr_deadlock.sh \
+	asynwr_deadlock2.sh \
+	asynwr_deadlock4.sh \
+	gzipwr_large.sh \
+	gzipwr_large_dynfile.sh \
+	dynfile_invld_async.sh \
+	dynfile_invld_sync.sh \
+	dynfile_invalid2.sh \
+	complex1.sh \
 	queue-persist.sh \
 	pipeaction.sh \
 	execonlyonce.sh \
@@ -99,6 +114,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/2.parse1 \
 	   testsuites/3.parse1 \
 	   testsuites/4.parse1 \
+	   testsuites/snare.parse1 \
 	   testsuites/oversizeTag-1.parse1 \
 	   testsuites/weird.parse1 \
 	   testsuites/date1.parse1 \
@@ -117,10 +133,15 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/reallife.parse2 \
 	   testsuites/parse3.conf \
 	   testsuites/reallife.parse3 \
+	   testsuites/parse-nodate.conf \
 	   testsuites/parse_invld_regex.conf \
 	   testsuites/samples.parse_invld_regex \
 	   testsuites/parse-3164-buggyday.conf \
 	   testsuites/samples.parse-3164-buggyday \
+	   testsuites/snare_ccoff_udp.conf \
+	   testsuites/samples.snare_ccoff_udp \
+	   testsuites/snare_ccoff_udp2.conf \
+	   testsuites/samples.snare_ccoff_udp2 \
 	   testsuites/omod-if-array.conf \
 	   testsuites/1.omod-if-array \
 	   testsuites/1.field1 \
@@ -184,6 +205,36 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/sndrcv_gzip_rcvr.conf \
 	   pipeaction.sh \
 	   testsuites/pipeaction.conf \
+	   asynwr_simple.sh \
+	   testsuites/asynwr_simple.conf \
+	   asynwr_timeout.sh \
+	   testsuites/asynwr_timeout.conf \
+	   asynwr_small.sh \
+	   testsuites/asynwr_small.conf \
+	   asynwr_tinybuf.sh \
+	   testsuites/asynwr_tinybuf.conf \
+	   wr_large_async.sh \
+	   wr_large_sync.sh \
+	   wr_large.sh \
+	   testsuites/wr_large.conf \
+	   asynwr_deadlock.sh \
+	   testsuites/asynwr_deadlock.conf \
+	   asynwr_deadlock2.sh \
+	   testsuites/asynwr_deadlock2.conf \
+	   asynwr_deadlock4.sh \
+	   testsuites/asynwr_deadlock4.conf \
+	   gzipwr_large.sh \
+	   testsuites/gzipwr_large.conf \
+	   gzipwr_large_dynfile.sh \
+	   testsuites/gzipwr_large_dynfile.conf \
+	   complex1.sh \
+	   testsuites/complex1.conf \
+	   dynfile_invld_async.sh \
+	   dynfile_invld_sync.sh \
+	   dynfile_cachemiss.sh \
+	   testsuites/dynfile_cachemiss.conf \
+	   dynfile_invalid2.sh \
+	   testsuites/dynfile_invalid2.conf \
 	   proprepltest.sh \
 	   testsuites/rfctag.conf \
 	   testsuites/master.rfctag \
diff --git a/tests/asynwr_deadlock.sh b/tests/asynwr_deadlock.sh
new file mode 100755
index 00000000..dc08355e
--- /dev/null
+++ b/tests/asynwr_deadlock.sh
@@ -0,0 +1,23 @@
+# This is test case from practice, with the version we introduced it, it
+# caused a deadlock on shutdown. I have added it to the test suite to automatically
+# detect such things in the future.
+#
+# added 2010-03-17 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ================================================================================
+echo TEST: \[asynwr_deadlock.sh\]: a case known to have caused a deadlock in the past
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_deadlock.conf
+# just send one message
+source $srcdir/diag.sh tcpflood -m1
+# sleep is important! need to make sure the instance is inactive
+sleep 1
+# now try shutdown. The actual test is if the process does hang here!
+echo "processing must continue soon"
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 0
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_deadlock2.sh b/tests/asynwr_deadlock2.sh
new file mode 100755
index 00000000..1190f67a
--- /dev/null
+++ b/tests/asynwr_deadlock2.sh
@@ -0,0 +1,69 @@
+# This is test case from practice, with the version we introduced it, it
+# caused a deadlock during processing (when the a stream was purged from the
+# dynafile cache).
+# We added this as a standard test in the hopes that iw will help
+# detect such things in the future.
+#
+# The problem that originally caused this test to fail was:
+# We write to files asynchronously (with the async writer thread). There is
+# no signaling done when the file stream is closed. That can lead to the writer
+# process hanging in memory, that in turn leads to the main thread waiting on a
+# condition that never occurs (because it would need to be signalled by the
+# async writer). Even worse, in that case, the async writer was signalled invalid
+# in such a way that when it received a wakeup, it thought it shall not terminate,
+# but received a spurios wakeup due to timeout and no data to write. In that case
+# it (correctly) concluded that it would not need to timeout until a new buffer write
+# was done (in which case it would receive a wakeup). As such, it went into an eternal
+# wait. However, the invalid signaling did not take into account that it did not
+# signal the async writer to shut down. So the main thread went into a condition
+# wait - and thus we had a deadlock. That situation occured only under very specific
+# cirumstances. As far as the analysis goes, the following need to happen:
+# 1. buffers on that file are being flushed
+# 2. no new data arrives
+# 3. the inactivity timeout has not yet expired
+# 4. *then* (and only then) the stream is closed or destructed
+# In that, 1 to 4 are prequisites for the deadlock which will happen in 4. However,
+# for it to happen, we also need the right "timing". There is a race between the
+# main thread and the async writer thread. The deadlock will only happen under
+# the "right" circumstances, which basically means it will not happen always.
+# In order to create this case as reliable as possible, I have used
+# the "$OMFileFlushOnTXEnd on" directive
+# inside my test case. It makes sure that #1 above happens. The test uses a dynafile
+# cache size of 4, and the load generator generates data for 5 different dynafiles.
+# So over time, we will hit a spot where 4 dynafiles are open and the 5th file name
+# is generated. As such, one file needs to be discarded. Thanks to FlushOnTXEnd, we
+# now likely have #2 in place and thanks to the load pattern generated, we most
+# probably have #3 in place. During the dynafile cache displacement of the oldest
+# entry, #4 is generated. At this point, we have the deadlock we are testing for.
+# Note that this deadlock does not necessarily lead to a total lockup of rsyslogd.
+# Parts of it continue to operate. But in our test setup, this means data is
+# received and placed into the main queue. Once it's high water mark is hit, data 
+# is still being enqueued, but at a slow rate. So if one is patient enough, the load
+# generator will be able to finish. However, rsyslogd will never process the data
+# it received because it is locked in the deadlock caused by #4 above.
+# Note that "$OMFileFlushOnTXEnd on" is not causing this behaviour. We just use it
+# to (quite) reliably cause the failure condition. The failure described above
+# (in version 4.6.1) was also present when the setting was set to "off", but its
+# occurence was very much less probable - because the perquisites are then much
+# harder to hit. without it, the test may need to run for several hours before
+# we hit all failure conditions.
+#
+# added 2010-03-17 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo =================================================================================
+echo TEST: \[asynwr_deadlock2.sh\]: a case known to have caused a deadlock in the past
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_deadlock2.conf
+# send 20000 messages, each close to 2K (non-randomized!), so that we can fill
+# the buffers and hopefully run into the "deadlock".
+source $srcdir/diag.sh tcpflood -m20000 -d1800 -P129 -i1 -f5
+# the sleep below is needed to prevent too-early termination of the tcp listener
+sleep 1
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+cat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 1 20000 -E
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_deadlock4.sh b/tests/asynwr_deadlock4.sh
new file mode 100755
index 00000000..a3452f5b
--- /dev/null
+++ b/tests/asynwr_deadlock4.sh
@@ -0,0 +1,25 @@
+# This is test case from practice, with the version we introduced it, it
+# caused a deadlock during processing.
+# We added this as a standard test in the hopes that iw will help
+# detect such things in the future.
+#
+# This is a test that is constructed similar to asynwr_deadlock2.sh, but
+# can produce problems in a simpler way.
+#
+# added 2010-03-18 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo =================================================================================
+echo TEST: \[asynwr_deadlock4.sh\]: a case known to have caused a deadlock in the past
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_deadlock4.conf
+# send 20000 messages, each close to 2K (non-randomized!), so that we can fill
+# the buffers and hopefully run into the "deadlock".
+source $srcdir/diag.sh tcpflood -m20000 -d18 -P129 -i1 -f5
+# sleep is important! need to make sure the instance is inactive
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 1 20000 -E
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_simple.sh b/tests/asynwr_simple.sh
new file mode 100755
index 00000000..eb87443c
--- /dev/null
+++ b/tests/asynwr_simple.sh
@@ -0,0 +1,18 @@
+# This is test driver for testing asynchronous file output.
+#
+# added 2010-03-09 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_simple.sh\]: simple test for async file writing
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_simple.conf
+# send 35555 messages, make sure file size is not a multiple of
+# 10K, the buffer size!
+source $srcdir/diag.sh tcpflood -m35555
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 35554
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_small.sh b/tests/asynwr_small.sh
new file mode 100755
index 00000000..97818f6e
--- /dev/null
+++ b/tests/asynwr_small.sh
@@ -0,0 +1,26 @@
+# This tests async writing with only a small set of data. That
+# shall result in data staying in buffers until shutdown, what
+# then will trigger some somewhat complex logic in the stream
+# writer (open, write, close all during the stream close
+# opertion). It is vital that only few messages be sent.
+#
+# The main effort of this test is not (only) to see if we
+# receive the data, but rather to see if we get into an abort
+# condition.
+#
+# added 2010-03-09 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_small.sh\]: test for async file writing for few messages
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_small.conf
+# send 4000 messages
+source $srcdir/diag.sh tcpflood -m2
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 1
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_timeout.sh b/tests/asynwr_timeout.sh
new file mode 100755
index 00000000..b0bc5c28
--- /dev/null
+++ b/tests/asynwr_timeout.sh
@@ -0,0 +1,21 @@
+# This test writes to the output buffers, let's the output
+# write timeout (and write data) and then continue. The conf file
+# has a 2 second timeout, so we wait 4 seconds to be on the save side.
+#
+# added 2010-03-09 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_timeout.sh\]: test async file writing timeout writes
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_timeout.conf
+# send 35555 messages, make sure file size is not a multiple of
+# 10K, the buffer size!
+source $srcdir/diag.sh tcpflood -m 35555
+sleep 4 # wait for output writer to write and empty buffer
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 35554
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_tinybuf.sh b/tests/asynwr_tinybuf.sh
new file mode 100755
index 00000000..8eae1e26
--- /dev/null
+++ b/tests/asynwr_tinybuf.sh
@@ -0,0 +1,19 @@
+# This tests async writing with a very small output buffer (1 byte!),
+# so it stresses output buffer handling. This also means operations will
+# be somewhat slow, so we send only a small amounts of data.
+#
+# added 2010-03-09 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_tinybuf.sh\]: test async file writing with 1-byte buffer
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_tinybuf.conf
+# send 1000 messages, fairly enough to trigger problems
+source $srcdir/diag.sh tcpflood -m1000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 999
+source $srcdir/diag.sh exit
diff --git a/tests/chkseq.c b/tests/chkseq.c
index 6334d787..b22c8992 100644
--- a/tests/chkseq.c
+++ b/tests/chkseq.c
@@ -32,6 +32,7 @@
 #include "config.h"
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <getopt.h>
 
 int main(int argc, char *argv[])
@@ -40,14 +41,18 @@ int main(int argc, char *argv[])
 	int val;
 	int i;
 	int ret = 0;
+	int scanfOK;
 	int verbose = 0;
+	int bHaveExtraData = 0;
 	int dupsPermitted = 0;
 	int start = 0, end = 0;
 	int opt;
 	int nDups = 0;
+	int edLen;	/* length of extra data */
+	static char edBuf[500*1024]; /* buffer for extra data (pretty large to be on the save side...) */
 	char *file = NULL;
 
-	while((opt = getopt(argc, argv, "e:f:ds:v")) != EOF) {
+	while((opt = getopt(argc, argv, "e:f:ds:vE")) != EOF) {
 		switch((char)opt) {
 		case 'f':
 			file = optarg;
@@ -64,8 +69,11 @@ int main(int argc, char *argv[])
                 case 'v':
 			++verbose;
 			break;
-		default:printf("Invalid call of chkseq\n");
-			printf("Usage: chkseq file -sstart -eend -d\n");
+                case 'E':
+			bHaveExtraData = 1;
+			break;
+		default:printf("Invalid call of chkseq, optchar='%c'\n", opt);
+			printf("Usage: chkseq file -sstart -eend -d -E\n");
 			exit(1);
 		}
 	}
@@ -93,7 +101,17 @@ int main(int argc, char *argv[])
 	}
 
 	for(i = start ; i < end+1 ; ++i) {
-		if(fscanf(fp, "%d\n", &val) != 1) {
+		if(bHaveExtraData) {
+			scanfOK = fscanf(fp, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0;
+			if(edLen != (int) strlen(edBuf)) {
+				printf("extra data length specified %d, but actually is %ld in record %d\n",
+					edLen, (long) strlen(edBuf), i);
+				exit(1);
+			}
+		} else {
+			scanfOK = fscanf(fp, "%d\n", &val) == 1 ? 1 : 0;
+		}
+		if(!scanfOK) {
 			printf("scanf error in index i=%d\n", i);
 			exit(1);
 		}
@@ -116,5 +134,10 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
+	if(!feof(fp)) {
+		printf("end of processing, but NOT end of file!\n");
+		exit(1);
+	}
+
 	exit(ret);
 }
diff --git a/tests/complex1.sh b/tests/complex1.sh
new file mode 100755
index 00000000..b5dc2c9d
--- /dev/null
+++ b/tests/complex1.sh
@@ -0,0 +1,21 @@
+# This is a rather complex test that runs a number of features together.
+#
+# added 2010-03-16 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ====================================================================================
+echo TEST: \[complex1.sh\]: complex test with gzip and multiple action queues
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup complex1.conf
+# send 30,000 messages of 400 bytes plus header max, via three dest ports
+source $srcdir/diag.sh tcpflood -m40000 -rd400 -P129 -f5 -n3 -c15 -i1
+sleep 2 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+ls rsyslog.out.*.log
+zcat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 1 40000 -E
+source $srcdir/diag.sh exit
diff --git a/tests/diag.sh b/tests/diag.sh
index 98228b12..40b8db2e 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -8,6 +8,7 @@
 #valgrind="valgrind --malloc-fill=ff --free-fill=fe --log-fd=1"
 #valgrind="valgrind --tool=drd --log-fd=1"
 #valgrind="valgrind --tool=helgrind --log-fd=1"
+#valgrind="valgrind --tool=exp-ptrcheck --log-fd=1"
 #set -o xtrace
 #export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
 #export RSYSLOG_DEBUGLOG="log"
@@ -15,16 +16,19 @@ case $1 in
    'init')	$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
 		cp $srcdir/testsuites/diag-common.conf diag-common.conf
 		cp $srcdir/testsuites/diag-common2.conf diag-common2.conf
+		rm -f rsyslog.action.*.include
 		rm -f rsyslogd.started work-*.conf
 		rm -f rsyslogd2.started work-*.conf
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
+		rm -f rsyslog.out.*.log
 		rm -rf test-spool
 		rm -f core.* vgcore.*
 		mkdir test-spool
 		;;
    'exit')	rm -f rsyslogd.started work-*.conf diag-common.conf
-   		rm -f rsyslogd2.started diag-common2.conf
+   		rm -f rsyslogd2.started diag-common2.conf rsyslog.action.*.include
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
+		rm -f rsyslog.out.*.log
 		rm -rf test-spool
 		echo  -------------------------------------------------------------------------------
 		;;
@@ -35,14 +39,16 @@ case $1 in
 		;;
    'wait-startup') # wait for rsyslogd startup ($2 is the instance)
 		while test ! -f rsyslogd$2.started; do
-			true
+			#true
+			sleep 0.1 # if this is not supported by all platforms, use above!
 		done
 		echo "rsyslogd$2 started with pid " `cat rsyslog$2.pid`
 		;;
    'wait-shutdown')  # actually, we wait for rsyslog.pid to be deleted. $2 is the
    		# instance
 		while test -f rsyslog$2.pid; do
-			true
+			#true
+			sleep 0.1 # if this is not supported by all platforms, use above!
 		done
 		if [ -e core.* ]
 		then
@@ -69,7 +75,7 @@ case $1 in
 		# note: we do not wait for the actual termination!
 		;;
    'tcpflood') # do a tcpflood run and check if it worked params are passed to tcpflood
-		./tcpflood $2 $3 $4 $5 $6 $7 $8
+		./tcpflood $2 $3 $4 $5 $6 $7 $8 $9
 		if [ "$?" -ne "0" ]; then
 		  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
 		  cp rsyslog.out.log rsyslog.out.log.save
@@ -92,6 +98,7 @@ case $1 in
 		;;
    'seq-check') # do the usual sequence check to see if everything was properly received. $2 is the instance.
 		rm -f work
+		cp rsyslog.out.log work-presort
 		sort < rsyslog.out.log > work
 		# $4... are just to have the abilit to pass in more options...
 		# add -v to chkseq if you need more verbose output
@@ -113,6 +120,18 @@ case $1 in
 		  exit 1
 		fi
 		;;
+   'gzip-seq-check') # do the usual sequence check, but for gzip files
+		rm -f work
+		ls -l rsyslog.out.log
+		gunzip < rsyslog.out.log | sort > work
+		ls -l work
+		# $4... are just to have the abilit to pass in more options...
+		./chkseq -fwork -v -s$2 -e$3 $4 $5 $6 $7
+		if [ "$?" -ne "0" ]; then
+		  echo "sequence error detected"
+		  exit 1
+		fi
+		;;
    'nettester') # perform nettester-based tests
    		# use -v for verbose output!
 		./nettester -t$2 -i$3 $4
diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh
index c826d32c..17b0ba4e 100755
--- a/tests/diskqueue-fsync.sh
+++ b/tests/diskqueue-fsync.sh
@@ -9,7 +9,7 @@ echo \[diskqueue-fsync.sh\]: testing queue disk-only mode, fsync case
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup diskqueue-fsync.conf
 # 1000 messages should be enough - the disk fsync test is very slow!
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000
+source $srcdir/diag.sh tcpflood -m1000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 999
diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh
index e91f0633..b871e9eb 100755
--- a/tests/diskqueue.sh
+++ b/tests/diskqueue.sh
@@ -6,11 +6,14 @@
 # This file is part of the rsyslog project, released  under GPLv3
 # uncomment for debugging support:
 echo \[diskqueue.sh\]: testing queue disk-only mode
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup diskqueue.conf
 # 20000 messages should be enough - the disk test is slow enough ;)
 sleep 4
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000
+source $srcdir/diag.sh tcpflood -m20000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 19999
diff --git a/tests/dynfile_cachemiss.sh b/tests/dynfile_cachemiss.sh
new file mode 100755
index 00000000..6e2d9cca
--- /dev/null
+++ b/tests/dynfile_cachemiss.sh
@@ -0,0 +1,34 @@
+# This test checks if omfile segfaults when a file open() in dynacache mode fails.
+# The test is mimiced after a real-life scenario (which, of course, was much more
+# complex).
+#
+# added 2010-03-09 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[dynfile_cachemiss.sh\]: test open fail for dynafiles with `cat rsyslog.action.1.include`
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup dynfile_cachemiss.conf
+# we send handcrafted message. We have a dynafile cache of 4, and now send one message
+# each to fill up the cache.
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:0"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:1"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:2"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:3"
+# the next one has caused a segfault in practice
+# note that /proc/rsyslog.error.file must not be creatable
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:/proc/rsyslog.error.file:boom"
+# some more writes
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:4"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:5"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:6"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:7"
+# done message generation
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+cat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 7
+source $srcdir/diag.sh exit
diff --git a/tests/dynfile_invalid2.sh b/tests/dynfile_invalid2.sh
new file mode 100755
index 00000000..b6fb3945
--- /dev/null
+++ b/tests/dynfile_invalid2.sh
@@ -0,0 +1,34 @@
+# This test checks if omfile segfaults when a file open() in dynacache mode fails.
+# The test is mimiced after a real-life scenario (which, of course, was much more
+# complex).
+#
+# added 2010-03-22 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[dynfile_invalid2.sh\]: test open fail for dynafiles
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup dynfile_invalid2.conf
+# we send handcrafted message. We have a dynafile cache of 4, and now send one message
+# each to fill up the cache.
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:0"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:1"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:2"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:3"
+# the next one has caused a segfault in practice
+# note that /proc/rsyslog.error.file must not be creatable
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:/proc/rsyslog.error.file:boom"
+# some more writes
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:4"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:5"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:6"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:7"
+# done message generation
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+cat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 7
+source $srcdir/diag.sh exit
diff --git a/tests/dynfile_invld_async.sh b/tests/dynfile_invld_async.sh
new file mode 100755
index 00000000..3c9b2045
--- /dev/null
+++ b/tests/dynfile_invld_async.sh
@@ -0,0 +1,2 @@
+echo "\$OMFileAsyncWriting on" > rsyslog.action.1.include
+source $srcdir/dynfile_cachemiss.sh
diff --git a/tests/dynfile_invld_sync.sh b/tests/dynfile_invld_sync.sh
new file mode 100755
index 00000000..cc6b6451
--- /dev/null
+++ b/tests/dynfile_invld_sync.sh
@@ -0,0 +1,2 @@
+echo "\$OMFileAsyncWriting off" > rsyslog.action.1.include
+source $srcdir/dynfile_cachemiss.sh
diff --git a/tests/execonlyonce.sh b/tests/execonlyonce.sh
index 6ddc89b5..8e184070 100755
--- a/tests/execonlyonce.sh
+++ b/tests/execonlyonce.sh
@@ -10,11 +10,11 @@ echo ===========================================================================
 echo \[execonlyonce.sh\]: test for the $ActionExecOnlyOnceEveryInterval directive
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup execonlyonce.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 1
+source $srcdir/diag.sh tcpflood -m10 -i1
 # now wait until the interval definitely expires
 sleep 4 # one more than the once inerval!
 # and inject another couple of messages
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 100
+source $srcdir/diag.sh tcpflood -m10 -i100
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 
diff --git a/tests/gzipwr_large.sh b/tests/gzipwr_large.sh
new file mode 100755
index 00000000..ffce06f6
--- /dev/null
+++ b/tests/gzipwr_large.sh
@@ -0,0 +1,20 @@
+# This tests writing large data records in gzip mode. We use up to 10K
+# record size.
+#
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[gzipwr_large.sh\]: test for gzip file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup gzipwr_large.conf
+# send 4000 messages of 10.000bytes plus header max, randomized
+source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129
+sleep 1 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh gzip-seq-check 0 3999 -E
+source $srcdir/diag.sh exit
diff --git a/tests/gzipwr_large_dynfile.sh b/tests/gzipwr_large_dynfile.sh
new file mode 100755
index 00000000..ebd4c558
--- /dev/null
+++ b/tests/gzipwr_large_dynfile.sh
@@ -0,0 +1,36 @@
+# This tests writing large data records in gzip mode. We also write it to
+# 5 different dynafiles, with a dynafile cache size set to 4. So this stresses
+# both the input side, as well as zip writing, async writing and the dynafile
+# cache logic.
+#
+# This test is a bit timing-dependent on the tcp reception side, so if it fails
+# one may look into the timing first. The main issue is that the testbench
+# currently has no good way to know if the tcp receiver is finished. This is NOT
+# a problem in rsyslogd, but only of the testbench.
+#
+# Note that we do not yet have sufficient support for dynafiles in diag.sh,
+# so we mangle some files here manually.
+#
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ====================================================================================
+echo TEST: \[gzipwr_large_dynfile.sh\]: test for gzip file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup gzipwr_large_dynfile.conf
+# send 4000 messages of 10.000bytes plus header max, randomized
+source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129 -f5
+sleep 2 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+gunzip < rsyslog.out.0.log > rsyslog.out.log
+gunzip < rsyslog.out.1.log >> rsyslog.out.log
+gunzip < rsyslog.out.2.log >> rsyslog.out.log
+gunzip < rsyslog.out.3.log >> rsyslog.out.log
+gunzip < rsyslog.out.4.log >> rsyslog.out.log
+#cat rsyslog.out.* > rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 3999 -E
+source $srcdir/diag.sh exit
diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh
index 47a33cb9..ad2b44f8 100755
--- a/tests/imtcp-multiport.sh
+++ b/tests/imtcp-multiport.sh
@@ -11,7 +11,7 @@ echo ===========================================================================
 echo \[imtcp-multiport.sh\]: testing imtcp multiple listeners
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup imtcp-multiport.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000
+source $srcdir/diag.sh tcpflood -p13514 -m10000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 9999
@@ -23,7 +23,7 @@ source $srcdir/diag.sh exit
 #
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup imtcp-multiport.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13515 1 10000
+source $srcdir/diag.sh tcpflood -p13515 -m10000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 9999
@@ -35,7 +35,7 @@ source $srcdir/diag.sh exit
 #
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup imtcp-multiport.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13516 1 10000
+source $srcdir/diag.sh tcpflood -p13516 -m10000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 9999
diff --git a/tests/longrun.sh b/tests/longrun.sh
new file mode 100755
index 00000000..407e9fa9
--- /dev/null
+++ b/tests/longrun.sh
@@ -0,0 +1,30 @@
+# This is a test-aid script to try running some tests through many iterations.
+# It is not yet used in the automated testbench, but I keep this file so that
+# I can use it whenever there is need to. As such, it currently does not have
+# parameters but is expected to be edited as needed. -- rgerhards, 2010-03-10
+#
+# use: ./longrun.sh testname.sh
+#
+# where testname.sh is the test to be run
+# to change other params, you need to edit the settings here below:
+MAXRUNS=10
+DISPLAYALIVE=100
+LOGFILE=runlog
+
+echo "logfile is $LOGFILE"
+echo "executing test $1"
+
+date > $LOGFILE
+
+for (( i=0; $i < 10000; i++ ))
+  do
+    if [ $(( i % DISPLAYALIVE )) -eq 0 ]; then
+      echo "$i iterations done"
+    fi
+    $1 >> $LOGFILE
+    if [ "$?" -ne "0" ]; then
+      echo "Test failed in iteration $i, review $LOGFILE for details!"
+      exit 1
+    fi
+  done
+echo "No failure in $i iterations."
diff --git a/tests/manytcp.sh b/tests/manytcp.sh
index 06e2fac9..ec8f2453 100755
--- a/tests/manytcp.sh
+++ b/tests/manytcp.sh
@@ -3,7 +3,10 @@ echo \[manytcp.sh\]: test concurrent tcp connections
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup manytcp.conf
 # the config file specifies exactly 1100 connections
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1000 40000
+source $srcdir/diag.sh tcpflood -c1000 -m40000
+# the sleep below is needed to prevent too-early termination of the tcp listener
+sleep 1
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown	# we need to wait until rsyslogd is finished!
 source $srcdir/diag.sh seq-check 0 39999
 source $srcdir/diag.sh exit
diff --git a/tests/nettester.c b/tests/nettester.c
index 7f1a6b1f..6932381e 100644
--- a/tests/nettester.c
+++ b/tests/nettester.c
@@ -48,6 +48,7 @@
 #include <netinet/in.h>
 #include <getopt.h>
 #include <errno.h>
+#include <ctype.h>
 
 #define EXIT_FAILURE 1
 #define INVALID_SOCKET -1
@@ -63,6 +64,7 @@ static int iPort = 12514; /* port which shall be used for sending data */
 static char* pszCustomConf = NULL;	/* custom config file, use -c conf to specify */
 static int verbose = 0;	/* verbose output? -v option */
 static int IPv4Only = 0;	/* use only IPv4 in rsyslogd call? */
+static int useDebugEnv = 0; /* activate debugging environment (for rsyslog debug log)? */
 
 /* these two are quick hacks... */
 int iFailed = 0;
@@ -225,10 +227,8 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 			   "-M../runtime/.libs:../.libs", NULL, NULL};
 	char confFile[1024];
 	char *newenviron[] = { NULL };
-	/* debug aide...
-	char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout",
+	char *newenvironDeb[] = { "RSYSLOG_DEBUG=debug nostdout",
 				"RSYSLOG_DEBUGLOG=log", NULL };
-	*/
 
 	sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir,
 		(pszCustomConf == NULL) ? configFile : pszCustomConf);
@@ -254,7 +254,7 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 		close(pipefd[1]);
 		close(pipefd[0]);
 		fclose(stdin);
-		execve("../tools/rsyslogd", newargv, newenviron);
+		execve("../tools/rsyslogd", newargv, (useDebugEnv) ? newenvironDeb : newenviron);
 	} else {            
 		close(pipefd[1]);
 		*pid = cpid;
@@ -265,6 +265,62 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 }
 
 
+/* This function unescapes a string of testdata. That it, escape sequences
+ * are converted into their one-character equivalent. While doing so, it applies
+ * C-like semantics. This was made necessary for easy integration of control
+ * characters inside test cases.  -- rgerhards, 2009-03-11
+ * Currently supported:
+ * \\	single backslash
+ * \n, \t, \r as in C
+ * \nnn where nnn is a 1 to 3 character octal sequence 
+ * Note that when a problem occurs, the end result is undefined. After all, this
+ * is for a testsuite generatort, it must not be 100% bullet proof (so do not
+ * copy this code into something that must be!). Also note that we do in-memory
+ * unescaping and assume that the string gets shorter but NEVER longer!
+ */
+void unescapeTestdata(char *testdata)
+{
+	char *pDst;
+	char *pSrc;
+	int i;
+	int c;
+
+	pDst = pSrc = testdata;
+	while(*pSrc) {
+		if(*pSrc == '\\') {
+			switch(*++pSrc) {
+			case '\\':	*pDst++ = *pSrc++;
+					break;
+			case 'n':	*pDst++ = '\n';
+					++pSrc;
+					break;
+			case 'r':	*pDst++ = '\r';
+					++pSrc;
+					break;
+			case 't':	*pDst++ = '\t';
+					++pSrc;
+					break;
+			case '0':
+			case '1':
+			case '2':
+			case '3':	c = *pSrc++ - '0';
+					i = 1; /* we already processed one digit! */
+					while(i < 3 && isdigit(*pSrc)) {
+						c = c * 8 + *pSrc++ - '0';
+						++i;
+					}
+					*pDst++ = c;
+					break;
+			default:	break;
+			}
+		} else {
+			*pDst++ = *pSrc++;
+		}
+	}
+	*pDst = '\0';
+}
+
+
 /* Process a specific test case. File name is provided.
  * Needs to return 0 if all is OK, something else otherwise.
  */
@@ -302,6 +358,7 @@ processTestFile(int fd, char *pszFileName)
 
 		testdata[strlen(testdata)-1] = '\0'; /* remove \n */
 		/* now we have the test data to send (we could use function pointers here...) */
+		unescapeTestdata(testdata);
 		if(inputMode == inputUDP) {
 			if(udpSend(testdata, strlen(testdata)) != 0)
 				return(2);
@@ -330,7 +387,7 @@ processTestFile(int fd, char *pszFileName)
 				ret = 1;
 		}
 
-		/* clean up after the try */
+		/* we need to free buffers, as we have potentially modified them! */
 		free(testdata);
 		testdata = NULL;
 		free(expected);
@@ -430,7 +487,7 @@ int main(int argc, char *argv[])
 	char buf[4096];
 	char testcases[4096];
 
-	while((opt = getopt(argc, argv, "4c:i:p:t:v")) != EOF) {
+	while((opt = getopt(argc, argv, "4dc:i:p:t:v")) != EOF) {
 		switch((char)opt) {
                 case '4':
 			IPv4Only = 1;
@@ -438,6 +495,9 @@ int main(int argc, char *argv[])
                 case 'c':
 			pszCustomConf = optarg;
 			break;
+                case 'd': 
+			useDebugEnv = 1;
+			break;
                 case 'i':
 			if(!strcmp(optarg, "udp"))
 				inputMode = inputUDP;
@@ -458,7 +518,7 @@ int main(int argc, char *argv[])
 			verbose = 1;
 			break;
 		default:printf("Invalid call of nettester, invalid option '%c'.\n", opt);
-			printf("Usage: nettester -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n");
+			printf("Usage: nettester -d -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n");
 			exit(1);
 		}
 	}
diff --git a/tests/parsertest.sh b/tests/parsertest.sh
index ea2334c2..28f4b088 100755
--- a/tests/parsertest.sh
+++ b/tests/parsertest.sh
@@ -10,6 +10,12 @@ source $srcdir/diag.sh nettester parse_invld_regex udp
 source $srcdir/diag.sh nettester parse_invld_regex tcp
 source $srcdir/diag.sh nettester parse-3164-buggyday udp
 source $srcdir/diag.sh nettester parse-3164-buggyday tcp
+source $srcdir/diag.sh nettester parse-nodate udp
+source $srcdir/diag.sh nettester parse-nodate tcp
+# the following samples can only be run over UDP as they are so
+# malformed they break traditional syslog/tcp framing...
+source $srcdir/diag.sh nettester snare_ccoff_udp udp
+source $srcdir/diag.sh nettester snare_ccoff_udp2 udp
 
 echo \[parsertest.sh]: redoing tests in IPv4-only mode
 source $srcdir/diag.sh nettester parse1 udp -4
@@ -22,4 +28,10 @@ source $srcdir/diag.sh nettester parse_invld_regex udp -4
 source $srcdir/diag.sh nettester parse_invld_regex tcp -4
 source $srcdir/diag.sh nettester parse-3164-buggyday udp -4
 source $srcdir/diag.sh nettester parse-3164-buggyday tcp -4
+source $srcdir/diag.sh nettester parse-nodate udp -4
+source $srcdir/diag.sh nettester parse-nodate tcp -4
+# UDP-only tests
+source $srcdir/diag.sh nettester snare_ccoff_udp udp -4
+source $srcdir/diag.sh nettester snare_ccoff_udp2 udp -4
+
 source $srcdir/diag.sh exit
diff --git a/tests/pipeaction.sh b/tests/pipeaction.sh
index 26a4c22a..c2201011 100755
--- a/tests/pipeaction.sh
+++ b/tests/pipeaction.sh
@@ -17,7 +17,7 @@ echo background cp process id is $CPPROCESS
 # now do the usual run
 source $srcdir/diag.sh startup pipeaction.conf
 # 20000 messages should be enough
-#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000
+#source $srcdir/diag.sh tcpflood -m20000
 source $srcdir/diag.sh injectmsg 0 20000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
diff --git a/tests/sndrcv_drvr.sh b/tests/sndrcv_drvr.sh
index 63afd6b5..3d613069 100755
--- a/tests/sndrcv_drvr.sh
+++ b/tests/sndrcv_drvr.sh
@@ -34,7 +34,7 @@ source $srcdir/diag.sh wait-startup 2
 
 # now inject the messages into instance 2. It will connect to instance 1,
 # and that instance will record the data.
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 $2 1
+source $srcdir/diag.sh tcpflood -m$2 -i1
 sleep 2 # make sure all data is received in input buffers
 # shut down sender when everything is sent, receiver continues to run concurrently
 # may be needed by TLS (once we do it): sleep 60
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 0439e33e..32bf959d 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -2,15 +2,28 @@
  * messages over them. This is used for stress-testing.
  *
  * Params
- * argv[1]	target address
- * argv[2]	target port
- * argv[3]	number of connections
- * argv[4]	number of messages to send (connection is random)
- * argv[5]	initial message number (optional)
+ * -t	target address (default 127.0.0.1)
+ * -p	target port (default 13514)
+ * -n	number of target ports (targets are in range -p..(-p+-n-1)
+ * -c	number of connections (default 1)
+ * -m	number of messages to send (connection is random)
+ * -i	initial message number (optional)
+ * -P	PRI to be used for generated messages (default is 167).
+ *      Specify the plain number without leading zeros
+ * -d   amount of extra data to add to message. If present, the
+ *      number itself will be added as third field, and the data
+ *      bytes as forth. Add -r to randomize the amount of extra
+ *      data included in the range 1..(value of -d).
+ * -r	randomize amount of extra data added (-d must be > 0)
+ * -f	support for testing dynafiles. If given, include a dynafile ID
+ *      in the range 0..(f-1) as the SECOND field, shifting all field values
+ *      one field to the right. Zero (default) disables this functionality.
+ * -M   the message to be sent. Disables all message format options, as
+ *      only that exact same message is sent.
  *
  * Part of the testbench for rsyslog.
  *
- * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2009, 2010 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -47,12 +60,21 @@
 /* Name of input file, must match $IncludeConfig in test suite .conf files */
 #define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */
 
-static char *targetIP;
-static int targetPort;
+#define MAX_EXTRADATA_LEN 100*1024
+
+static char *targetIP = "127.0.0.1";
+static char *msgPRI = "167";
+static int targetPort = 13514;
+static int numTargetPorts = 1;
+static int dynFileIDs = 0;
+static int extraDataLen = 0; /* amount of extra data to add to message */
+static int bRandomizeExtraData = 0; /* randomize amount of extra data added */
 static int numMsgsToSend; /* number of messages to send */
-static int numConnections; /* number of connections to create */
+static int numConnections = 1; /* number of connections to create */
 static int *sockArray;  /* array of sockets to use */
 static int msgNum = 0;	/* initial message number to start with */
+static int bShowProgress = 1; /* show progress messages */
+static char *MsgToSend = NULL; /* if non-null, this is the actual message to send */
 
 
 /* open a single tcp connection
@@ -61,16 +83,25 @@ int openConn(int *fd)
 {
 	int sock;
 	struct sockaddr_in addr;
+	int port;
 	int retries = 0;
+	int rnd;
 
 	if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) {
 		perror("socket()");
 		return(1);
 	}
 
+	/* randomize port if required */
+	if(numTargetPorts > 1) {
+		rnd = rand(); /* easier if we need value for debug messages ;) */
+		port = targetPort + (rnd % numTargetPorts);
+	} else {
+		port = targetPort;
+	}
 	memset((char *) &addr, 0, sizeof(addr));
 	addr.sin_family = AF_INET;
-	addr.sin_port = htons(targetPort);
+	addr.sin_port = htons(port);
 	if(inet_aton(targetIP, &addr.sin_addr)==0) {
 		fprintf(stderr, "inet_aton() failed\n");
 		return(1);
@@ -103,11 +134,13 @@ int openConnections(void)
 	char msgBuf[128];
 	size_t lenMsg;
 
-	write(1, "      open connections", sizeof("      open connections")-1);
+	if(bShowProgress)
+		write(1, "      open connections", sizeof("      open connections")-1);
 	sockArray = calloc(numConnections, sizeof(int));
 	for(i = 0 ; i < numConnections ; ++i) {
 		if(i % 10 == 0) {
-			printf("\r%5.5d", i);
+			if(bShowProgress)
+				printf("\r%5.5d", i);
 			//lenMsg = sprintf(msgBuf, "\r%5.5d", i);
 			//write(1, msgBuf, lenMsg);
 		}
@@ -136,11 +169,14 @@ void closeConnections(void)
 	char msgBuf[128];
 	size_t lenMsg;
 
-	write(1, "      close connections", sizeof("      close connections")-1);
+	if(bShowProgress)
+		write(1, "      close connections", sizeof("      close connections")-1);
 	for(i = 0 ; i < numConnections ; ++i) {
 		if(i % 10 == 0) {
-			lenMsg = sprintf(msgBuf, "\r%5.5d", i);
-			write(1, msgBuf, lenMsg);
+			if(bShowProgress) {
+				lenMsg = sprintf(msgBuf, "\r%5.5d", i);
+				write(1, msgBuf, lenMsg);
+			}
 		}
 		close(sockArray[i]);
 	}
@@ -164,12 +200,14 @@ int sendMessages(void)
 	int socknum;
 	int lenBuf;
 	int lenSend;
-	char buf[2048];
-
-	srand(time(NULL));	/* seed is good enough for our needs */
+	int edLen; /* actual extra data length to use */
+	char dynFileIDBuf[128] = "";
+	char buf[MAX_EXTRADATA_LEN + 1024];
+	char extraData[MAX_EXTRADATA_LEN + 1];
 
 	printf("Sending %d messages.\n", numMsgsToSend);
-	printf("\r%5.5d messages sent", 0);
+	if(bShowProgress)
+		printf("\r%8.8d messages sent", 0);
 	for(i = 0 ; i < numMsgsToSend ; ++i) {
 		if(i < numConnections)
 			socknum = i;
@@ -177,7 +215,27 @@ int sendMessages(void)
 			socknum = i - (numMsgsToSend - numConnections);
 		else
 			socknum = rand() % numConnections;
-		lenBuf = sprintf(buf, "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum);
+		if(MsgToSend == NULL) {
+			if(dynFileIDs > 0) {
+				sprintf(dynFileIDBuf, "%d:", rand() % dynFileIDs);
+			}
+			if(extraDataLen == 0) {
+				lenBuf = sprintf(buf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:\n",
+						       msgPRI, dynFileIDBuf, msgNum);
+			} else {
+				if(bRandomizeExtraData)
+					edLen = ((long) rand() + extraDataLen) % extraDataLen + 1;
+				else
+					edLen = extraDataLen;
+				memset(extraData, 'X', edLen);
+				extraData[edLen] = '\0';
+				lenBuf = sprintf(buf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s\n",
+						       msgPRI, dynFileIDBuf, msgNum, edLen, extraData);
+			}
+		} else {
+			/* use fixed message format from command line */
+			lenBuf = sprintf(buf, "%s\n", MsgToSend);
+		}
 		lenSend = send(sockArray[socknum], buf, lenBuf, 0);
 		if(lenSend != lenBuf) {
 			printf("\r%5.5d\n", i);
@@ -188,11 +246,12 @@ int sendMessages(void)
 			return(1);
 		}
 		if(i % 100 == 0) {
-			printf("\r%5.5d", i);
+			if(bShowProgress)
+				printf("\r%8.8d", i);
 		}
 		++msgNum;
 	}
-	printf("\r%5.5d messages sent\n", i);
+	printf("\r%8.8d messages sent\n", i);
 
 	return 0;
 }
@@ -251,17 +310,18 @@ tcpSend(char *buf, int lenBuf)
 }
 
 
-/* Run the test suite. This must be called with exactly one parameter, the
- * name of the test suite. For details, see file header comment at the top
- * of this file.
+/* Run the test.
  * rgerhards, 2009-04-03
  */
 int main(int argc, char *argv[])
 {
 	int ret = 0;
+	int opt;
 	struct sigaction sigAct;
 	static char buf[1024];
 
+	srand(time(NULL));	/* seed is good enough for our needs */
+
 	/* on Solaris, we do not HAVE MSG_NOSIGNAL, so for this reason
 	 * we block SIGPIPE (not an issue for this program)
 	 */
@@ -271,19 +331,44 @@ int main(int argc, char *argv[])
 	sigaction(SIGPIPE, &sigAct, NULL);
 
 	setvbuf(stdout, buf, _IONBF, 48);
-
-	if(argc != 5 && argc != 6) {
-		printf("Invalid call of tcpflood\n");
-		printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n");
-		exit(1);
-	}
 	
-	targetIP = argv[1];
-	targetPort = atoi(argv[2]);
-	numConnections = atoi(argv[3]);
-	numMsgsToSend = atoi(argv[4]);
-	if(argc == 6)
-		msgNum = atoi(argv[5]);
+	if(!isatty(1))
+		bShowProgress = 0;
+
+	while((opt = getopt(argc, argv, "f:t:p:c:m:i:P:d:n:M:r")) != -1) {
+		switch (opt) {
+		case 't':	targetIP = optarg;
+				break;
+		case 'p':	targetPort = atoi(optarg);
+				break;
+		case 'n':	numTargetPorts = atoi(optarg);
+				break;
+		case 'c':	numConnections = atoi(optarg);
+				break;
+		case 'm':	numMsgsToSend = atoi(optarg);
+				break;
+		case 'i':	msgNum = atoi(optarg);
+				break;
+		case 'P':	msgPRI = optarg;
+				break;
+		case 'd':	extraDataLen = atoi(optarg);
+				if(extraDataLen > MAX_EXTRADATA_LEN) {
+					fprintf(stderr, "-d max is %d!\n",
+						MAX_EXTRADATA_LEN);
+					exit(1);
+				}
+				break;
+		case 'r':	bRandomizeExtraData = 1;
+				break;
+		case 'f':	dynFileIDs = atoi(optarg);
+				break;
+		case 'M':	MsgToSend = optarg;
+				break;
+		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
+				exit (1);
+				break;
+		}
+	}
 
 	if(openConnections() != 0) {
 		printf("error opening connections\n");
@@ -295,7 +380,6 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
-	//closeConnections();
 	printf("End of tcpflood Run\n");
 
 	exit(ret);
diff --git a/tests/testsuites/asynwr_deadlock.conf b/tests/testsuites/asynwr_deadlock.conf
new file mode 100644
index 00000000..dc4045b0
--- /dev/null
+++ b/tests/testsuites/asynwr_deadlock.conf
@@ -0,0 +1,14 @@
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+
+$OMFileFlushOnTXEnd on
+$OMFileFlushInterval 10
+$OMFileFlushIOBufferSize 10k
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/asynwr_deadlock2.conf b/tests/testsuites/asynwr_deadlock2.conf
new file mode 100644
index 00000000..07811613
--- /dev/null
+++ b/tests/testsuites/asynwr_deadlock2.conf
@@ -0,0 +1,16 @@
+# rgerhards, 2010-03-17
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.%msg:F,58:2%.log" # use multiple dynafiles
+
+$OMFileFlushOnTXEnd on
+$OMFileFlushInterval 10
+$OMFileIOBufferSize 10k
+$OMFileAsyncWriting on
+$DynaFileCacheSize 4
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_deadlock4.conf b/tests/testsuites/asynwr_deadlock4.conf
new file mode 100644
index 00000000..f4308ff1
--- /dev/null
+++ b/tests/testsuites/asynwr_deadlock4.conf
@@ -0,0 +1,16 @@
+# rgerhards, 2010-03-17
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.log" # use multiple dynafiles
+
+$OMFileFlushOnTXEnd on
+$OMFileFlushInterval 10
+$OMFileIOBufferSize 10k
+$OMFileAsyncWriting on
+$DynaFileCacheSize 4
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_simple.conf b/tests/testsuites/asynwr_simple.conf
new file mode 100644
index 00000000..44b03f2b
--- /dev/null
+++ b/tests/testsuites/asynwr_simple.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileFlushIOBufferSize 10k
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_small.conf b/tests/testsuites/asynwr_small.conf
new file mode 100644
index 00000000..f04ce962
--- /dev/null
+++ b/tests/testsuites/asynwr_small.conf
@@ -0,0 +1,14 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_timeout.conf b/tests/testsuites/asynwr_timeout.conf
new file mode 100644
index 00000000..44b03f2b
--- /dev/null
+++ b/tests/testsuites/asynwr_timeout.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileFlushIOBufferSize 10k
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_tinybuf.conf b/tests/testsuites/asynwr_tinybuf.conf
new file mode 100644
index 00000000..01dec4d8
--- /dev/null
+++ b/tests/testsuites/asynwr_tinybuf.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileIOBufferSize 1
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/complex1.conf b/tests/testsuites/complex1.conf
new file mode 100644
index 00000000..9e2441d4
--- /dev/null
+++ b/tests/testsuites/complex1.conf
@@ -0,0 +1,81 @@
+# complex test case with multiple actions in gzip mode
+# rgerhards, 2009-05-22
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.%inputname%.%msg:F,58:2%.log"
+
+## RULESET with listener
+$Ruleset R13514
+# queue params:
+$ActionQueueTimeoutShutdown 60000
+$ActionQueueSize 5000
+$ActionQueueSaveOnShutdown on
+$ActionQueueHighWaterMark 4900
+$ActionQueueLowWaterMark 3500
+$ActionQueueType FixedArray
+$ActionQueueWorkerThreads 1
+# action params:
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+#$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+*.* ?dynfile;outfmt
+# listener
+$InputTCPServerInputName 13514
+$InputTCPServerBindRuleset R13514
+$InputTCPServerRun 13514
+
+
+## RULESET with listener
+$Ruleset R13515
+# queue params:
+$ActionQueueTimeoutShutdown 60000
+$ActionQueueSize 5000
+$ActionQueueSaveOnShutdown on
+$ActionQueueHighWaterMark 4900
+$ActionQueueLowWaterMark 3500
+$ActionQueueType FixedArray
+$ActionQueueWorkerThreads 1
+# action params:
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+*.* ?dynfile;outfmt
+# listener
+$InputTCPServerInputName 13515
+$InputTCPServerBindRuleset R13515
+$InputTCPServerRun 13515
+
+
+
+## RULESET with listener
+$Ruleset R13516
+# queue params:
+$ActionQueueTimeoutShutdown 60000
+$ActionQueueSize 5000
+$ActionQueueSaveOnShutdown on
+$ActionQueueHighWaterMark 4900
+$ActionQueueLowWaterMark 3500
+$ActionQueueType FixedArray
+$ActionQueueWorkerThreads 1
+# action params:
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+*.* ?dynfile;outfmt
+# listener
+$InputTCPServerInputName 13516
+$InputTCPServerBindRuleset R13516
+$InputTCPServerRun 13516
+
+
diff --git a/tests/testsuites/dynfile_cachemiss.conf b/tests/testsuites/dynfile_cachemiss.conf
new file mode 100644
index 00000000..273ff176
--- /dev/null
+++ b/tests/testsuites/dynfile_cachemiss.conf
@@ -0,0 +1,14 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%\n"
+$template dynfile,"%msg:F,58:2%.log" # complete name is in message
+$OMFileFlushOnTXEnd on
+$DynaFileCacheSize 4
+$IncludeConfig rsyslog.action.1.include
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/dynfile_invalid2.conf b/tests/testsuites/dynfile_invalid2.conf
new file mode 100644
index 00000000..6d94c40d
--- /dev/null
+++ b/tests/testsuites/dynfile_invalid2.conf
@@ -0,0 +1,14 @@
+# simple async writing test
+# rgerhards, 2010-03-22
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%\n"
+$template dynfile,"%msg:F,58:2%.log" # complete name is in message
+$OMFileFlushOnTXEnd off
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/gzipwr_large.conf b/tests/testsuites/gzipwr_large.conf
new file mode 100644
index 00000000..54ad3bb3
--- /dev/null
+++ b/tests/testsuites/gzipwr_large.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/gzipwr_large_dynfile.conf b/tests/testsuites/gzipwr_large_dynfile.conf
new file mode 100644
index 00000000..3a1b255a
--- /dev/null
+++ b/tests/testsuites/gzipwr_large_dynfile.conf
@@ -0,0 +1,17 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.%msg:F,58:2%.log" # use multiple dynafiles
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/parse-nodate.conf b/tests/testsuites/parse-nodate.conf
new file mode 100644
index 00000000..570638d9
--- /dev/null
+++ b/tests/testsuites/parse-nodate.conf
@@ -0,0 +1,14 @@
+# test is a test config that does not include the timestamp. This is necessary to
+# test some illformed messages that do not contain a date. In that case, the system's
+# current timestamp is used, and that of course is a bit hard to compare against
+# a fixed template. So the solution in this case is to use a format that does
+# not contain any timestamp. Maybe not optimal, but it works ;)
+# rgerhards, 2010-03-19
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# use a special format that we can easily parse
+$template fmt,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/samples.parse-nodate b/tests/testsuites/samples.parse-nodate
new file mode 100644
index 00000000..7f16181c
--- /dev/null
+++ b/tests/testsuites/samples.parse-nodate
@@ -0,0 +1,6 @@
+<27>xapi: [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+27,daemon,err,localhost,xapi,xapi:, [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+# a message with just text (as permitted by rfc 3164)
+# it is questionable if the current sample result is really correct as of 3164!
+This is a message!
+13,user,notice,This,is,is, a message!
diff --git a/tests/testsuites/samples.snare_ccoff_udp b/tests/testsuites/samples.snare_ccoff_udp
new file mode 100644
index 00000000..010e44d5
--- /dev/null
+++ b/tests/testsuites/samples.snare_ccoff_udp
@@ -0,0 +1,14 @@
+# see comments in snare_ccoff_udp.conf
+# note that some of these samples look pretty wild, but they are
+# *real* cases (just mangled to anonymize them...)
+# Sample 1 - note the absence of PRI!
+windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
+13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1167	Fri, Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733
+# Sample 2
+# the samples below need to be disabled for the "workaround patch" for the message
+# parser to work. They need to be re-enabled once a final solution has been crafted
+#windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+#13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1166	Fri, Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732
+# Sample 3
+#windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+#13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1165	Fri, Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731
diff --git a/tests/testsuites/samples.snare_ccoff_udp2 b/tests/testsuites/samples.snare_ccoff_udp2
new file mode 100644
index 00000000..7837b820
--- /dev/null
+++ b/tests/testsuites/samples.snare_ccoff_udp2
@@ -0,0 +1,22 @@
+# see comments in snare_ccoff_udp.conf
+# note that some of these samples look pretty wild, but they are
+# *real* cases (just mangled to anonymize them...)
+#
+# NOTE
+# The current responses are probably not correct (handling of messages without PRI).
+# However, we keep them inside the test to be consistent. We should look at how
+# PRI-less messages are handled and once we have fixed that, the test cases may need
+# to be adapted. We do NOT try to preserve misbehaviour on such seriously malformed
+# messages.
+#
+# Sample 1 - note the absence of PRI!
+windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733 ', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
+# Sample 2
+# the samples below need to be disabled for the "workaround patch" for the message
+# parser to work. They need to be re-enabled once a final solution has been crafted
+#windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+#insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
+# Sample 3
+#windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+#insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')
diff --git a/tests/testsuites/snare.parse1 b/tests/testsuites/snare.parse1
new file mode 100644
index 00000000..550b0703
--- /dev/null
+++ b/tests/testsuites/snare.parse1
@@ -0,0 +1,83 @@
+# some parse test build around data in snare-format
+<141>Mar 10 09:30:20 zuse.xysystems.local MSWinEventLog\0111\011Security\011563\011Wed Mar 10 09:30:15 2010\011538\011Security\011XYWS011$\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: XYWS011$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x5984789C)     Logon Type: 3    \011552
+141,local1,notice,Mar 10 09:30:20,zuse.xysystems.local,MSWinEventLog#0111#011Security#011563#011Wed,MSWinEventLog#0111#011Security#011563#011Wed, Mar 10 09:30:15 2010#011538#011Security#011XYWS011$#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: XYWS011$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x5984789C)     Logon Type: 3    #011552
+#
+# NEXT MESSAGE
+#
+Mar 10 09:30:20 zuse.xysystems.local MSWinEventLog\0111\011Security\011564\011Wed Mar 10 09:30:19 2010\011540\011Security\011BACKUP1$\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: BACKUP1$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x59848DB4)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {f6f65903-1932-d229-4b75-64816121d569}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.31     Source Port: 0    \011553
+13,user,notice,Mar 10 09:30:20,zuse.xysystems.local,MSWinEventLog#0111#011Security#011564#011Wed,MSWinEventLog#0111#011Security#011564#011Wed, Mar 10 09:30:19 2010#011540#011Security#011BACKUP1$#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: BACKUP1$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x59848DB4)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {f6f65903-1932-d229-4b75-64816121d569}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.31     Source Port: 0    #011553
+#
+# NEXT MESSAGE
+# 
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011566\011Wed Mar 10 09:30:21 2010\011540\011Security\011aadminps\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3     Logon Process: Authz        Authentication Package: Kerberos     Workstation Name: ZUSE     Logon GUID: -     Caller User Name: ZUSE$     Caller Domain: XYSYSTEMS     Caller Logon ID: (0x0,0x3E7)     Caller Process ID: 1004     Transited Services: -     Source Network Address: -     Source Port: -    \011555
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011566#011Wed,MSWinEventLog#0111#011Security#011566#011Wed, Mar 10 09:30:21 2010#011540#011Security#011aadminps#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3     Logon Process: Authz        Authentication Package: Kerberos     Workstation Name: ZUSE     Logon GUID: -     Caller User Name: ZUSE$     Caller Domain: XYSYSTEMS     Caller Logon ID: (0x0,0x3E7)     Caller Process ID: 1004     Transited Services: -     Source Network Address: -     Source Port: -    #011555
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011567\011Wed Mar 10 09:30:21 2010\011538\011Security\011aadminps\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3    \011556
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011567#011Wed,MSWinEventLog#0111#011Security#011567#011Wed, Mar 10 09:30:21 2010#011538#011Security#011aadminps#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3    #011556
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011568\011Wed Mar 10 09:30:25 2010\011540\011Security\011ANONYMOUS LOGON\011Well Known Group\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name:      Domain:      Logon ID: (0x0,0x5984AB6F)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    \011557
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011568#011Wed,MSWinEventLog#0111#011Security#011568#011Wed, Mar 10 09:30:25 2010#011540#011Security#011ANONYMOUS LOGON#011Well Known Group#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name:      Domain:      Logon ID: (0x0,0x5984AB6F)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    #011557
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011569\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2318    \011558
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011569#011Wed,MSWinEventLog#0111#011Security#011569#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2318    #011558
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011570\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3    \011559
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011570#011Wed,MSWinEventLog#0111#011Security#011570#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3    #011559
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011571\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2319    \011560\
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011571#011Wed,MSWinEventLog#0111#011Security#011571#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2319    #011560
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011572\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3    \011561
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011572#011Wed,MSWinEventLog#0111#011Security#011572#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3    #011561
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011573\011Wed Mar 10 09:30:25 2010\011680\011Security\011ettore.trezzani\011User\011Success Audit\011ZUSE\011Account Logon\011\011Logon attempt by: MICROSOFT_AUTHENTICATION_PACKAGE_V1_0    Logon account: ettore.trezzani    Source Workstation: XYWS083    Error Code: 0x0    \011562
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011573#011Wed,MSWinEventLog#0111#011Security#011573#011Wed, Mar 10 09:30:25 2010#011680#011Security#011ettore.trezzani#011User#011Success Audit#011ZUSE#011Account Logon#011#011Logon attempt by: MICROSOFT_AUTHENTICATION_PACKAGE_V1_0    Logon account: ettore.trezzani    Source Workstation: XYWS083    Error Code: 0x0    #011562
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011574\011Wed Mar 10 09:30:25 2010\011540\011Security\011ettore.trezzani\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ettore.trezzani     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ADD5)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    \011563
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011574#011Wed,MSWinEventLog#0111#011Security#011574#011Wed, Mar 10 09:30:25 2010#011540#011Security#011ettore.trezzani#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ettore.trezzani     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ADD5)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    #011563
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011575\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2320    \011564
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011575#011Wed,MSWinEventLog#0111#011Security#011575#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2320    #011564
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011576\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3    \011565
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011576#011Wed,MSWinEventLog#0111#011Security#011576#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3    #011565
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011577\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2321    \011566
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011577#011Wed,MSWinEventLog#0111#011Security#011577#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2321    #011566
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011578\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3    \011567
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011578#011Wed,MSWinEventLog#0111#011Security#011578#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3    #011567
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011579\011Wed Mar 10 09:30:25 2010\011538\011Security\011ANONYMOUS LOGON\011Well Known Group\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ANONYMOUS LOGON     Domain: NT AUTHORITY     Logon ID: (0x0,0x5984AB6F)     Logon Type: 3    \011568
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011579#011Wed,MSWinEventLog#0111#011Security#011579#011Wed, Mar 10 09:30:25 2010#011538#011Security#011ANONYMOUS LOGON#011Well Known Group#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ANONYMOUS LOGON     Domain: NT AUTHORITY     Logon ID: (0x0,0x5984AB6F)     Logon Type: 3    #011568
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:30 zuse.xysystems.local MSWinEventLog\0111\011Security\011580\011Wed Mar 10 09:30:29 2010\011540\011Security\011XYWSBADGE$\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: XYWSBADGE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x59852D73)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {4bc3c075-5a77-4648-5822-bfdf88b4c211}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.18     Source Port: 0    \011569
+141,local1,notice,Mar 10 09:30:30,zuse.xysystems.local,MSWinEventLog#0111#011Security#011580#011Wed,MSWinEventLog#0111#011Security#011580#011Wed, Mar 10 09:30:29 2010#011540#011Security#011XYWSBADGE$#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: XYWSBADGE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x59852D73)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {4bc3c075-5a77-4648-5822-bfdf88b4c211}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.18     Source Port: 0    #011569
diff --git a/tests/testsuites/snare_ccoff_udp.conf b/tests/testsuites/snare_ccoff_udp.conf
new file mode 100644
index 00000000..6abbedf4
--- /dev/null
+++ b/tests/testsuites/snare_ccoff_udp.conf
@@ -0,0 +1,21 @@
+# This test some real-world snare cases. I don't like snare (no wonder
+# as I have written EventReporter, the ultimate Windows-to-Syslog tool),
+# but besides that snare generates severely malformed messages that
+# really stress-test the rsyslog engine. They deserve to be beaten by someone ;)
+# This test needs to be run over UDP only, as snare puts LF INTO some of the messages,
+# which makes it impossible to try these out via traditional syslog/tcp
+# added 2010-03-21 rgerhards
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# snare usses HT as field delimiter, so many users have turned off
+# control character escaping to make any sense at all from these messages...
+$EscapeControlCharactersOnReceive off
+
+# use a special format that we can easily check. We do NOT include a timestamp because
+# the malformed snare messages usually do not contain one (and we can not check against
+# the system time in our test cases).
+$template fmt,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/snare_ccoff_udp2.conf b/tests/testsuites/snare_ccoff_udp2.conf
new file mode 100644
index 00000000..9115c14f
--- /dev/null
+++ b/tests/testsuites/snare_ccoff_udp2.conf
@@ -0,0 +1,17 @@
+# Similar to snare_ccoff_udp_2, but with a different template. This template
+# has triggered problems in the past, thus a test is granted.
+# added 2010-03-21 rgerhards
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# snare usses HT as field delimiter, so many users have turned off
+# control character escaping to make any sense at all from these messages...
+$EscapeControlCharactersOnReceive off
+
+# we need to use a fixed timestamp, as otherwise we can not compare :(
+# This could be improved in later versions of the testing tools, but requires
+# modification to the rsyslog core...
+$template fmt,"insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg:::space-cc%', %syslogfacility%, '%HOSTNAME%',%syslogpriority%, '20100321185328', '20100321185328', %iut%, '%syslogtag:::space-cc%')\n",sql
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/wr_large.conf b/tests/testsuites/wr_large.conf
new file mode 100644
index 00000000..b64f132b
--- /dev/null
+++ b/tests/testsuites/wr_large.conf
@@ -0,0 +1,16 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileIOBufferSize 256k
+$IncludeConfig rsyslog.action.1.include
+local0.* ?dynfile;outfmt
diff --git a/tests/threadingmqaq.sh b/tests/threadingmqaq.sh
index 83f5fedc..29047bea 100755
--- a/tests/threadingmqaq.sh
+++ b/tests/threadingmqaq.sh
@@ -9,7 +9,7 @@
 echo \[threadingmqaq.sh\]: main/action queue concurrency
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup threadingmqaq.conf
-#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 2 100000
+#source $srcdir/diag.sh tcpflood -c2 -m100000
 #source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh injectmsg 0 100000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
diff --git a/tests/wr_large.sh b/tests/wr_large.sh
new file mode 100755
index 00000000..84f12989
--- /dev/null
+++ b/tests/wr_large.sh
@@ -0,0 +1,16 @@
+# This tests async writing large data records. We use up to 10K
+# record size.
+
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+cat rsyslog.action.1.include
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup wr_large.conf
+# send 4000 messages of 10.000bytes plus header max, randomized
+source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129
+sleep 1 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 3999 -E
+source $srcdir/diag.sh exit
diff --git a/tests/wr_large_async.sh b/tests/wr_large_async.sh
new file mode 100755
index 00000000..88a1acf8
--- /dev/null
+++ b/tests/wr_large_async.sh
@@ -0,0 +1,14 @@
+# This tests async writing large data records. We use up to 10K
+# record size.
+
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[wr_large_async.sh\]: test for file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+echo "\$OMFileAsyncWriting on" > rsyslog.action.1.include
+source $srcdir/wr_large.sh
diff --git a/tests/wr_large_sync.sh b/tests/wr_large_sync.sh
new file mode 100755
index 00000000..a1c4fd77
--- /dev/null
+++ b/tests/wr_large_sync.sh
@@ -0,0 +1,14 @@
+# This tests async writing large data records. We use up to 10K
+# record size.
+
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[wr_large_sync.sh\]: test for file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+echo "\$OMFileAsyncWriting off" > rsyslog.action.1.include
+source $srcdir/wr_large.sh