Merge branch 'v4-stable-solaris' into v4-devel

Conflicts:
	ChangeLog
	configure.ac
	doc/manual.html
	tools/omfile.c
	tools/syslogd.c

67 files changed, 1662 insertions, 84 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 6091896b..051a9403 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,6 @@
 if ENABLE_TESTBENCH
 TESTRUNS = rt_init rscript
-check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq
+check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen
 TESTS = $(TESTRUNS) cfg.sh \
 	validation-run.sh \
 	imtcp-multiport.sh \
@@ -10,6 +10,22 @@ TESTS = $(TESTRUNS) cfg.sh \
 	rsf_getenv.sh \
 	sndrcv.sh \
 	sndrcv_gzip.sh \
+	asynwr_simple.sh \
+	asynwr_timeout.sh \
+	asynwr_small.sh \
+	asynwr_tinybuf.sh \
+	wr_large_async.sh \
+	wr_large_sync.sh \
+	asynwr_deadlock.sh \
+	asynwr_deadlock2.sh \
+	asynwr_deadlock4.sh \
+	gzipwr_large.sh \
+	gzipwr_large_dynfile.sh \
+	dynfile_invld_async.sh \
+	dynfile_invld_sync.sh \
+	dynfile_invalid2.sh \
+	complex1.sh \
+	random.sh \
 	queue-persist.sh \
 	pipeaction.sh
 	execonlyonce.sh \
@@ -79,7 +95,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/1.parse1 \
 	   testsuites/2.parse1 \
 	   testsuites/3.parse1 \
+	   testsuites/snare.parse1 \
 	   testsuites/oversizeTag-1.parse1 \
+	   testsuites/weird.parse1 \
 	   testsuites/date1.parse1 \
 	   testsuites/date2.parse1 \
 	   testsuites/date3.parse1 \
@@ -90,6 +108,17 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/rfc5424-2.parse1 \
 	   testsuites/rfc5424-3.parse1 \
 	   testsuites/rfc5424-4.parse1 \
+	   testsuites/parse3.conf \
+	   testsuites/reallife.parse3 \
+	   testsuites/parse-nodate.conf \
+	   testsuites/parse_invld_regex.conf \
+	   testsuites/samples.parse_invld_regex \
+	   testsuites/parse-3164-buggyday.conf \
+	   testsuites/samples.parse-3164-buggyday \
+	   testsuites/snare_ccoff_udp.conf \
+	   testsuites/samples.snare_ccoff_udp \
+	   testsuites/snare_ccoff_udp2.conf \
+	   testsuites/samples.snare_ccoff_udp2 \
 	   testsuites/omod-if-array.conf \
 	   testsuites/1.omod-if-array \
 	   testsuites/1.field1 \
@@ -131,6 +160,38 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/sndrcv_gzip_rcvr.conf \
 	   pipeaction.sh \
 	   testsuites/pipeaction.conf \
+	   asynwr_simple.sh \
+	   testsuites/asynwr_simple.conf \
+	   asynwr_timeout.sh \
+	   testsuites/asynwr_timeout.conf \
+	   asynwr_small.sh \
+	   testsuites/asynwr_small.conf \
+	   asynwr_tinybuf.sh \
+	   testsuites/asynwr_tinybuf.conf \
+	   wr_large_async.sh \
+	   wr_large_sync.sh \
+	   wr_large.sh \
+	   testsuites/wr_large.conf \
+	   asynwr_deadlock.sh \
+	   testsuites/asynwr_deadlock.conf \
+	   asynwr_deadlock2.sh \
+	   testsuites/asynwr_deadlock2.conf \
+	   asynwr_deadlock4.sh \
+	   testsuites/asynwr_deadlock4.conf \
+	   gzipwr_large.sh \
+	   testsuites/gzipwr_large.conf \
+	   gzipwr_large_dynfile.sh \
+	   testsuites/gzipwr_large_dynfile.conf \
+	   complex1.sh \
+	   testsuites/complex1.conf \
+	   random.sh \
+	   testsuites/random.conf \
+	   dynfile_invld_async.sh \
+	   dynfile_invld_sync.sh \
+	   dynfile_cachemiss.sh \
+	   testsuites/dynfile_cachemiss.conf \
+	   dynfile_invalid2.sh \
+	   testsuites/dynfile_invalid2.conf \
 	   proprepltest.sh \
 	   testsuites/rfctag.conf \
 	   testsuites/master.rfctag \
@@ -143,11 +204,15 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   cfg.sh
 
 ourtail_SOURCES = ourtail.c
+msleep_SOURCES = msleep.c
 chkseq_SOURCES = chkseq.c
 
 tcpflood_SOURCES = tcpflood.c
 tcpflood_LDADD = $(SOL_LIBS)
 
+randomgen_SOURCES = randomgen.c
+randomgen_LDADD = $(SOL_LIBS)
+
 nettester_SOURCES = nettester.c getline.c
 nettester_LDADD = $(SOL_LIBS)
 
diff --git a/tests/asynwr_deadlock.sh b/tests/asynwr_deadlock.sh
new file mode 100755
index 00000000..dc08355e
--- /dev/null
+++ b/tests/asynwr_deadlock.sh
@@ -0,0 +1,23 @@
+# This is test case from practice, with the version we introduced it, it
+# caused a deadlock on shutdown. I have added it to the test suite to automatically
+# detect such things in the future.
+#
+# added 2010-03-17 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ================================================================================
+echo TEST: \[asynwr_deadlock.sh\]: a case known to have caused a deadlock in the past
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_deadlock.conf
+# just send one message
+source $srcdir/diag.sh tcpflood -m1
+# sleep is important! need to make sure the instance is inactive
+sleep 1
+# now try shutdown. The actual test is if the process does hang here!
+echo "processing must continue soon"
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 0
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_deadlock2.sh b/tests/asynwr_deadlock2.sh
new file mode 100755
index 00000000..1190f67a
--- /dev/null
+++ b/tests/asynwr_deadlock2.sh
@@ -0,0 +1,69 @@
+# This is test case from practice, with the version we introduced it, it
+# caused a deadlock during processing (when the a stream was purged from the
+# dynafile cache).
+# We added this as a standard test in the hopes that iw will help
+# detect such things in the future.
+#
+# The problem that originally caused this test to fail was:
+# We write to files asynchronously (with the async writer thread). There is
+# no signaling done when the file stream is closed. That can lead to the writer
+# process hanging in memory, that in turn leads to the main thread waiting on a
+# condition that never occurs (because it would need to be signalled by the
+# async writer). Even worse, in that case, the async writer was signalled invalid
+# in such a way that when it received a wakeup, it thought it shall not terminate,
+# but received a spurios wakeup due to timeout and no data to write. In that case
+# it (correctly) concluded that it would not need to timeout until a new buffer write
+# was done (in which case it would receive a wakeup). As such, it went into an eternal
+# wait. However, the invalid signaling did not take into account that it did not
+# signal the async writer to shut down. So the main thread went into a condition
+# wait - and thus we had a deadlock. That situation occured only under very specific
+# cirumstances. As far as the analysis goes, the following need to happen:
+# 1. buffers on that file are being flushed
+# 2. no new data arrives
+# 3. the inactivity timeout has not yet expired
+# 4. *then* (and only then) the stream is closed or destructed
+# In that, 1 to 4 are prequisites for the deadlock which will happen in 4. However,
+# for it to happen, we also need the right "timing". There is a race between the
+# main thread and the async writer thread. The deadlock will only happen under
+# the "right" circumstances, which basically means it will not happen always.
+# In order to create this case as reliable as possible, I have used
+# the "$OMFileFlushOnTXEnd on" directive
+# inside my test case. It makes sure that #1 above happens. The test uses a dynafile
+# cache size of 4, and the load generator generates data for 5 different dynafiles.
+# So over time, we will hit a spot where 4 dynafiles are open and the 5th file name
+# is generated. As such, one file needs to be discarded. Thanks to FlushOnTXEnd, we
+# now likely have #2 in place and thanks to the load pattern generated, we most
+# probably have #3 in place. During the dynafile cache displacement of the oldest
+# entry, #4 is generated. At this point, we have the deadlock we are testing for.
+# Note that this deadlock does not necessarily lead to a total lockup of rsyslogd.
+# Parts of it continue to operate. But in our test setup, this means data is
+# received and placed into the main queue. Once it's high water mark is hit, data 
+# is still being enqueued, but at a slow rate. So if one is patient enough, the load
+# generator will be able to finish. However, rsyslogd will never process the data
+# it received because it is locked in the deadlock caused by #4 above.
+# Note that "$OMFileFlushOnTXEnd on" is not causing this behaviour. We just use it
+# to (quite) reliably cause the failure condition. The failure described above
+# (in version 4.6.1) was also present when the setting was set to "off", but its
+# occurence was very much less probable - because the perquisites are then much
+# harder to hit. without it, the test may need to run for several hours before
+# we hit all failure conditions.
+#
+# added 2010-03-17 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo =================================================================================
+echo TEST: \[asynwr_deadlock2.sh\]: a case known to have caused a deadlock in the past
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_deadlock2.conf
+# send 20000 messages, each close to 2K (non-randomized!), so that we can fill
+# the buffers and hopefully run into the "deadlock".
+source $srcdir/diag.sh tcpflood -m20000 -d1800 -P129 -i1 -f5
+# the sleep below is needed to prevent too-early termination of the tcp listener
+sleep 1
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+cat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 1 20000 -E
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_deadlock4.sh b/tests/asynwr_deadlock4.sh
new file mode 100755
index 00000000..a3452f5b
--- /dev/null
+++ b/tests/asynwr_deadlock4.sh
@@ -0,0 +1,25 @@
+# This is test case from practice, with the version we introduced it, it
+# caused a deadlock during processing.
+# We added this as a standard test in the hopes that iw will help
+# detect such things in the future.
+#
+# This is a test that is constructed similar to asynwr_deadlock2.sh, but
+# can produce problems in a simpler way.
+#
+# added 2010-03-18 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo =================================================================================
+echo TEST: \[asynwr_deadlock4.sh\]: a case known to have caused a deadlock in the past
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_deadlock4.conf
+# send 20000 messages, each close to 2K (non-randomized!), so that we can fill
+# the buffers and hopefully run into the "deadlock".
+source $srcdir/diag.sh tcpflood -m20000 -d18 -P129 -i1 -f5
+# sleep is important! need to make sure the instance is inactive
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 1 20000 -E
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_simple.sh b/tests/asynwr_simple.sh
new file mode 100755
index 00000000..eb87443c
--- /dev/null
+++ b/tests/asynwr_simple.sh
@@ -0,0 +1,18 @@
+# This is test driver for testing asynchronous file output.
+#
+# added 2010-03-09 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_simple.sh\]: simple test for async file writing
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_simple.conf
+# send 35555 messages, make sure file size is not a multiple of
+# 10K, the buffer size!
+source $srcdir/diag.sh tcpflood -m35555
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 35554
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_small.sh b/tests/asynwr_small.sh
new file mode 100755
index 00000000..97818f6e
--- /dev/null
+++ b/tests/asynwr_small.sh
@@ -0,0 +1,26 @@
+# This tests async writing with only a small set of data. That
+# shall result in data staying in buffers until shutdown, what
+# then will trigger some somewhat complex logic in the stream
+# writer (open, write, close all during the stream close
+# opertion). It is vital that only few messages be sent.
+#
+# The main effort of this test is not (only) to see if we
+# receive the data, but rather to see if we get into an abort
+# condition.
+#
+# added 2010-03-09 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_small.sh\]: test for async file writing for few messages
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_small.conf
+# send 4000 messages
+source $srcdir/diag.sh tcpflood -m2
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 1
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_timeout.sh b/tests/asynwr_timeout.sh
new file mode 100755
index 00000000..b0bc5c28
--- /dev/null
+++ b/tests/asynwr_timeout.sh
@@ -0,0 +1,21 @@
+# This test writes to the output buffers, let's the output
+# write timeout (and write data) and then continue. The conf file
+# has a 2 second timeout, so we wait 4 seconds to be on the save side.
+#
+# added 2010-03-09 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_timeout.sh\]: test async file writing timeout writes
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_timeout.conf
+# send 35555 messages, make sure file size is not a multiple of
+# 10K, the buffer size!
+source $srcdir/diag.sh tcpflood -m 35555
+sleep 4 # wait for output writer to write and empty buffer
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 35554
+source $srcdir/diag.sh exit
diff --git a/tests/asynwr_tinybuf.sh b/tests/asynwr_tinybuf.sh
new file mode 100755
index 00000000..8eae1e26
--- /dev/null
+++ b/tests/asynwr_tinybuf.sh
@@ -0,0 +1,19 @@
+# This tests async writing with a very small output buffer (1 byte!),
+# so it stresses output buffer handling. This also means operations will
+# be somewhat slow, so we send only a small amounts of data.
+#
+# added 2010-03-09 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[asynwr_tinybuf.sh\]: test async file writing with 1-byte buffer
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup asynwr_tinybuf.conf
+# send 1000 messages, fairly enough to trigger problems
+source $srcdir/diag.sh tcpflood -m1000
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh seq-check 0 999
+source $srcdir/diag.sh exit
diff --git a/tests/chkseq.c b/tests/chkseq.c
index 6334d787..b22c8992 100644
--- a/tests/chkseq.c
+++ b/tests/chkseq.c
@@ -32,6 +32,7 @@
 #include "config.h"
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <getopt.h>
 
 int main(int argc, char *argv[])
@@ -40,14 +41,18 @@ int main(int argc, char *argv[])
 	int val;
 	int i;
 	int ret = 0;
+	int scanfOK;
 	int verbose = 0;
+	int bHaveExtraData = 0;
 	int dupsPermitted = 0;
 	int start = 0, end = 0;
 	int opt;
 	int nDups = 0;
+	int edLen;	/* length of extra data */
+	static char edBuf[500*1024]; /* buffer for extra data (pretty large to be on the save side...) */
 	char *file = NULL;
 
-	while((opt = getopt(argc, argv, "e:f:ds:v")) != EOF) {
+	while((opt = getopt(argc, argv, "e:f:ds:vE")) != EOF) {
 		switch((char)opt) {
 		case 'f':
 			file = optarg;
@@ -64,8 +69,11 @@ int main(int argc, char *argv[])
                 case 'v':
 			++verbose;
 			break;
-		default:printf("Invalid call of chkseq\n");
-			printf("Usage: chkseq file -sstart -eend -d\n");
+                case 'E':
+			bHaveExtraData = 1;
+			break;
+		default:printf("Invalid call of chkseq, optchar='%c'\n", opt);
+			printf("Usage: chkseq file -sstart -eend -d -E\n");
 			exit(1);
 		}
 	}
@@ -93,7 +101,17 @@ int main(int argc, char *argv[])
 	}
 
 	for(i = start ; i < end+1 ; ++i) {
-		if(fscanf(fp, "%d\n", &val) != 1) {
+		if(bHaveExtraData) {
+			scanfOK = fscanf(fp, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0;
+			if(edLen != (int) strlen(edBuf)) {
+				printf("extra data length specified %d, but actually is %ld in record %d\n",
+					edLen, (long) strlen(edBuf), i);
+				exit(1);
+			}
+		} else {
+			scanfOK = fscanf(fp, "%d\n", &val) == 1 ? 1 : 0;
+		}
+		if(!scanfOK) {
 			printf("scanf error in index i=%d\n", i);
 			exit(1);
 		}
@@ -116,5 +134,10 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
+	if(!feof(fp)) {
+		printf("end of processing, but NOT end of file!\n");
+		exit(1);
+	}
+
 	exit(ret);
 }
diff --git a/tests/complex1.sh b/tests/complex1.sh
new file mode 100755
index 00000000..7395bf21
--- /dev/null
+++ b/tests/complex1.sh
@@ -0,0 +1,22 @@
+# This is a rather complex test that runs a number of features together.
+#
+# added 2010-03-16 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ====================================================================================
+echo TEST: \[complex1.sh\]: complex test with gzip and multiple action queues
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup complex1.conf
+# send 40,000 messages of 400 bytes plus header max, via three dest ports
+source $srcdir/diag.sh tcpflood -m40000 -rd400 -P129 -f5 -n3 -c15 -i1
+sleep 2 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+ls rsyslog.out.*.log
+source $srcdir/diag.sh setzcat		   # find out which zcat to use
+$ZCAT rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 1 40000 -E
+source $srcdir/diag.sh exit
diff --git a/tests/diag.sh b/tests/diag.sh
index 3198d7d4..f65d197b 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -8,6 +8,7 @@
 #valgrind="valgrind --log-fd=1"
 #valgrind="valgrind --tool=drd --log-fd=1"
 #valgrind="valgrind --tool=helgrind --log-fd=1"
+#valgrind="valgrind --tool=exp-ptrcheck --log-fd=1"
 #set -o xtrace
 #export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
 #export RSYSLOG_DEBUGLOG="log"
@@ -15,16 +16,19 @@ case $1 in
    'init')	$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
 		cp $srcdir/testsuites/diag-common.conf diag-common.conf
 		cp $srcdir/testsuites/diag-common2.conf diag-common2.conf
-		rm -f rsyslogd.started work-*.conf
+		rm -f rsyslog.action.*.include
+		rm -f rsyslogd.started work-*.conf rsyslog.random.data
 		rm -f rsyslogd2.started work-*.conf
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
+		rm -f rsyslog.out.*.log
 		rm -rf test-spool
 		rm -f core.* vgcore.*
 		mkdir test-spool
 		;;
    'exit')	rm -f rsyslogd.started work-*.conf diag-common.conf
-   		rm -f rsyslogd2.started diag-common2.conf
+   		rm -f rsyslogd2.started diag-common2.conf rsyslog.action.*.include
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
+		rm -f rsyslog.out.*.log rsyslog.random.data
 		rm -rf test-spool
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
@@ -34,14 +38,14 @@ case $1 in
 		;;
    'wait-startup') # wait for rsyslogd startup ($2 is the instance)
 		while test ! -f rsyslogd$2.started; do
-			true
+			./msleep 100 # wait 100 milliseconds
 		done
 		echo "rsyslogd$2 started with pid " `cat rsyslog$2.pid`
 		;;
    'wait-shutdown')  # actually, we wait for rsyslog.pid to be deleted. $2 is the
    		# instance
 		while test -f rsyslog$2.pid; do
-			true
+			./msleep 100 # wait 100 milliseconds
 		done
 		;;
    'wait-queueempty') # wait for main message queue to be empty. $2 is the instance.
@@ -62,7 +66,7 @@ case $1 in
 		# note: we do not wait for the actual termination!
 		;;
    'tcpflood') # do a tcpflood run and check if it worked params are passed to tcpflood
-		./tcpflood $2 $3 $4 $5 $6 $7 $8
+		./tcpflood $2 $3 $4 $5 $6 $7 $8 $9
 		if [ "$?" -ne "0" ]; then
 		  echo "error during tcpflood! see rsyslog.out.log.save for what was written"
 		  cp rsyslog.out.log rsyslog.out.log.save
@@ -85,6 +89,7 @@ case $1 in
 		;;
    'seq-check') # do the usual sequence check to see if everything was properly received. $2 is the instance.
 		rm -f work
+		cp rsyslog.out.log work-presort
 		sort < rsyslog.out.log > work
 		# $4... are just to have the abilit to pass in more options...
 		./chkseq -fwork -v -s$2 -e$3 $4 $5 $6 $7
@@ -105,6 +110,18 @@ case $1 in
 		  exit 1
 		fi
 		;;
+   'gzip-seq-check') # do the usual sequence check, but for gzip files
+		rm -f work
+		ls -l rsyslog.out.log
+		gunzip < rsyslog.out.log | sort > work
+		ls -l work
+		# $4... are just to have the abilit to pass in more options...
+		./chkseq -fwork -v -s$2 -e$3 $4 $5 $6 $7
+		if [ "$?" -ne "0" ]; then
+		  echo "sequence error detected"
+		  exit 1
+		fi
+		;;
    'nettester') # perform nettester-based tests
    		# use -v for verbose output!
 		./nettester -t$2 -i$3
@@ -112,5 +129,12 @@ case $1 in
 		  exit 1
 		fi
 		;;
+   'setzcat')   # find out name of zcat tool
+		if [ `uname` == SunOS ]; then
+		   ZCAT=gzcat
+		else
+		   ZCAT=zcat
+		fi
+		;;
    *)		echo "invalid argument" $1
 esac
diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh
index 0282202d..c7745930 100755
--- a/tests/diskqueue-fsync.sh
+++ b/tests/diskqueue-fsync.sh
@@ -9,7 +9,7 @@ echo testing queue disk-only mode, fsync case
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup diskqueue-fsync.conf
 # 1000 messages should be enough - the disk fsync test is very slow!
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000
+source $srcdir/diag.sh tcpflood -m1000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 999
diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh
index 8233d569..7a50d82e 100755
--- a/tests/diskqueue.sh
+++ b/tests/diskqueue.sh
@@ -6,11 +6,14 @@
 # This file is part of the rsyslog project, released  under GPLv3
 # uncomment for debugging support:
 echo diskqueue.sh: testing queue disk-only mode
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup diskqueue.conf
 # 20000 messages should be enough - the disk test is slow enough ;)
 sleep 4
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000
+source $srcdir/diag.sh tcpflood -m20000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 19999
diff --git a/tests/dynfile_cachemiss.sh b/tests/dynfile_cachemiss.sh
new file mode 100755
index 00000000..6e2d9cca
--- /dev/null
+++ b/tests/dynfile_cachemiss.sh
@@ -0,0 +1,34 @@
+# This test checks if omfile segfaults when a file open() in dynacache mode fails.
+# The test is mimiced after a real-life scenario (which, of course, was much more
+# complex).
+#
+# added 2010-03-09 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[dynfile_cachemiss.sh\]: test open fail for dynafiles with `cat rsyslog.action.1.include`
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup dynfile_cachemiss.conf
+# we send handcrafted message. We have a dynafile cache of 4, and now send one message
+# each to fill up the cache.
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:0"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:1"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:2"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:3"
+# the next one has caused a segfault in practice
+# note that /proc/rsyslog.error.file must not be creatable
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:/proc/rsyslog.error.file:boom"
+# some more writes
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:4"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:5"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:6"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:7"
+# done message generation
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+cat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 7
+source $srcdir/diag.sh exit
diff --git a/tests/dynfile_invalid2.sh b/tests/dynfile_invalid2.sh
new file mode 100755
index 00000000..b6fb3945
--- /dev/null
+++ b/tests/dynfile_invalid2.sh
@@ -0,0 +1,34 @@
+# This test checks if omfile segfaults when a file open() in dynacache mode fails.
+# The test is mimiced after a real-life scenario (which, of course, was much more
+# complex).
+#
+# added 2010-03-22 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[dynfile_invalid2.sh\]: test open fail for dynafiles
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup dynfile_invalid2.conf
+# we send handcrafted message. We have a dynafile cache of 4, and now send one message
+# each to fill up the cache.
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:0"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:1"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:2"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:3"
+# the next one has caused a segfault in practice
+# note that /proc/rsyslog.error.file must not be creatable
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:/proc/rsyslog.error.file:boom"
+# some more writes
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.0.log:4"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.1.log:5"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.2.log:6"
+./tcpflood -m1 -M "<129>Mar 10 01:00:00 172.20.245.8 tag msg:rsyslog.out.3.log:7"
+# done message generation
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+cat rsyslog.out.*.log > rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 7
+source $srcdir/diag.sh exit
diff --git a/tests/dynfile_invld_async.sh b/tests/dynfile_invld_async.sh
new file mode 100755
index 00000000..3c9b2045
--- /dev/null
+++ b/tests/dynfile_invld_async.sh
@@ -0,0 +1,2 @@
+echo "\$OMFileAsyncWriting on" > rsyslog.action.1.include
+source $srcdir/dynfile_cachemiss.sh
diff --git a/tests/dynfile_invld_sync.sh b/tests/dynfile_invld_sync.sh
new file mode 100755
index 00000000..cc6b6451
--- /dev/null
+++ b/tests/dynfile_invld_sync.sh
@@ -0,0 +1,2 @@
+echo "\$OMFileAsyncWriting off" > rsyslog.action.1.include
+source $srcdir/dynfile_cachemiss.sh
diff --git a/tests/execonlyonce.sh b/tests/execonlyonce.sh
index a9b9d932..b7f60849 100755
--- a/tests/execonlyonce.sh
+++ b/tests/execonlyonce.sh
@@ -10,11 +10,11 @@ echo ===========================================================================
 echo \[execonlyonce.sh\]: test for the $ActionExecOnlyOnceEveryInterval directive
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup execonlyonce.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 1
+source $srcdir/diag.sh tcpflood -m10 -i1
 # now wait until the interval definitely expires
 sleep 4 # one more than the once inerval!
 # and inject another couple of messages
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10 100
+source $srcdir/diag.sh tcpflood -m10 -i100
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 
 # now we need your custom logic to see if the result is equal to the
diff --git a/tests/gzipwr_large.sh b/tests/gzipwr_large.sh
new file mode 100755
index 00000000..ffce06f6
--- /dev/null
+++ b/tests/gzipwr_large.sh
@@ -0,0 +1,20 @@
+# This tests writing large data records in gzip mode. We use up to 10K
+# record size.
+#
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[gzipwr_large.sh\]: test for gzip file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup gzipwr_large.conf
+# send 4000 messages of 10.000bytes plus header max, randomized
+source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129
+sleep 1 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh gzip-seq-check 0 3999 -E
+source $srcdir/diag.sh exit
diff --git a/tests/gzipwr_large_dynfile.sh b/tests/gzipwr_large_dynfile.sh
new file mode 100755
index 00000000..ebd4c558
--- /dev/null
+++ b/tests/gzipwr_large_dynfile.sh
@@ -0,0 +1,36 @@
+# This tests writing large data records in gzip mode. We also write it to
+# 5 different dynafiles, with a dynafile cache size set to 4. So this stresses
+# both the input side, as well as zip writing, async writing and the dynafile
+# cache logic.
+#
+# This test is a bit timing-dependent on the tcp reception side, so if it fails
+# one may look into the timing first. The main issue is that the testbench
+# currently has no good way to know if the tcp receiver is finished. This is NOT
+# a problem in rsyslogd, but only of the testbench.
+#
+# Note that we do not yet have sufficient support for dynafiles in diag.sh,
+# so we mangle some files here manually.
+#
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ====================================================================================
+echo TEST: \[gzipwr_large_dynfile.sh\]: test for gzip file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+export RSYSLOG_DEBUG="debug nostdout"
+export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup gzipwr_large_dynfile.conf
+# send 4000 messages of 10.000bytes plus header max, randomized
+source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129 -f5
+sleep 2 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+gunzip < rsyslog.out.0.log > rsyslog.out.log
+gunzip < rsyslog.out.1.log >> rsyslog.out.log
+gunzip < rsyslog.out.2.log >> rsyslog.out.log
+gunzip < rsyslog.out.3.log >> rsyslog.out.log
+gunzip < rsyslog.out.4.log >> rsyslog.out.log
+#cat rsyslog.out.* > rsyslog.out.log
+source $srcdir/diag.sh seq-check 0 3999 -E
+source $srcdir/diag.sh exit
diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh
index 47a33cb9..ad2b44f8 100755
--- a/tests/imtcp-multiport.sh
+++ b/tests/imtcp-multiport.sh
@@ -11,7 +11,7 @@ echo ===========================================================================
 echo \[imtcp-multiport.sh\]: testing imtcp multiple listeners
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup imtcp-multiport.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000
+source $srcdir/diag.sh tcpflood -p13514 -m10000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 9999
@@ -23,7 +23,7 @@ source $srcdir/diag.sh exit
 #
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup imtcp-multiport.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13515 1 10000
+source $srcdir/diag.sh tcpflood -p13515 -m10000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 9999
@@ -35,7 +35,7 @@ source $srcdir/diag.sh exit
 #
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup imtcp-multiport.conf
-source $srcdir/diag.sh tcpflood 127.0.0.1 13516 1 10000
+source $srcdir/diag.sh tcpflood -p13516 -m10000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
 source $srcdir/diag.sh seq-check 0 9999
diff --git a/tests/longrun.sh b/tests/longrun.sh
new file mode 100755
index 00000000..407e9fa9
--- /dev/null
+++ b/tests/longrun.sh
@@ -0,0 +1,30 @@
+# This is a test-aid script to try running some tests through many iterations.
+# It is not yet used in the automated testbench, but I keep this file so that
+# I can use it whenever there is need to. As such, it currently does not have
+# parameters but is expected to be edited as needed. -- rgerhards, 2010-03-10
+#
+# use: ./longrun.sh testname.sh
+#
+# where testname.sh is the test to be run
+# to change other params, you need to edit the settings here below:
+MAXRUNS=10
+DISPLAYALIVE=100
+LOGFILE=runlog
+
+echo "logfile is $LOGFILE"
+echo "executing test $1"
+
+date > $LOGFILE
+
+for (( i=0; $i < 10000; i++ ))
+  do
+    if [ $(( i % DISPLAYALIVE )) -eq 0 ]; then
+      echo "$i iterations done"
+    fi
+    $1 >> $LOGFILE
+    if [ "$?" -ne "0" ]; then
+      echo "Test failed in iteration $i, review $LOGFILE for details!"
+      exit 1
+    fi
+  done
+echo "No failure in $i iterations."
diff --git a/tests/manytcp.sh b/tests/manytcp.sh
index c55eb9c0..94a5c035 100755
--- a/tests/manytcp.sh
+++ b/tests/manytcp.sh
@@ -2,7 +2,10 @@
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup manytcp.conf
 # the config file specifies exactly 1100 connections
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1000 40000
+source $srcdir/diag.sh tcpflood -c1000 -m40000
+# the sleep below is needed to prevent too-early termination of the tcp listener
+sleep 1
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown	# we need to wait until rsyslogd is finished!
 source $srcdir/diag.sh seq-check 0 39999
 source $srcdir/diag.sh exit
diff --git a/tests/msleep.c b/tests/msleep.c
new file mode 100644
index 00000000..6fa57b79
--- /dev/null
+++ b/tests/msleep.c
@@ -0,0 +1,50 @@
+/* sleeps for the specified number of MILLIseconds.
+ * Primarily meant as a portable tool available everywhere for the
+ * testbench (sleep 0.1 does not work on all platforms).
+ *
+ * Part of the testbench for rsyslog.
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	struct timeval tvSelectTimeout;
+	long sleepTime;
+
+	if(argc != 2) {
+		fprintf(stderr, "usage: msleep <milliseconds>\n");
+		exit(1);
+	}
+
+	sleepTime = atoi(argv[1]);
+	tvSelectTimeout.tv_sec = sleepTime / 1000;
+	tvSelectTimeout.tv_usec = (sleepTime % 1000) * 1000; /* micro seconds */
+	if(select(0, NULL, NULL, NULL, &tvSelectTimeout) == -1) {
+		perror("select");
+		exit(1);
+	}
+
+	return 0;
+}
+
diff --git a/tests/nettester.c b/tests/nettester.c
index 209c2a6f..eff5929b 100644
--- a/tests/nettester.c
+++ b/tests/nettester.c
@@ -47,6 +47,7 @@
 #include <signal.h>
 #include <netinet/in.h>
 #include <getopt.h>
+#include <ctype.h>
 
 #define EXIT_FAILURE 1
 #define INVALID_SOCKET -1
@@ -61,6 +62,7 @@ static char *testSuite = NULL; /* name of current test suite */
 static int iPort = 12514; /* port which shall be used for sending data */
 static char* pszCustomConf = NULL;	/* custom config file, use -c conf to specify */
 static int verbose = 0;	/* verbose output? -v option */
+static char **ourEnvp;
 
 /* these two are quick hacks... */
 int iFailed = 0;
@@ -216,11 +218,6 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 	char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid",
 			   "-M../runtime/.libs:../.libs", NULL };
 	char confFile[1024];
-	char *newenviron[] = { NULL };
-	/* debug aide...
-	char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout",
-				"RSYSLOG_DEBUGLOG=log", NULL };
-	*/
 
 	sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir,
 		(pszCustomConf == NULL) ? configFile : pszCustomConf);
@@ -243,7 +240,7 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 		close(pipefd[1]);
 		close(pipefd[0]);
 		fclose(stdin);
-		execve("../tools/rsyslogd", newargv, newenviron);
+		execve("../tools/rsyslogd", newargv, ourEnvp);
 	} else {            
 		close(pipefd[1]);
 		*pid = cpid;
@@ -254,6 +251,62 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 }
 
 
+/* This function unescapes a string of testdata. That it, escape sequences
+ * are converted into their one-character equivalent. While doing so, it applies
+ * C-like semantics. This was made necessary for easy integration of control
+ * characters inside test cases.  -- rgerhards, 2009-03-11
+ * Currently supported:
+ * \\	single backslash
+ * \n, \t, \r as in C
+ * \nnn where nnn is a 1 to 3 character octal sequence 
+ * Note that when a problem occurs, the end result is undefined. After all, this
+ * is for a testsuite generatort, it must not be 100% bullet proof (so do not
+ * copy this code into something that must be!). Also note that we do in-memory
+ * unescaping and assume that the string gets shorter but NEVER longer!
+ */
+void unescapeTestdata(char *testdata)
+{
+	char *pDst;
+	char *pSrc;
+	int i;
+	int c;
+
+	pDst = pSrc = testdata;
+	while(*pSrc) {
+		if(*pSrc == '\\') {
+			switch(*++pSrc) {
+			case '\\':	*pDst++ = *pSrc++;
+					break;
+			case 'n':	*pDst++ = '\n';
+					++pSrc;
+					break;
+			case 'r':	*pDst++ = '\r';
+					++pSrc;
+					break;
+			case 't':	*pDst++ = '\t';
+					++pSrc;
+					break;
+			case '0':
+			case '1':
+			case '2':
+			case '3':	c = *pSrc++ - '0';
+					i = 1; /* we already processed one digit! */
+					while(i < 3 && isdigit(*pSrc)) {
+						c = c * 8 + *pSrc++ - '0';
+						++i;
+					}
+					*pDst++ = c;
+					break;
+			default:	break;
+			}
+		} else {
+			*pDst++ = *pSrc++;
+		}
+	}
+	*pDst = '\0';
+}
+
+
 /* Process a specific test case. File name is provided.
  * Needs to return 0 if all is OK, something else otherwise.
  */
@@ -291,6 +344,7 @@ processTestFile(int fd, char *pszFileName)
 
 		testdata[strlen(testdata)-1] = '\0'; /* remove \n */
 		/* now we have the test data to send (we could use function pointers here...) */
+		unescapeTestdata(testdata);
 		if(inputMode == inputUDP) {
 			if(udpSend(testdata, strlen(testdata)) != 0)
 				return(2);
@@ -314,10 +368,13 @@ processTestFile(int fd, char *pszFileName)
 				expected, buf);
 				ret = 1;
 		}
-
+		/* we need to free buffers, as we have potentially modified them! */
+		free(testdata);
+		testdata = NULL;
+		free(expected);
+		expected = NULL;
 	}
 
-	free(testdata);
 	free(expected);
 	fclose(fp);
 	return(ret);
@@ -390,7 +447,7 @@ void doAtExit(void)
  * of this file.
  * rgerhards, 2009-04-03
  */
-int main(int argc, char *argv[])
+int main(int argc, char *argv[], char *envp[])
 {
 	int fd;
 	int opt;
@@ -399,7 +456,8 @@ int main(int argc, char *argv[])
 	char buf[4096];
 	char testcases[4096];
 
-	while((opt = getopt(argc, argv, "c:i:p:t:v")) != EOF) {
+	ourEnvp = envp;
+	while((opt = getopt(argc, argv, "dc:i:p:t:v")) != EOF) {
 		switch((char)opt) {
                 case 'c':
 			pszCustomConf = optarg;
@@ -424,7 +482,7 @@ int main(int argc, char *argv[])
 			verbose = 1;
 			break;
 		default:printf("Invalid call of nettester, invalid option '%c'.\n", opt);
-			printf("Usage: nettester -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n");
+			printf("Usage: nettester -d -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n");
 			exit(1);
 		}
 	}
diff --git a/tests/ourtail.c b/tests/ourtail.c
index 4e8a6412..c31babb9 100644
--- a/tests/ourtail.c
+++ b/tests/ourtail.c
@@ -26,6 +26,7 @@
  *
  * A copy of the GPL can be found in the file "COPYING" in this distribution.
  */
+#include "config.h"
 #include <stdio.h>
 
 int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[])
diff --git a/tests/parsertest.sh b/tests/parsertest.sh
index ef33256e..fc68ab84 100755
--- a/tests/parsertest.sh
+++ b/tests/parsertest.sh
@@ -1,5 +1,33 @@
-echo TEST: parsertest.sh - various parser tests
+echo TEST: \[parsertest.sh\]: various parser tests
 source $srcdir/diag.sh init
 source $srcdir/diag.sh nettester parse1 udp
 source $srcdir/diag.sh nettester parse1 tcp
-source $srcdir/diag.sh init
+source $srcdir/diag.sh nettester parse3 udp
+source $srcdir/diag.sh nettester parse3 tcp
+source $srcdir/diag.sh nettester parse_invld_regex udp
+source $srcdir/diag.sh nettester parse_invld_regex tcp
+source $srcdir/diag.sh nettester parse-3164-buggyday udp
+source $srcdir/diag.sh nettester parse-3164-buggyday tcp
+source $srcdir/diag.sh nettester parse-nodate udp
+source $srcdir/diag.sh nettester parse-nodate tcp
+# the following samples can only be run over UDP as they are so
+# malformed they break traditional syslog/tcp framing...
+source $srcdir/diag.sh nettester snare_ccoff_udp udp
+source $srcdir/diag.sh nettester snare_ccoff_udp2 udp
+
+echo \[parsertest.sh]: redoing tests in IPv4-only mode
+source $srcdir/diag.sh nettester parse1 udp -4
+source $srcdir/diag.sh nettester parse1 tcp -4
+source $srcdir/diag.sh nettester parse3 udp -4
+source $srcdir/diag.sh nettester parse3 tcp -4
+source $srcdir/diag.sh nettester parse_invld_regex udp -4
+source $srcdir/diag.sh nettester parse_invld_regex tcp -4
+source $srcdir/diag.sh nettester parse-3164-buggyday udp -4
+source $srcdir/diag.sh nettester parse-3164-buggyday tcp -4
+source $srcdir/diag.sh nettester parse-nodate udp -4
+source $srcdir/diag.sh nettester parse-nodate tcp -4
+# UDP-only tests
+source $srcdir/diag.sh nettester snare_ccoff_udp udp -4
+source $srcdir/diag.sh nettester snare_ccoff_udp2 udp -4
+
+source $srcdir/diag.sh exit
diff --git a/tests/pipeaction.sh b/tests/pipeaction.sh
index 26a4c22a..c2201011 100755
--- a/tests/pipeaction.sh
+++ b/tests/pipeaction.sh
@@ -17,7 +17,7 @@ echo background cp process id is $CPPROCESS
 # now do the usual run
 source $srcdir/diag.sh startup pipeaction.conf
 # 20000 messages should be enough
-#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000
+#source $srcdir/diag.sh tcpflood -m20000
 source $srcdir/diag.sh injectmsg 0 20000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown
diff --git a/tests/random.sh b/tests/random.sh
new file mode 100755
index 00000000..79f704c7
--- /dev/null
+++ b/tests/random.sh
@@ -0,0 +1,20 @@
+# Test if rsyslog survives sending truely random data to it...
+#
+# added 2010-04-01 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[random.sh\]: testing random data
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup random.conf
+# generate random data
+./randomgen -f rsyslog.random.data -s 100000
+ls -l rsyslog.random.data
+source $srcdir/diag.sh tcpflood -B -I rsyslog.random.data -c5 -C10
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+# we do not check anything yet, the point is if rsyslog survived ;)
+# TODO: check for exit message, but we'll notice an abort anyhow, so not that important
+#source $srcdir/diag.sh exit
diff --git a/tests/randomgen.c b/tests/randomgen.c
new file mode 100644
index 00000000..9ba56954
--- /dev/null
+++ b/tests/randomgen.c
@@ -0,0 +1,130 @@
+/* generates random data for later use in test cases. Of course,
+ * we could generate random data during the testcase itself, but
+ * the core idea is that we record the random data so that we have
+ * a chance to reproduce a problem should it occur. IMHO this 
+ * provides the best compromise, by a) having randomness but
+ * b) knowing what was used during the test.
+ *
+ * Params
+ * -f	output file name (stdout if not given)
+ * -s	size of test data, plain number is size in k, 1MB default
+ * -u   uses /dev/urandom instead of libc random number generator
+ *      (when available). Note that this is usually much slower.
+ *
+ * Part of the testbench for rsyslog.
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+#include <netinet/in.h>
+
+#define EXIT_FAILURE 1
+
+static char *fileName = NULL;		/* name of output file */
+static int tryUseURandom = 0;		/* try to use /dev/urandom? */
+static long long fileSize = 1024*1024;	/* file size in K, 1MB default */
+
+
+/* generate the random file. This code really can be improved (e.g. read /dev/urandom
+ * when available)
+ */
+static inline void
+genFile()
+{
+	long i;
+	FILE *fp;
+	FILE *rfp = NULL;
+
+	if(fileName == NULL) {
+		fp = stdout;
+	} else {
+		if((fp = fopen(fileName, "w")) == NULL) {
+			perror(fileName);
+		}
+	}
+
+	/* try to use /dev/urandom, if available */
+	if(tryUseURandom)
+		rfp = fopen("/dev/urandom", "r");
+
+	if(rfp == NULL) {
+		/* fallback, use libc random number generator */
+		for(i = 0 ; i < fileSize ; ++i) {
+			if(fputc((char) rand(), fp) == EOF) {
+				perror(fileName);
+				exit(1);
+			}
+		}
+	} else {
+		/* use /dev/urandom */
+		printf("using /dev/urandom");
+		for(i = 0 ; i < fileSize ; ++i) {
+			if(fputc(fgetc(rfp), fp) == EOF) {
+				perror(fileName);
+				exit(1);
+			}
+		}
+	}
+
+	if(fileName != NULL)
+		fclose(fp);
+}
+
+
+/* Run the test.
+ * rgerhards, 2009-04-03
+ */
+int main(int argc, char *argv[])
+{
+	int ret = 0;
+	int opt;
+
+	srand(time(NULL));	/* seed is good enough for our needs */
+
+	while((opt = getopt(argc, argv, "f:s:u")) != -1) {
+		switch (opt) {
+		case 'f':	fileName = optarg;
+				break;
+		case 's':	fileSize = atol(optarg) * 1024;
+				break;
+		case 'u':	tryUseURandom = 1;
+				break;
+		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
+				exit (1);
+				break;
+		}
+	}
+
+	printf("generating random data file '%s' of %ldkb - may take a short while...\n",
+		fileName, (long) (fileSize / 1024));
+	genFile();
+
+	exit(ret);
+}
diff --git a/tests/rt-init.c b/tests/rt-init.c
index aaac7ed1..dbe94b4a 100644
--- a/tests/rt-init.c
+++ b/tests/rt-init.c
@@ -21,10 +21,10 @@
  *
  * A copy of the GPL can be found in the file "COPYING" in this distribution.
  */
-#include <stdio.h>
-
+#include "config.h"
 #include "rsyslog.h"
 #include "testbench.h"
+#include <stdio.h>	/* must be last, else we get a zlib compile error on some platforms */
 
 MODULE_TYPE_TESTBENCH
 
@@ -39,6 +39,6 @@ ENDExit
 
 BEGINTest
 CODESTARTTest
-finalize_it:
+/*finalize_it:*/
 	/* room for custom error reporter, leave blank if not needed */
 ENDTest
diff --git a/tests/sndrcv_drvr.sh b/tests/sndrcv_drvr.sh
index 63afd6b5..3d613069 100755
--- a/tests/sndrcv_drvr.sh
+++ b/tests/sndrcv_drvr.sh
@@ -34,7 +34,7 @@ source $srcdir/diag.sh wait-startup 2
 
 # now inject the messages into instance 2. It will connect to instance 1,
 # and that instance will record the data.
-source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 $2 1
+source $srcdir/diag.sh tcpflood -m$2 -i1
 sleep 2 # make sure all data is received in input buffers
 # shut down sender when everything is sent, receiver continues to run concurrently
 # may be needed by TLS (once we do it): sleep 60
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 0439e33e..e92d1308 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -2,15 +2,36 @@
  * messages over them. This is used for stress-testing.
  *
  * Params
- * argv[1]	target address
- * argv[2]	target port
- * argv[3]	number of connections
- * argv[4]	number of messages to send (connection is random)
- * argv[5]	initial message number (optional)
+ * -t	target address (default 127.0.0.1)
+ * -p	target port (default 13514)
+ * -n	number of target ports (targets are in range -p..(-p+-n-1)
+ * -c	number of connections (default 1)
+ * -m	number of messages to send (connection is random)
+ * -i	initial message number (optional)
+ * -P	PRI to be used for generated messages (default is 167).
+ *      Specify the plain number without leading zeros
+ * -d   amount of extra data to add to message. If present, the
+ *      number itself will be added as third field, and the data
+ *      bytes as forth. Add -r to randomize the amount of extra
+ *      data included in the range 1..(value of -d).
+ * -r	randomize amount of extra data added (-d must be > 0)
+ * -f	support for testing dynafiles. If given, include a dynafile ID
+ *      in the range 0..(f-1) as the SECOND field, shifting all field values
+ *      one field to the right. Zero (default) disables this functionality.
+ * -M   the message to be sent. Disables all message format options, as
+ *      only that exact same message is sent.
+ * -I   read specified input file, do NOT generate own test data. The test
+ *      completes when eof is reached.
+ * -B   The specified file (-I) is binary. No data processing is done by
+ *      tcpflood. If multiple connections are specified, data is read in
+ *      chunks and spread across the connections without taking any record
+ *      delemiters into account.
+ * -C	when input from a file is read, this file is transmitted -C times
+ *      (C like cycle, running out of meaningful option switches ;))
  *
  * Part of the testbench for rsyslog.
  *
- * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2009, 2010 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -41,18 +62,32 @@
 #include <unistd.h>
 #include <string.h>
 #include <netinet/in.h>
+#include <sys/resource.h>
 
 #define EXIT_FAILURE 1
 #define INVALID_SOCKET -1
 /* Name of input file, must match $IncludeConfig in test suite .conf files */
 #define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */
 
-static char *targetIP;
-static int targetPort;
+#define MAX_EXTRADATA_LEN 100*1024
+
+static char *targetIP = "127.0.0.1";
+static char *msgPRI = "167";
+static int targetPort = 13514;
+static int numTargetPorts = 1;
+static int dynFileIDs = 0;
+static int extraDataLen = 0; /* amount of extra data to add to message */
+static int bRandomizeExtraData = 0; /* randomize amount of extra data added */
 static int numMsgsToSend; /* number of messages to send */
-static int numConnections; /* number of connections to create */
+static int numConnections = 1; /* number of connections to create */
 static int *sockArray;  /* array of sockets to use */
 static int msgNum = 0;	/* initial message number to start with */
+static int bShowProgress = 1; /* show progress messages */
+static char *MsgToSend = NULL; /* if non-null, this is the actual message to send */
+static int bBinaryFile = 0;	/* is -I file binary */
+static char *dataFile = NULL;	/* name of data file, if NULL, generate own data */
+static int numFileIterations = 1;/* how often is file data to be sent? */
+FILE *dataFP = NULL;		/* file pointer for data file, if used */
 
 
 /* open a single tcp connection
@@ -61,16 +96,25 @@ int openConn(int *fd)
 {
 	int sock;
 	struct sockaddr_in addr;
+	int port;
 	int retries = 0;
+	int rnd;
 
 	if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) {
 		perror("socket()");
 		return(1);
 	}
 
+	/* randomize port if required */
+	if(numTargetPorts > 1) {
+		rnd = rand(); /* easier if we need value for debug messages ;) */
+		port = targetPort + (rnd % numTargetPorts);
+	} else {
+		port = targetPort;
+	}
 	memset((char *) &addr, 0, sizeof(addr));
 	addr.sin_family = AF_INET;
-	addr.sin_port = htons(targetPort);
+	addr.sin_port = htons(port);
 	if(inet_aton(targetIP, &addr.sin_addr)==0) {
 		fprintf(stderr, "inet_aton() failed\n");
 		return(1);
@@ -103,11 +147,13 @@ int openConnections(void)
 	char msgBuf[128];
 	size_t lenMsg;
 
-	write(1, "      open connections", sizeof("      open connections")-1);
+	if(bShowProgress)
+		write(1, "      open connections", sizeof("      open connections")-1);
 	sockArray = calloc(numConnections, sizeof(int));
 	for(i = 0 ; i < numConnections ; ++i) {
 		if(i % 10 == 0) {
-			printf("\r%5.5d", i);
+			if(bShowProgress)
+				printf("\r%5.5d", i);
 			//lenMsg = sprintf(msgBuf, "\r%5.5d", i);
 			//write(1, msgBuf, lenMsg);
 		}
@@ -133,14 +179,17 @@ int openConnections(void)
 void closeConnections(void)
 {
 	int i;
-	char msgBuf[128];
 	size_t lenMsg;
+	char msgBuf[128];
 
-	write(1, "      close connections", sizeof("      close connections")-1);
+	if(bShowProgress)
+		write(1, "      close connections", sizeof("      close connections")-1);
 	for(i = 0 ; i < numConnections ; ++i) {
 		if(i % 10 == 0) {
-			lenMsg = sprintf(msgBuf, "\r%5.5d", i);
-			write(1, msgBuf, lenMsg);
+			if(bShowProgress) {
+				lenMsg = sprintf(msgBuf, "\r%5.5d", i);
+				write(1, msgBuf, lenMsg);
+			}
 		}
 		close(sockArray[i]);
 	}
@@ -150,6 +199,62 @@ void closeConnections(void)
 }
 
 
+/* generate the message to be sent according to program command line parameters.
+ * this has been moved to its own function as we now have various different ways
+ * of constructing test messages. -- rgerhards, 2010-03-31
+ */
+static inline void
+genMsg(char *buf, size_t maxBuf, int *pLenBuf)
+{
+	int edLen; /* actual extra data length to use */
+	char extraData[MAX_EXTRADATA_LEN + 1];
+	char dynFileIDBuf[128] = "";
+	static int numMsgsGen = 0;
+	int done;
+
+	if(dataFP != NULL) {
+		/* get message from file */
+		do {
+			done = 1;
+			*pLenBuf = fread(buf, 1, 1024, dataFP);
+			if(feof(dataFP)) {
+				if(--numFileIterations > 0)  {
+					rewind(dataFP);
+					done = 0; /* need new iteration */
+				} else {
+					*pLenBuf = 0;
+					goto finalize_it;
+				}
+			}
+		} while(!done); /* Attention: do..while()! */
+	} else if(MsgToSend == NULL) {
+		if(dynFileIDs > 0) {
+			snprintf(dynFileIDBuf, maxBuf, "%d:", rand() % dynFileIDs);
+		}
+		if(extraDataLen == 0) {
+			*pLenBuf = snprintf(buf, maxBuf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:\n",
+					       msgPRI, dynFileIDBuf, msgNum);
+		} else {
+			if(bRandomizeExtraData)
+				edLen = ((long) rand() + extraDataLen) % extraDataLen + 1;
+			else
+				edLen = extraDataLen;
+			memset(extraData, 'X', edLen);
+			extraData[edLen] = '\0';
+			*pLenBuf = snprintf(buf, maxBuf, "<%s>Mar  1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s\n",
+					       msgPRI, dynFileIDBuf, msgNum, edLen, extraData);
+		}
+	} else {
+		/* use fixed message format from command line */
+		*pLenBuf = snprintf(buf, maxBuf, "%s\n", MsgToSend);
+	}
+
+	if(numMsgsGen++ >= numMsgsToSend)
+		*pLenBuf = 0; /* indicate end of run */
+
+finalize_it: ;
+}
+
 /* send messages to the tcp connections we keep open. We use
  * a very basic format that helps identify the message
  * (via msgnum:<number>: e.g. msgnum:00000001:). This format is suitable
@@ -160,39 +265,53 @@ void closeConnections(void)
  */
 int sendMessages(void)
 {
-	int i;
+	int i = 0;
 	int socknum;
 	int lenBuf;
 	int lenSend;
-	char buf[2048];
-
-	srand(time(NULL));	/* seed is good enough for our needs */
-
-	printf("Sending %d messages.\n", numMsgsToSend);
-	printf("\r%5.5d messages sent", 0);
-	for(i = 0 ; i < numMsgsToSend ; ++i) {
+	char *statusText;
+	char buf[MAX_EXTRADATA_LEN + 1024];
+
+	if(dataFile == NULL) {
+		printf("Sending %d messages.\n", numMsgsToSend);
+		statusText = "messages";
+	} else {
+		printf("Sending file '%s' %d times.\n", dataFile, numFileIterations);
+		statusText = "kb";
+	}
+	if(bShowProgress)
+		printf("\r%8.8d %s sent", 0, statusText);
+	while(1) { /* broken inside loop! */
 		if(i < numConnections)
 			socknum = i;
 		else if(i >= numMsgsToSend - numConnections)
 			socknum = i - (numMsgsToSend - numConnections);
-		else
-			socknum = rand() % numConnections;
-		lenBuf = sprintf(buf, "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum);
+		else {
+			int rnd = rand();
+			//socknum = rand() % numConnections;
+			socknum = rnd % numConnections;
+		}
+		genMsg(buf, sizeof(buf), &lenBuf); /* generate the message to send according to params */
+		if(lenBuf == 0)
+			break; /* end of processing! */
 		lenSend = send(sockArray[socknum], buf, lenBuf, 0);
 		if(lenSend != lenBuf) {
 			printf("\r%5.5d\n", i);
 			fflush(stdout);
 			perror("send test data");
-			printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum);
+			printf("send() failed at socket %d, index %d, msgNum %d\n",
+				sockArray[socknum], i, msgNum);
 			fflush(stderr);
 			return(1);
 		}
 		if(i % 100 == 0) {
-			printf("\r%5.5d", i);
+			if(bShowProgress)
+				printf("\r%8.8d", i);
 		}
 		++msgNum;
+		++i;
 	}
-	printf("\r%5.5d messages sent\n", i);
+	printf("\r%8.8d %s sent\n", i, statusText);
 
 	return 0;
 }
@@ -251,17 +370,18 @@ tcpSend(char *buf, int lenBuf)
 }
 
 
-/* Run the test suite. This must be called with exactly one parameter, the
- * name of the test suite. For details, see file header comment at the top
- * of this file.
+/* Run the test.
  * rgerhards, 2009-04-03
  */
 int main(int argc, char *argv[])
 {
 	int ret = 0;
+	int opt;
 	struct sigaction sigAct;
 	static char buf[1024];
 
+	srand(time(NULL));	/* seed is good enough for our needs */
+
 	/* on Solaris, we do not HAVE MSG_NOSIGNAL, so for this reason
 	 * we block SIGPIPE (not an issue for this program)
 	 */
@@ -271,19 +391,75 @@ int main(int argc, char *argv[])
 	sigaction(SIGPIPE, &sigAct, NULL);
 
 	setvbuf(stdout, buf, _IONBF, 48);
+	
+	if(!isatty(1))
+		bShowProgress = 0;
+
+	while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) {
+		switch (opt) {
+		case 't':	targetIP = optarg;
+				break;
+		case 'p':	targetPort = atoi(optarg);
+				break;
+		case 'n':	numTargetPorts = atoi(optarg);
+				break;
+		case 'c':	numConnections = atoi(optarg);
+				break;
+		case 'C':	numFileIterations = atoi(optarg);
+				break;
+		case 'm':	numMsgsToSend = atoi(optarg);
+				break;
+		case 'i':	msgNum = atoi(optarg);
+				break;
+		case 'P':	msgPRI = optarg;
+				break;
+		case 'd':	extraDataLen = atoi(optarg);
+				if(extraDataLen > MAX_EXTRADATA_LEN) {
+					fprintf(stderr, "-d max is %d!\n",
+						MAX_EXTRADATA_LEN);
+					exit(1);
+				}
+				break;
+		case 'r':	bRandomizeExtraData = 1;
+				break;
+		case 'f':	dynFileIDs = atoi(optarg);
+				break;
+		case 'M':	MsgToSend = optarg;
+				break;
+		case 'I':	dataFile = optarg;
+				/* in this mode, we do not know the num messages to send, so
+				 * we set a (high) number to keep the code happy.
+				 */
+				numMsgsToSend = 1000000;
+				break;
+		case 'B':	bBinaryFile = 1;
+				break;
+		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
+				exit (1);
+				break;
+		}
+	}
 
-	if(argc != 5 && argc != 6) {
-		printf("Invalid call of tcpflood\n");
-		printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n");
-		exit(1);
+	if(numConnections > 20) {
+		/* if we use many (whatever this means, 20 is randomly picked)
+		 * connections, we need to make sure we have a high enough
+		 * limit. -- rgerhards, 2010-03-25
+		 */
+		struct rlimit maxFiles;
+		maxFiles.rlim_cur = numConnections + 20;
+		maxFiles.rlim_max = numConnections + 20;
+		if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) {
+			perror("setrlimit to increase file handles failed");
+			exit(1);
+		}
 	}
 	
-	targetIP = argv[1];
-	targetPort = atoi(argv[2]);
-	numConnections = atoi(argv[3]);
-	numMsgsToSend = atoi(argv[4]);
-	if(argc == 6)
-		msgNum = atoi(argv[5]);
+	if(dataFile != NULL) {
+		if((dataFP = fopen(dataFile, "r")) == NULL) {
+			perror(dataFile);
+			exit(1);
+		}
+	}
 
 	if(openConnections() != 0) {
 		printf("error opening connections\n");
@@ -295,7 +471,6 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
-	//closeConnections();
 	printf("End of tcpflood Run\n");
 
 	exit(ret);
diff --git a/tests/testsuites/asynwr_deadlock.conf b/tests/testsuites/asynwr_deadlock.conf
new file mode 100644
index 00000000..dc4045b0
--- /dev/null
+++ b/tests/testsuites/asynwr_deadlock.conf
@@ -0,0 +1,14 @@
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+
+$OMFileFlushOnTXEnd on
+$OMFileFlushInterval 10
+$OMFileFlushIOBufferSize 10k
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/asynwr_deadlock2.conf b/tests/testsuites/asynwr_deadlock2.conf
new file mode 100644
index 00000000..07811613
--- /dev/null
+++ b/tests/testsuites/asynwr_deadlock2.conf
@@ -0,0 +1,16 @@
+# rgerhards, 2010-03-17
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.%msg:F,58:2%.log" # use multiple dynafiles
+
+$OMFileFlushOnTXEnd on
+$OMFileFlushInterval 10
+$OMFileIOBufferSize 10k
+$OMFileAsyncWriting on
+$DynaFileCacheSize 4
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_deadlock4.conf b/tests/testsuites/asynwr_deadlock4.conf
new file mode 100644
index 00000000..f4308ff1
--- /dev/null
+++ b/tests/testsuites/asynwr_deadlock4.conf
@@ -0,0 +1,16 @@
+# rgerhards, 2010-03-17
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.log" # use multiple dynafiles
+
+$OMFileFlushOnTXEnd on
+$OMFileFlushInterval 10
+$OMFileIOBufferSize 10k
+$OMFileAsyncWriting on
+$DynaFileCacheSize 4
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_simple.conf b/tests/testsuites/asynwr_simple.conf
new file mode 100644
index 00000000..44b03f2b
--- /dev/null
+++ b/tests/testsuites/asynwr_simple.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileFlushIOBufferSize 10k
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_small.conf b/tests/testsuites/asynwr_small.conf
new file mode 100644
index 00000000..f04ce962
--- /dev/null
+++ b/tests/testsuites/asynwr_small.conf
@@ -0,0 +1,14 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_timeout.conf b/tests/testsuites/asynwr_timeout.conf
new file mode 100644
index 00000000..44b03f2b
--- /dev/null
+++ b/tests/testsuites/asynwr_timeout.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileFlushIOBufferSize 10k
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/asynwr_tinybuf.conf b/tests/testsuites/asynwr_tinybuf.conf
new file mode 100644
index 00000000..01dec4d8
--- /dev/null
+++ b/tests/testsuites/asynwr_tinybuf.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileIOBufferSize 1
+$OMFileAsyncWriting on
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/complex1.conf b/tests/testsuites/complex1.conf
new file mode 100644
index 00000000..9e2441d4
--- /dev/null
+++ b/tests/testsuites/complex1.conf
@@ -0,0 +1,81 @@
+# complex test case with multiple actions in gzip mode
+# rgerhards, 2009-05-22
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.%inputname%.%msg:F,58:2%.log"
+
+## RULESET with listener
+$Ruleset R13514
+# queue params:
+$ActionQueueTimeoutShutdown 60000
+$ActionQueueSize 5000
+$ActionQueueSaveOnShutdown on
+$ActionQueueHighWaterMark 4900
+$ActionQueueLowWaterMark 3500
+$ActionQueueType FixedArray
+$ActionQueueWorkerThreads 1
+# action params:
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+#$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+*.* ?dynfile;outfmt
+# listener
+$InputTCPServerInputName 13514
+$InputTCPServerBindRuleset R13514
+$InputTCPServerRun 13514
+
+
+## RULESET with listener
+$Ruleset R13515
+# queue params:
+$ActionQueueTimeoutShutdown 60000
+$ActionQueueSize 5000
+$ActionQueueSaveOnShutdown on
+$ActionQueueHighWaterMark 4900
+$ActionQueueLowWaterMark 3500
+$ActionQueueType FixedArray
+$ActionQueueWorkerThreads 1
+# action params:
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+*.* ?dynfile;outfmt
+# listener
+$InputTCPServerInputName 13515
+$InputTCPServerBindRuleset R13515
+$InputTCPServerRun 13515
+
+
+
+## RULESET with listener
+$Ruleset R13516
+# queue params:
+$ActionQueueTimeoutShutdown 60000
+$ActionQueueSize 5000
+$ActionQueueSaveOnShutdown on
+$ActionQueueHighWaterMark 4900
+$ActionQueueLowWaterMark 3500
+$ActionQueueType FixedArray
+$ActionQueueWorkerThreads 1
+# action params:
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+*.* ?dynfile;outfmt
+# listener
+$InputTCPServerInputName 13516
+$InputTCPServerBindRuleset R13516
+$InputTCPServerRun 13516
+
+
diff --git a/tests/testsuites/diag-common2.conf b/tests/testsuites/diag-common2.conf
new file mode 100644
index 00000000..94f7e87f
--- /dev/null
+++ b/tests/testsuites/diag-common2.conf
@@ -0,0 +1,16 @@
+# This is a config include file. It sets up rsyslog so that the
+# diag system can successfully be used. Also, it generates a file
+# "rsyslogd.started" after rsyslogd is initialized. This config file
+# should be included in all tests that intend to use common code for
+# controlling the daemon.
+# NOTE: we assume that rsyslogd's current working directory is 
+# ./tests (or the distcheck equivalent), in particlular that this
+# config file resides in the testsuites subdirectory.
+# rgerhards, 2009-05-27
+$ModLoad ../plugins/imdiag/.libs/imdiag
+$IMDiagServerRun 13501
+
+$template startupfile,"rsyslogd2.started" # trick to use relative path names!
+:syslogtag, contains, "rsyslogd"  ?startupfile
+
+$ErrorMessagesToStderr off
diff --git a/tests/testsuites/dynfile_cachemiss.conf b/tests/testsuites/dynfile_cachemiss.conf
new file mode 100644
index 00000000..273ff176
--- /dev/null
+++ b/tests/testsuites/dynfile_cachemiss.conf
@@ -0,0 +1,14 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%\n"
+$template dynfile,"%msg:F,58:2%.log" # complete name is in message
+$OMFileFlushOnTXEnd on
+$DynaFileCacheSize 4
+$IncludeConfig rsyslog.action.1.include
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/dynfile_invalid2.conf b/tests/testsuites/dynfile_invalid2.conf
new file mode 100644
index 00000000..6d94c40d
--- /dev/null
+++ b/tests/testsuites/dynfile_invalid2.conf
@@ -0,0 +1,14 @@
+# simple async writing test
+# rgerhards, 2010-03-22
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%\n"
+$template dynfile,"%msg:F,58:2%.log" # complete name is in message
+$OMFileFlushOnTXEnd off
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/gzipwr_large.conf b/tests/testsuites/gzipwr_large.conf
new file mode 100644
index 00000000..54ad3bb3
--- /dev/null
+++ b/tests/testsuites/gzipwr_large.conf
@@ -0,0 +1,15 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/gzipwr_large_dynfile.conf b/tests/testsuites/gzipwr_large_dynfile.conf
new file mode 100644
index 00000000..3a1b255a
--- /dev/null
+++ b/tests/testsuites/gzipwr_large_dynfile.conf
@@ -0,0 +1,17 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:3%,%msg:F,58:4%,%msg:F,58:5%\n"
+$template dynfile,"rsyslog.out.%msg:F,58:2%.log" # use multiple dynafiles
+$OMFileFlushOnTXEnd off
+$OMFileZipLevel 6
+$OMFileIOBufferSize 256k
+$DynaFileCacheSize 4
+$omfileFlushInterval 1
+local0.* ?dynfile;outfmt
diff --git a/tests/testsuites/oversizeTag-1.parse1 b/tests/testsuites/oversizeTag-1.parse1
index 56510c63..d45ba1f2 100644
--- a/tests/testsuites/oversizeTag-1.parse1
+++ b/tests/testsuites/oversizeTag-1.parse1
@@ -1,3 +1,2 @@
-<38>Mar 27 19:06:53 source_server 0123456780123456780123456780123456789: MSG part
-38,auth,info,Mar 27 19:06:53,source_server,0123456780123456780123456780123456789,0123456780123456780123456780123456789:, MSG part
-# yet another real-life sample where we had some issues with
+<38>Mar 27 19:06:53 source_server 0123456789012345678901234567890123456789: MSG part
+38,auth,info,Mar 27 19:06:53,source_server,0123456789012345678901234567890123456789,0123456789012345678901234567890123456789:, MSG part
diff --git a/tests/testsuites/parse-3164-buggyday.conf b/tests/testsuites/parse-3164-buggyday.conf
new file mode 100644
index 00000000..937f423a
--- /dev/null
+++ b/tests/testsuites/parse-3164-buggyday.conf
@@ -0,0 +1,8 @@
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# use a special format that we can easily parse in expect
+$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp:::date-rfc3164-buggyday%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;expect
diff --git a/tests/testsuites/parse-nodate.conf b/tests/testsuites/parse-nodate.conf
new file mode 100644
index 00000000..570638d9
--- /dev/null
+++ b/tests/testsuites/parse-nodate.conf
@@ -0,0 +1,14 @@
+# test is a test config that does not include the timestamp. This is necessary to
+# test some illformed messages that do not contain a date. In that case, the system's
+# current timestamp is used, and that of course is a bit hard to compare against
+# a fixed template. So the solution in this case is to use a format that does
+# not contain any timestamp. Maybe not optimal, but it works ;)
+# rgerhards, 2010-03-19
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# use a special format that we can easily parse
+$template fmt,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/parse3.conf b/tests/testsuites/parse3.conf
new file mode 100644
index 00000000..8a3cb317
--- /dev/null
+++ b/tests/testsuites/parse3.conf
@@ -0,0 +1,10 @@
+# note: we need to strip off the TZ designator in the rfc3339 timestamp
+# as this test otherwise fails in different timezones!
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# use a special format that we can easily parse in expect
+$Template output,"%timereported:1:19:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9]+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
+*.* :omstdout:;output
diff --git a/tests/testsuites/parse_invld_regex.conf b/tests/testsuites/parse_invld_regex.conf
new file mode 100644
index 00000000..d18a2b3c
--- /dev/null
+++ b/tests/testsuites/parse_invld_regex.conf
@@ -0,0 +1,10 @@
+# note: we need to strip off the TZ designator in the rfc3339 timestamp
+# as this test otherwise fails in different timezones!
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# use a special format that we can easily parse in expect
+$Template output,"%timereported:1:19:date-rfc3339,csv%, %hostname:::csv%, %programname:::csv%, %syslogtag:R,ERE,0,BLANK:[0-9+--end:csv%, %syslogseverity:::csv%, %msg:::drop-last-lf,csv%\n" 
+*.* :omstdout:;output
diff --git a/tests/testsuites/random.conf b/tests/testsuites/random.conf
new file mode 100644
index 00000000..a7079df1
--- /dev/null
+++ b/tests/testsuites/random.conf
@@ -0,0 +1,13 @@
+# we write to /dev/null, as we have no chance to verify the output
+# in any case. What we really check is that rsyslogd does not
+# segfault or otherwise abort.
+# rgerhards, 2010-04-01
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%rawmsg%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+*.* /dev/null
diff --git a/tests/testsuites/reallife.parse3 b/tests/testsuites/reallife.parse3
new file mode 100644
index 00000000..dad3f56e
--- /dev/null
+++ b/tests/testsuites/reallife.parse3
@@ -0,0 +1,15 @@
+# New tests should be added to this file if there is no specific
+# reason for not doing that. Initially, we could only handle one test
+# case per file, but this restriction has been removed some time ago.
+# So it is less troublesome (and easier to overlook) to have all related
+# tests in a single file.
+# This file contains a lot of real-life samples (of course mangled so
+# that they can not be traced back to the original submitter). Note
+# that IP addr 192.0.2.1 is specifically set aside for testing and
+# documentation by IANA.
+# rgerhards, 2009-10-19
+<175>Oct 16 2009 23:47:31 hostname tag This is a message
+"2009-10-16T23:47:31", "hostname", "tag", "", "7", " This is a message"
+#
+<175>Oct 16 2009 23:47:31 hostname tag[1234] This is a message
+"2009-10-16T23:47:31", "hostname", "tag", "1234", "7", " This is a message"
diff --git a/tests/testsuites/samples.parse-3164-buggyday b/tests/testsuites/samples.parse-3164-buggyday
new file mode 100644
index 00000000..e21df980
--- /dev/null
+++ b/tests/testsuites/samples.parse-3164-buggyday
@@ -0,0 +1,6 @@
+# in 3164-buggyday mode, we need to have a leading zero in front of the day
+<38> Mar  7 19:06:53 example tag: testmessage (only date actually tested)
+38,auth,info,Mar 07 19:06:53,example,tag,tag:, testmessage (only date actually tested)
+# and now one with a complete date:
+<38> Mar 17 19:06:53 example tag: testmessage (only date actually tested)
+38,auth,info,Mar 17 19:06:53,example,tag,tag:, testmessage (only date actually tested)
diff --git a/tests/testsuites/samples.parse-nodate b/tests/testsuites/samples.parse-nodate
new file mode 100644
index 00000000..7f16181c
--- /dev/null
+++ b/tests/testsuites/samples.parse-nodate
@@ -0,0 +1,6 @@
+<27>xapi: [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+27,daemon,err,localhost,xapi,xapi:, [error|xen3|15|Guest liveness monitor D:bca30ab3f1c1|master_connection] Connection to master died. I will continue to retry indefinitely (supressing future logging of this message)
+# a message with just text (as permitted by rfc 3164)
+# it is questionable if the current sample result is really correct as of 3164!
+This is a message!
+13,user,notice,This,is,is, a message!
diff --git a/tests/testsuites/samples.parse_invld_regex b/tests/testsuites/samples.parse_invld_regex
new file mode 100644
index 00000000..0d0e4ce3
--- /dev/null
+++ b/tests/testsuites/samples.parse_invld_regex
@@ -0,0 +1,16 @@
+# New tests should be added to this file if there is no specific
+# reason for not doing that. Initially, we could only handle one test
+# case per file, but this restriction has been removed some time ago.
+# So it is less troublesome (and easier to overlook) to have all related
+# tests in a single file.
+# the actual message is not important. There is an error inside the conf
+# file, and all messages will trigger the same problem.
+# NOTE: it is correct that the "BAD REGULAR EXPRESSION" error message is
+# *NOT* run through the rest of the propety replace, in specific through
+# the CSV escaper. We do not do this because it could potentially lead
+# to an obfuscated error message, and thus making problems hard to find. As
+# this is a real error case, there is no problem in not obeying to the 
+# configured format.
+# rgerhards, 2010-02-08
+<175>Feb 08 2008 23:47:31 hostname tag This is a message
+"2008-02-08T23:47:31", "hostname", "tag", **NO MATCH** **BAD REGULAR EXPRESSION**, "7", " This is a message"
diff --git a/tests/testsuites/samples.snare_ccoff_udp b/tests/testsuites/samples.snare_ccoff_udp
new file mode 100644
index 00000000..010e44d5
--- /dev/null
+++ b/tests/testsuites/samples.snare_ccoff_udp
@@ -0,0 +1,14 @@
+# see comments in snare_ccoff_udp.conf
+# note that some of these samples look pretty wild, but they are
+# *real* cases (just mangled to anonymize them...)
+# Sample 1 - note the absence of PRI!
+windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
+13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1167	Fri, Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733
+# Sample 2
+# the samples below need to be disabled for the "workaround patch" for the message
+# parser to work. They need to be re-enabled once a final solution has been crafted
+#windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+#13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1166	Fri, Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732
+# Sample 3
+#windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+#13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1165	Fri, Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731
diff --git a/tests/testsuites/samples.snare_ccoff_udp2 b/tests/testsuites/samples.snare_ccoff_udp2
new file mode 100644
index 00000000..337cd97c
--- /dev/null
+++ b/tests/testsuites/samples.snare_ccoff_udp2
@@ -0,0 +1,26 @@
+# see comments in snare_ccoff_udp.conf
+# note that some of these samples look pretty wild, but they are
+# *real* cases (just mangled to anonymize them...)
+#
+# NOTE
+# The current responses are probably not correct (handling of messages without PRI).
+# However, we keep them inside the test to be consistent. We should look at how
+# PRI-less messages are handled and once we have fixed that, the test cases may need
+# to be adapted. We do NOT try to preserve misbehaviour on such seriously malformed
+# messages.
+#
+# this is a very simple test, though not snare-based
+test
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('', 1, 'test',5, '20100321185328', '20100321185328', 1, '')
+# and yet another one we have seen in practice
+UX=Abcd-efg-hij-klmno; XXXXX=1111111111, Z123=192.12.231.245:11111, S1234=123456789, XXXXXX=111111111
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' XXXXX=1111111111, Z123=192.12.231.245:11111, S1234=123456789, XXXXXX=111111111', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'UX=Abcd-efg-hij-klmno;')
+# Sample 1 - note the absence of PRI!
+windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
+# Sample 2
+windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
+# Sample 3
+windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')
diff --git a/tests/testsuites/snare.parse1 b/tests/testsuites/snare.parse1
new file mode 100644
index 00000000..550b0703
--- /dev/null
+++ b/tests/testsuites/snare.parse1
@@ -0,0 +1,83 @@
+# some parse test build around data in snare-format
+<141>Mar 10 09:30:20 zuse.xysystems.local MSWinEventLog\0111\011Security\011563\011Wed Mar 10 09:30:15 2010\011538\011Security\011XYWS011$\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: XYWS011$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x5984789C)     Logon Type: 3    \011552
+141,local1,notice,Mar 10 09:30:20,zuse.xysystems.local,MSWinEventLog#0111#011Security#011563#011Wed,MSWinEventLog#0111#011Security#011563#011Wed, Mar 10 09:30:15 2010#011538#011Security#011XYWS011$#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: XYWS011$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x5984789C)     Logon Type: 3    #011552
+#
+# NEXT MESSAGE
+#
+Mar 10 09:30:20 zuse.xysystems.local MSWinEventLog\0111\011Security\011564\011Wed Mar 10 09:30:19 2010\011540\011Security\011BACKUP1$\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: BACKUP1$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x59848DB4)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {f6f65903-1932-d229-4b75-64816121d569}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.31     Source Port: 0    \011553
+13,user,notice,Mar 10 09:30:20,zuse.xysystems.local,MSWinEventLog#0111#011Security#011564#011Wed,MSWinEventLog#0111#011Security#011564#011Wed, Mar 10 09:30:19 2010#011540#011Security#011BACKUP1$#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: BACKUP1$     Domain: XYZSYSTEMS     Logon ID: (0x0,0x59848DB4)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {f6f65903-1932-d229-4b75-64816121d569}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.31     Source Port: 0    #011553
+#
+# NEXT MESSAGE
+# 
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011566\011Wed Mar 10 09:30:21 2010\011540\011Security\011aadminps\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3     Logon Process: Authz        Authentication Package: Kerberos     Workstation Name: ZUSE     Logon GUID: -     Caller User Name: ZUSE$     Caller Domain: XYSYSTEMS     Caller Logon ID: (0x0,0x3E7)     Caller Process ID: 1004     Transited Services: -     Source Network Address: -     Source Port: -    \011555
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011566#011Wed,MSWinEventLog#0111#011Security#011566#011Wed, Mar 10 09:30:21 2010#011540#011Security#011aadminps#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3     Logon Process: Authz        Authentication Package: Kerberos     Workstation Name: ZUSE     Logon GUID: -     Caller User Name: ZUSE$     Caller Domain: XYSYSTEMS     Caller Logon ID: (0x0,0x3E7)     Caller Process ID: 1004     Transited Services: -     Source Network Address: -     Source Port: -    #011555
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011567\011Wed Mar 10 09:30:21 2010\011538\011Security\011aadminps\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3    \011556
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011567#011Wed,MSWinEventLog#0111#011Security#011567#011Wed, Mar 10 09:30:21 2010#011538#011Security#011aadminps#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: aadminps     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984973C)     Logon Type: 3    #011556
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011568\011Wed Mar 10 09:30:25 2010\011540\011Security\011ANONYMOUS LOGON\011Well Known Group\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name:      Domain:      Logon ID: (0x0,0x5984AB6F)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    \011557
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011568#011Wed,MSWinEventLog#0111#011Security#011568#011Wed, Mar 10 09:30:25 2010#011540#011Security#011ANONYMOUS LOGON#011Well Known Group#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name:      Domain:      Logon ID: (0x0,0x5984AB6F)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    #011557
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011569\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2318    \011558
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011569#011Wed,MSWinEventLog#0111#011Security#011569#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2318    #011558
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011570\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3    \011559
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011570#011Wed,MSWinEventLog#0111#011Security#011570#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ACA7)     Logon Type: 3    #011559
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011571\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2319    \011560\
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011571#011Wed,MSWinEventLog#0111#011Security#011571#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2319    #011560
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011572\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3    \011561
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011572#011Wed,MSWinEventLog#0111#011Security#011572#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AD7C)     Logon Type: 3    #011561
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011573\011Wed Mar 10 09:30:25 2010\011680\011Security\011ettore.trezzani\011User\011Success Audit\011ZUSE\011Account Logon\011\011Logon attempt by: MICROSOFT_AUTHENTICATION_PACKAGE_V1_0    Logon account: ettore.trezzani    Source Workstation: XYWS083    Error Code: 0x0    \011562
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011573#011Wed,MSWinEventLog#0111#011Security#011573#011Wed, Mar 10 09:30:25 2010#011680#011Security#011ettore.trezzani#011User#011Success Audit#011ZUSE#011Account Logon#011#011Logon attempt by: MICROSOFT_AUTHENTICATION_PACKAGE_V1_0    Logon account: ettore.trezzani    Source Workstation: XYWS083    Error Code: 0x0    #011562
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011574\011Wed Mar 10 09:30:25 2010\011540\011Security\011ettore.trezzani\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ettore.trezzani     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ADD5)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    \011563
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011574#011Wed,MSWinEventLog#0111#011Security#011574#011Wed, Mar 10 09:30:25 2010#011540#011Security#011ettore.trezzani#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ettore.trezzani     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984ADD5)     Logon Type: 3     Logon Process: NtLmSsp      Authentication Package: NTLM     Workstation Name: XYWS083     Logon GUID: -     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.91     Source Port: 0    #011563
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011575\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2320    \011564
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011575#011Wed,MSWinEventLog#0111#011Security#011575#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2320    #011564
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011576\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3    \011565
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011576#011Wed,MSWinEventLog#0111#011Security#011576#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AE49)     Logon Type: 3    #011565
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011577\011Wed Mar 10 09:30:25 2010\011540\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2321    \011566
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011577#011Wed,MSWinEventLog#0111#011Security#011577#011Wed, Mar 10 09:30:25 2010#011540#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {20014d9a-ce6c-6834-d1ed-607c08f0b6a7}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.0.15     Source Port: 2321    #011566
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011578\011Wed Mar 10 09:30:25 2010\011538\011Security\011SYSTEM\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3    \011567
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011578#011Wed,MSWinEventLog#0111#011Security#011578#011Wed, Mar 10 09:30:25 2010#011538#011Security#011SYSTEM#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ZUSE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x5984AF00)     Logon Type: 3    #011567
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:25 zuse.xysystems.local MSWinEventLog\0111\011Security\011579\011Wed Mar 10 09:30:25 2010\011538\011Security\011ANONYMOUS LOGON\011Well Known Group\011Success Audit\011ZUSE\011Logon/Logoff\011\011User Logoff:     User Name: ANONYMOUS LOGON     Domain: NT AUTHORITY     Logon ID: (0x0,0x5984AB6F)     Logon Type: 3    \011568
+141,local1,notice,Mar 10 09:30:25,zuse.xysystems.local,MSWinEventLog#0111#011Security#011579#011Wed,MSWinEventLog#0111#011Security#011579#011Wed, Mar 10 09:30:25 2010#011538#011Security#011ANONYMOUS LOGON#011Well Known Group#011Success Audit#011ZUSE#011Logon/Logoff#011#011User Logoff:     User Name: ANONYMOUS LOGON     Domain: NT AUTHORITY     Logon ID: (0x0,0x5984AB6F)     Logon Type: 3    #011568
+#
+# NEXT MESSAGE
+#
+<141>Mar 10 09:30:30 zuse.xysystems.local MSWinEventLog\0111\011Security\011580\011Wed Mar 10 09:30:29 2010\011540\011Security\011XYWSBADGE$\011User\011Success Audit\011ZUSE\011Logon/Logoff\011\011Successful Network Logon:     User Name: XYWSBADGE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x59852D73)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {4bc3c075-5a77-4648-5822-bfdf88b4c211}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.18     Source Port: 0    \011569
+141,local1,notice,Mar 10 09:30:30,zuse.xysystems.local,MSWinEventLog#0111#011Security#011580#011Wed,MSWinEventLog#0111#011Security#011580#011Wed, Mar 10 09:30:29 2010#011540#011Security#011XYWSBADGE$#011User#011Success Audit#011ZUSE#011Logon/Logoff#011#011Successful Network Logon:     User Name: XYWSBADGE$     Domain: XYSYSTEMS     Logon ID: (0x0,0x59852D73)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {4bc3c075-5a77-4648-5822-bfdf88b4c211}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 172.16.3.18     Source Port: 0    #011569
diff --git a/tests/testsuites/snare_ccoff_udp.conf b/tests/testsuites/snare_ccoff_udp.conf
new file mode 100644
index 00000000..6abbedf4
--- /dev/null
+++ b/tests/testsuites/snare_ccoff_udp.conf
@@ -0,0 +1,21 @@
+# This test some real-world snare cases. I don't like snare (no wonder
+# as I have written EventReporter, the ultimate Windows-to-Syslog tool),
+# but besides that snare generates severely malformed messages that
+# really stress-test the rsyslog engine. They deserve to be beaten by someone ;)
+# This test needs to be run over UDP only, as snare puts LF INTO some of the messages,
+# which makes it impossible to try these out via traditional syslog/tcp
+# added 2010-03-21 rgerhards
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# snare usses HT as field delimiter, so many users have turned off
+# control character escaping to make any sense at all from these messages...
+$EscapeControlCharactersOnReceive off
+
+# use a special format that we can easily check. We do NOT include a timestamp because
+# the malformed snare messages usually do not contain one (and we can not check against
+# the system time in our test cases).
+$template fmt,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/snare_ccoff_udp2.conf b/tests/testsuites/snare_ccoff_udp2.conf
new file mode 100644
index 00000000..9115c14f
--- /dev/null
+++ b/tests/testsuites/snare_ccoff_udp2.conf
@@ -0,0 +1,17 @@
+# Similar to snare_ccoff_udp_2, but with a different template. This template
+# has triggered problems in the past, thus a test is granted.
+# added 2010-03-21 rgerhards
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# snare usses HT as field delimiter, so many users have turned off
+# control character escaping to make any sense at all from these messages...
+$EscapeControlCharactersOnReceive off
+
+# we need to use a fixed timestamp, as otherwise we can not compare :(
+# This could be improved in later versions of the testing tools, but requires
+# modification to the rsyslog core...
+$template fmt,"insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg:::space-cc%', %syslogfacility%, '%HOSTNAME%',%syslogpriority%, '20100321185328', '20100321185328', %iut%, '%syslogtag:::space-cc%')\n",sql
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/weird.parse1 b/tests/testsuites/weird.parse1
new file mode 100644
index 00000000..e8b90c74
--- /dev/null
+++ b/tests/testsuites/weird.parse1
@@ -0,0 +1,37 @@
+# some really weird samples, some of them seen in practice,
+# some other deliberately generated. The main point is that they
+# should not cause an abort...
+<14>Aug 30 23:00:05 X4711 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+14,user,info,Aug 30 23:00:05,X4711,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA,
+# important: the following line has a SP at the end of the line!
+<14>Aug 30 23:00:05 X4711 
+14,user,info,Aug 30 23:00:05,X4711,,,
+# and this one NOT
+<14>Aug 30 23:00:05 X4711
+14,user,info,Aug 30 23:00:05,X4711,,,
+# there is a SP at the end of the line
+<14>Aug 30 23:00:05 
+14,user,info,Aug 30 23:00:05,localhost,,,
+# and here is no SP at the end of the line
+<14>Aug 30 23:00:05
+14,user,info,Aug 30 23:00:05,localhost,,,
+# unfortunately, I can not test missing dates with this test suite, because
+# we would have the current date in the response, which we can not check against
+#
+# and now the same tests with RFC3339 data - this can make a difference
+# as a different date parser is involved.
+# 
+<14>2010-08-30T23:00:05Z X4711 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+14,user,info,Aug 30 23:00:05,X4711,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA,
+# important: the following line has a SP at the end of the line!
+<14>2010-08-30T23:00:05Z X4711 
+14,user,info,Aug 30 23:00:05,X4711,,,
+# and this one NOT
+<14>2010-08-30T23:00:05Z X4711
+14,user,info,Aug 30 23:00:05,X4711,,,
+# there is a SP at the end of the line
+<14>2010-08-30T23:00:05Z 
+14,user,info,Aug 30 23:00:05,localhost,,,
+# and here is no SP at the end of the line
+<14>2010-08-30T23:00:05Z
+14,user,info,Aug 30 23:00:05,localhost,,,
diff --git a/tests/testsuites/wr_large.conf b/tests/testsuites/wr_large.conf
new file mode 100644
index 00000000..b64f132b
--- /dev/null
+++ b/tests/testsuites/wr_large.conf
@@ -0,0 +1,16 @@
+# simple async writing test
+# rgerhards, 2010-03-09
+$MaxMessageSize 10k
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%,%msg:F,58:3%,%msg:F,58:4%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+$OMFileFlushOnTXEnd off
+$OMFileFlushInterval 2
+$OMFileIOBufferSize 256k
+$IncludeConfig rsyslog.action.1.include
+local0.* ?dynfile;outfmt
diff --git a/tests/threadingmqaq.sh b/tests/threadingmqaq.sh
index 9f2ea793..b7764821 100755
--- a/tests/threadingmqaq.sh
+++ b/tests/threadingmqaq.sh
@@ -9,7 +9,7 @@
 echo TEST: threadingmqaq.sh - main/action queue concurrency
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup threadingmqaq.conf
-#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 2 100000
+#source $srcdir/diag.sh tcpflood -c2 -m100000
 #source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh injectmsg 0 100000
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
diff --git a/tests/wr_large.sh b/tests/wr_large.sh
new file mode 100755
index 00000000..84f12989
--- /dev/null
+++ b/tests/wr_large.sh
@@ -0,0 +1,16 @@
+# This tests async writing large data records. We use up to 10K
+# record size.
+
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+cat rsyslog.action.1.include
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup wr_large.conf
+# send 4000 messages of 10.000bytes plus header max, randomized
+source $srcdir/diag.sh tcpflood -m4000 -r -d10000 -P129
+sleep 1 # due to large messages, we need this time for the tcp receiver to settle...
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
+source $srcdir/diag.sh seq-check 0 3999 -E
+source $srcdir/diag.sh exit
diff --git a/tests/wr_large_async.sh b/tests/wr_large_async.sh
new file mode 100755
index 00000000..88a1acf8
--- /dev/null
+++ b/tests/wr_large_async.sh
@@ -0,0 +1,14 @@
+# This tests async writing large data records. We use up to 10K
+# record size.
+
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[wr_large_async.sh\]: test for file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+echo "\$OMFileAsyncWriting on" > rsyslog.action.1.include
+source $srcdir/wr_large.sh
diff --git a/tests/wr_large_sync.sh b/tests/wr_large_sync.sh
new file mode 100755
index 00000000..a1c4fd77
--- /dev/null
+++ b/tests/wr_large_sync.sh
@@ -0,0 +1,14 @@
+# This tests async writing large data records. We use up to 10K
+# record size.
+
+# added 2010-03-10 by Rgerhards
+#
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo TEST: \[wr_large_sync.sh\]: test for file writing for large message sets
+source $srcdir/diag.sh init
+# uncomment for debugging support:
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+echo "\$OMFileAsyncWriting off" > rsyslog.action.1.include
+source $srcdir/wr_large.sh