added some more tests for severely ill-formed snare messages

also improved nettester testbench tool a bit

7 files changed, 90 insertions, 6 deletions

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 98b97d44..d2a90869 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -110,6 +110,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/samples.parse_invld_regex \
 	   testsuites/parse-3164-buggyday.conf \
 	   testsuites/samples.parse-3164-buggyday \
+	   testsuites/snare_ccoff_udp.conf \
+	   testsuites/samples.snare_ccoff_udp \
+	   testsuites/snare_ccoff_udp2.conf \
+	   testsuites/samples.snare_ccoff_udp2 \
 	   testsuites/omod-if-array.conf \
 	   testsuites/1.omod-if-array \
 	   testsuites/1.field1 \
diff --git a/tests/nettester.c b/tests/nettester.c
index 22b5f16f..e1ecbcb5 100644
--- a/tests/nettester.c
+++ b/tests/nettester.c
@@ -62,6 +62,7 @@ static char *testSuite = NULL; /* name of current test suite */
 static int iPort = 12514; /* port which shall be used for sending data */
 static char* pszCustomConf = NULL;	/* custom config file, use -c conf to specify */
 static int verbose = 0;	/* verbose output? -v option */
+static int useDebugEnv = 0; /* activate debugging environment (for rsyslog debug log)? */
 
 /* these two are quick hacks... */
 int iFailed = 0;
@@ -218,10 +219,8 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 			   "-M../runtime/.libs:../.libs", NULL };
 	char confFile[1024];
 	char *newenviron[] = { NULL };
-	/* debug aide...
-	char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout",
+	char *newenvironDeb[] = { "RSYSLOG_DEBUG=debug nostdout",
 				"RSYSLOG_DEBUGLOG=log", NULL };
-	*/
 
 	sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir,
 		(pszCustomConf == NULL) ? configFile : pszCustomConf);
@@ -244,7 +243,7 @@ int openPipe(char *configFile, pid_t *pid, int *pfd)
 		close(pipefd[1]);
 		close(pipefd[0]);
 		fclose(stdin);
-		execve("../tools/rsyslogd", newargv, newenviron);
+		execve("../tools/rsyslogd", newargv, (useDebugEnv) ? newenvironDeb : newenviron);
 	} else {            
 		close(pipefd[1]);
 		*pid = cpid;
@@ -460,11 +459,14 @@ int main(int argc, char *argv[])
 	char buf[4096];
 	char testcases[4096];
 
-	while((opt = getopt(argc, argv, "c:i:p:t:v")) != EOF) {
+	while((opt = getopt(argc, argv, "dc:i:p:t:v")) != EOF) {
 		switch((char)opt) {
                 case 'c':
 			pszCustomConf = optarg;
 			break;
+                case 'd': 
+			useDebugEnv = 1;
+			break;
                 case 'i':
 			if(!strcmp(optarg, "udp"))
 				inputMode = inputUDP;
@@ -485,7 +487,7 @@ int main(int argc, char *argv[])
 			verbose = 1;
 			break;
 		default:printf("Invalid call of nettester, invalid option '%c'.\n", opt);
-			printf("Usage: nettester -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n");
+			printf("Usage: nettester -d -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n");
 			exit(1);
 		}
 	}
diff --git a/tests/parsertest.sh b/tests/parsertest.sh
index 10a7f450..fc68ab84 100755
--- a/tests/parsertest.sh
+++ b/tests/parsertest.sh
@@ -10,6 +10,10 @@ source $srcdir/diag.sh nettester parse-3164-buggyday udp
 source $srcdir/diag.sh nettester parse-3164-buggyday tcp
 source $srcdir/diag.sh nettester parse-nodate udp
 source $srcdir/diag.sh nettester parse-nodate tcp
+# the following samples can only be run over UDP as they are so
+# malformed they break traditional syslog/tcp framing...
+source $srcdir/diag.sh nettester snare_ccoff_udp udp
+source $srcdir/diag.sh nettester snare_ccoff_udp2 udp
 
 echo \[parsertest.sh]: redoing tests in IPv4-only mode
 source $srcdir/diag.sh nettester parse1 udp -4
@@ -22,4 +26,8 @@ source $srcdir/diag.sh nettester parse-3164-buggyday udp -4
 source $srcdir/diag.sh nettester parse-3164-buggyday tcp -4
 source $srcdir/diag.sh nettester parse-nodate udp -4
 source $srcdir/diag.sh nettester parse-nodate tcp -4
+# UDP-only tests
+source $srcdir/diag.sh nettester snare_ccoff_udp udp -4
+source $srcdir/diag.sh nettester snare_ccoff_udp2 udp -4
+
 source $srcdir/diag.sh exit
diff --git a/tests/testsuites/samples.snare_ccoff_udp b/tests/testsuites/samples.snare_ccoff_udp
new file mode 100644
index 00000000..334267f4
--- /dev/null
+++ b/tests/testsuites/samples.snare_ccoff_udp
@@ -0,0 +1,12 @@
+# see comments in snare_ccoff_udp.conf
+# note that some of these samples look pretty wild, but they are
+# *real* cases (just mangled to anonymize them...)
+# Sample 1 - note the absence of PRI!
+windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
+13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1167	Fri, Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733
+# Sample 2
+windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1166	Fri, Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732
+# Sample 3
+windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+13,user,notice,localhost,windowsserver,windowsserver	MSWinEventLog	1	Security	1165	Fri, Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731
diff --git a/tests/testsuites/samples.snare_ccoff_udp2 b/tests/testsuites/samples.snare_ccoff_udp2
new file mode 100644
index 00000000..8c14f0bb
--- /dev/null
+++ b/tests/testsuites/samples.snare_ccoff_udp2
@@ -0,0 +1,20 @@
+# see comments in snare_ccoff_udp.conf
+# note that some of these samples look pretty wild, but they are
+# *real* cases (just mangled to anonymize them...)
+#
+# NOTE
+# The current responses are probably not correct (handling of messages without PRI).
+# However, we keep them inside the test to be consistent. We should look at how
+# PRI-less messages are handled and once we have fixed that, the test cases may need
+# to be adapted. We do NOT try to preserve misbehaviour on such seriously malformed
+# messages.
+#
+# Sample 1 - note the absence of PRI!
+windowsserver	MSWinEventLog	1	Security	1167	Fri Mar 19 15:33:30 2010	540	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306    	733\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 540 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Successful Network Logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Logon Type: 3     Logon Process: Kerberos     Authentication Package: Kerberos     Workstation Name:      Logon GUID: {79b6eb79-7bcc-8a2e-7dad-953c51dc00fd}     Caller User Name: -     Caller Domain: -     Caller Logon ID: -     Caller Process ID: -     Transited Services: -     Source Network Address: 10.11.11.3     Source Port: 3306     733', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1167 Fri')
+# Sample 2
+windowsserver	MSWinEventLog	1	Security	1166	Fri Mar 19 15:33:30 2010	576	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege  	732\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 576 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  Special privileges assigned to new logon:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF88396)     Privileges: SeSecurityPrivilege   SeBackupPrivilege   SeRestorePrivilege   SeTakeOwnershipPrivilege   SeDebugPrivilege   SeSystemEnvironmentPrivilege   SeLoadDriverPrivilege   SeImpersonatePrivilege   SeEnableDelegationPrivilege   732', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1166 Fri')
+# Sample 3
+windowsserver	MSWinEventLog	1	Security	1165	Fri Mar 19 15:33:30 2010	538	Security	SYSTEM	User	Success Audit	WINDOWSSERVER	Logon/Logoff		User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3    	731\n
+insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values (' Mar 19 15:33:30 2010 538 Security SYSTEM User Success Audit WINDOWSSERVER Logon/Logoff  User Logoff:     User Name: WINDOWSSERVER$     Domain: DOMX     Logon ID: (0x0,0xF8830B)     Logon Type: 3     731', 1, 'localhost',5, '20100321185328', '20100321185328', 1, 'windowsserver MSWinEventLog 1 Security 1165 Fri')
diff --git a/tests/testsuites/snare_ccoff_udp.conf b/tests/testsuites/snare_ccoff_udp.conf
new file mode 100644
index 00000000..6abbedf4
--- /dev/null
+++ b/tests/testsuites/snare_ccoff_udp.conf
@@ -0,0 +1,21 @@
+# This test some real-world snare cases. I don't like snare (no wonder
+# as I have written EventReporter, the ultimate Windows-to-Syslog tool),
+# but besides that snare generates severely malformed messages that
+# really stress-test the rsyslog engine. They deserve to be beaten by someone ;)
+# This test needs to be run over UDP only, as snare puts LF INTO some of the messages,
+# which makes it impossible to try these out via traditional syslog/tcp
+# added 2010-03-21 rgerhards
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# snare usses HT as field delimiter, so many users have turned off
+# control character escaping to make any sense at all from these messages...
+$EscapeControlCharactersOnReceive off
+
+# use a special format that we can easily check. We do NOT include a timestamp because
+# the malformed snare messages usually do not contain one (and we can not check against
+# the system time in our test cases).
+$template fmt,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;fmt
diff --git a/tests/testsuites/snare_ccoff_udp2.conf b/tests/testsuites/snare_ccoff_udp2.conf
new file mode 100644
index 00000000..9115c14f
--- /dev/null
+++ b/tests/testsuites/snare_ccoff_udp2.conf
@@ -0,0 +1,17 @@
+# Similar to snare_ccoff_udp_2, but with a different template. This template
+# has triggered problems in the past, thus a test is granted.
+# added 2010-03-21 rgerhards
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# snare usses HT as field delimiter, so many users have turned off
+# control character escaping to make any sense at all from these messages...
+$EscapeControlCharactersOnReceive off
+
+# we need to use a fixed timestamp, as otherwise we can not compare :(
+# This could be improved in later versions of the testing tools, but requires
+# modification to the rsyslog core...
+$template fmt,"insert into windows (Message, Facility,FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg:::space-cc%', %syslogfacility%, '%HOSTNAME%',%syslogpriority%, '20100321185328', '20100321185328', %iut%, '%syslogtag:::space-cc%')\n",sql
+*.* :omstdout:;fmt