added new parser modules

most importantly pmlastmsg, which handles the pathetic "last message
repeated n times" messages that some syslogd's emit.
Also some minor fixes, like wrong files names in make dist (for new
files introduced after last release)

12 files changed, 341 insertions, 8 deletions

diff --git a/ChangeLog b/ChangeLog
index ce063363..da9379c2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,10 @@
 ---------------------------------------------------------------------------
 Version 5.5.6  [DEVEL] (rgerhards), 2010-06-??
+- added parser modules
+  * pmlastmsg, which supports the notoriously malformed "last message
+    repeated n times" messages from some syslogd's (namely sysklogd)
+  * pmrfc3164sd (contributed), supports RFC5424 structured data in 
+    RFC3164 messages [untested]
 - added new module type "string generator", used to speed up output
   processing. Expected speedup for (typical) rsyslog processing is
   roughly 5 to 6 percent compared to using string-based templates.
diff --git a/Makefile.am b/Makefile.am
index 2fc4b1d6..a7670f64 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -99,6 +99,10 @@ if ENABLE_OMSTDOUT
 SUBDIRS += plugins/omstdout
 endif
 
+if ENABLE_PMLASTMSG
+SUBDIRS += plugins/pmlastmsg
+endif
+
 if ENABLE_PMRFC3164SD
 SUBDIRS += plugins/pmrfc3164sd
 endif
@@ -173,7 +177,7 @@ DISTCHECK_CONFIGURE_FLAGS=	--enable-gssapi_krb5 \
 				--enable-diagtools \
 				--enable-gnutls \
 				--enable-omstdout \
-				--enable-pmrfc3164sd \
+				--enable-pmlastmsg \
 				--enable-omruleset \
 				--enable-omprog \
 				--enable-imdiag \
diff --git a/configure.ac b/configure.ac
index 89a42aa3..bfcd3f0d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -835,6 +835,20 @@ AC_ARG_ENABLE(omstdout,
 )
 AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes)
 
+
+# settings for pmlastmsg
+AC_ARG_ENABLE(pmlastmsg,
+        [AS_HELP_STRING([--enable-pmlastmsg],[Compiles lastmsg parser module @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_pmlastmsg="yes" ;;
+          no) enable_pmlastmsg="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmlastmsg) ;;
+         esac],
+        [enable_pmlastmsg=no]
+)
+AM_CONDITIONAL(ENABLE_PMLASTMSG, test x$enable_pmlastmsg = xyes)
+
+
 # settings for pmrfc3164sd
 AC_ARG_ENABLE(pmrfc3164sd,
         [AS_HELP_STRING([--enable-pmrfc3164sd],[Compiles rfc3164sd parser module @<:@default=no@:>@])],
@@ -966,6 +980,7 @@ AC_CONFIG_FILES([Makefile \
 		plugins/omprog/Makefile \
 		plugins/omstdout/Makefile \
 		plugins/pmrfc3164sd/Makefile \
+		plugins/pmlastmsg/Makefile \
 		plugins/omruleset/Makefile \
 		plugins/omdbalerting/Makefile \
 		plugins/imfile/Makefile \
@@ -1011,12 +1026,15 @@ echo "---{ output plugins }---"
 echo "    Mail support enabled:                     $enable_mail"
 echo "    omprog module will be compiled:           $enable_omprog"
 echo "    omstdout module will be compiled:         $enable_omstdout"
-echo "    pmrfc3164sd module will be compiled:      $enable_pmrfc3164sd"
 echo "    omruleset module will be compiled:        $enable_omruleset"
 echo "    omdbalerting module will be compiled:     $enable_omdbalerting"
 echo "    omudpspoof module will be compiled:       $enable_omudpspoof"
 echo "    output template module will be compiled:  $enable_omtemplate"
 echo
+echo "---{ parser modules }---"
+echo "    pmrfc3164sd module will be compiled:      $enable_pmrfc3164sd"
+echo "    pmlastmsg module will be compiled:        $enable_pmlastmsg"
+echo
 echo "---{ database support }---"
 echo "    MySql support enabled:                    $enable_mysql"
 echo "    libdbi support enabled:                   $enable_libdbi"
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 3d900cac..253fe998 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -33,6 +33,7 @@ html_files = \
 	dev_queue.html \
 	omstdout.html \
 	omudpspoof.html \
+	omruleset.html \
 	omsnmp.html \
 	ommysql.html \
 	omoracle.html \
@@ -44,6 +45,7 @@ html_files = \
 	imsolaris.html \
 	imuxsock.html \
 	imklog.html \
+	pmlastmsg.html \
 	professional_support.html \
 	queues.html \
 	src/queueWorkerLogic.dia \
diff --git a/doc/pmlastmsg.html b/doc/pmlastmsg.html
new file mode 100644
index 00000000..397016df
--- /dev/null
+++ b/doc/pmlastmsg.html
@@ -0,0 +1,62 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head>
+<meta http-equiv="Content-Language" content="en">
+<title>parser module for &quot;last message repeated n times&quot; (pmlastmsg)</title>
+</head>
+<body>
+<a href="rsyslog_conf_modules.html">rsyslog module reference</a>
+
+<h1>parser module for &quot;last message repeated n times&quot; (pmlastmsg)</h1>
+<p><b>Module Name:&nbsp;&nbsp;&nbsp; pmlastmsg</b></p>
+<p><b>Module Type:&nbsp;&nbsp;&nbsp; parser module</b></p>
+<p><b>Author: </b>Rainer Gerhards &lt;rgerhards@adiscon.com&gt;</p>
+<p><b>Available Since</b>: 5.5.6</p>
+<p><b>Description</b>:</p>
+<p>Some syslogds are known to emit severily malformed messages with content
+"last message repeated n times". These messages can mess up message reception, as
+they lead to wrong interpretation with the standard RFC3164 parser. Rather than
+trying to fix this issue in pmrfc3164, we have created a new parser module
+specifically for these messages. The reason is that some processing overhead is
+involved in processing these messages (they must be recognized) and we would
+not like to place this toll on every user but only on those actually in need
+of the feature. Note that the performance toll is not large -- but if you expect
+a very high message rate with tenthousands of messages per second, you will notice
+a difference.
+<p>This module should be loaded first inside <a href="messageparser.html">rsyslog's
+parser chain</a>. It processes all those messages that contain a PRI, then none or
+some spaces and then the exact text (case-insensitive) "last message repeated n times"
+where n must be an integer. All other messages are left untouched.
+
+<p><b>Configuration Directives</b>:</p>
+<p>There do not currently exist any configuration directives for this module.
+<p><b>Examples:</b></p>
+<p>This example is the typical use case, where some systems emit malformed
+"repeated msg" messages. Other than that, the default RFC5424 and RFC3164 parsers
+should be used. Note that when a parser is specified, the default parser chain
+is removed, so we need to specify all three parsers. We use this together with the
+default ruleset.
+</p>
+<textarea rows="15" cols="80">$ModLoad pmlastline # this parser is NOT a built-in module
+
+# note that parser are tried in the
+# order they appear in rsyslog.conf, so put pmlastline first
+$RulesetParser rsyslog.lastline
+# as we have removed the default parser chain, we
+# need to add the default parsers as well.
+$RulesetParser rsyslog.rfc5424 
+$RulesetParser rsyslog.rfc3164
+
+# now come the typical rules, like...
+*.* /path/to/file.log
+</textarea>
+<p><b>Caveats/Known Bugs:</b>
+<p>currently none
+<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
+[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a>
+project.<br>
+Copyright &copy; 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>.
+Released under the GNU GPL version 3 or higher.</font></p>
+</body></html>
diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html
index 57146e88..ef83d2ec 100644
--- a/doc/rsyslog_conf_modules.html
+++ b/doc/rsyslog_conf_modules.html
@@ -72,8 +72,16 @@ messages. For details, please see the <a href="messageparser.html">rsyslog
 message parser documentation</a>.
 <p>The current modules are currently provided as part of rsyslog:
 <ul>
-<li>pmrfc5424 - parses RFC5424-formatted messages (the new syslog standard)
-<li>pmrfc3164 - the traditional/legacy syslog parser
+<li>pmrfc5424[builtin] - rsyslog.rfc5424 -
+parses RFC5424-formatted messages (the new syslog standard)
+<li>pmrfc3164[builtin] - rsyslog.rfc3164 -
+the traditional/legacy syslog parser
+<li>pmrfc3164sd - rsyslog.rfc3164sd -
+a contributed module supporting RFC5424 structured data inside
+RFC3164 messages (not supported by the rsyslog team)
+<li><a href="pmlastmsg.html">pmlastmsg</a> - rsyslog.lastmsg -
+a parser module that handles the typically malformed "last messages
+repated n times" messages emitted by some syslogds.
 </ul>
 
 <a name="mm"></a><h2>Message Modification Modules</h2>
diff --git a/plugins/pmlastmsg/Makefile.am b/plugins/pmlastmsg/Makefile.am
new file mode 100644
index 00000000..f360243c
--- /dev/null
+++ b/plugins/pmlastmsg/Makefile.am
@@ -0,0 +1,8 @@
+pkglib_LTLIBRARIES = pmlastmsg.la
+
+pmlastmsg_la_SOURCES = pmlastmsg.c
+pmlastmsg_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools
+pmlastmsg_la_LDFLAGS = -module -avoid-version
+pmlastmsg_la_LIBADD = 
+
+EXTRA_DIST = 
diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c
new file mode 100644
index 00000000..67a742c9
--- /dev/null
+++ b/plugins/pmlastmsg/pmlastmsg.c
@@ -0,0 +1,176 @@
+/* pmlastmsg.c
+ * This is a parser module specifically for those horrible
+ * "<PRI>last message repeated n times" messages notoriously generated
+ * by some syslog implementations. Note that this parser should be placed
+ * on top of the parser stack -- it takes out only these messages and
+ * leaves all others for processing by the other parsers.
+ *
+ * NOTE: read comments in module-template.h to understand how this file
+ *       works!
+ *
+ * File begun on 2010-07-13 by RGerhards
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <ctype.h>
+#include "syslogd.h"
+#include "conf.h"
+#include "syslogd-types.h"
+#include "template.h"
+#include "msg.h"
+#include "module-template.h"
+#include "glbl.h"
+#include "errmsg.h"
+#include "parser.h"
+#include "datetime.h"
+#include "unicode-helper.h"
+
+MODULE_TYPE_PARSER
+PARSER_NAME("rsyslog.lastline")
+
+/* internal structures
+ */
+DEF_PMOD_STATIC_DATA
+DEFobjCurrIf(errmsg)
+DEFobjCurrIf(glbl)
+DEFobjCurrIf(parser)
+DEFobjCurrIf(datetime)
+
+
+/* static data */
+static int bParseHOSTNAMEandTAG;	/* cache for the equally-named global param - performance enhancement */
+
+
+BEGINisCompatibleWithFeature
+CODESTARTisCompatibleWithFeature
+	if(eFeat == sFEATUREAutomaticSanitazion)
+		iRet = RS_RET_OK;
+	if(eFeat == sFEATUREAutomaticPRIParsing)
+		iRet = RS_RET_OK;
+ENDisCompatibleWithFeature
+
+
+/* parse a legay-formatted syslog message.
+ */
+BEGINparse
+	uchar *p2parse;
+	int lenMsg;
+#define OpeningText "last message repeated "
+#define ClosingText " times"
+CODESTARTparse
+	dbgprintf("Message will now be parsed by \"last message repated n times\" parser.\n");
+	assert(pMsg != NULL);
+	assert(pMsg->pszRawMsg != NULL);
+	lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */
+	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
+
+	/* check if this message is of the type we handle in this (very limited) parser */
+	/* first, we permit SP */
+	while(lenMsg && *p2parse == ' ') {
+		--lenMsg;
+		++p2parse;
+	}
+dbgprintf("pmlastmsg: msg to look at: [%d]'%s'\n", lenMsg, p2parse);
+	if((unsigned) lenMsg < sizeof(OpeningText)-1 + sizeof(ClosingText)-1 + 1) {
+		/* too short, can not be "our" message */
+dbgprintf("msg too short!\n");
+		ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE);
+	}
+
+	if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) {
+		/* wrong opening text */
+dbgprintf("wrong opening text!\n");
+		ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE);
+	}
+	lenMsg -= sizeof(OpeningText) - 1;
+	p2parse += sizeof(OpeningText) - 1;
+
+	/* now we need an integer --> digits */
+	while(lenMsg && isdigit(*p2parse)) {
+		--lenMsg;
+		++p2parse;
+	}
+
+	if(lenMsg != sizeof(ClosingText)-1) {
+		/* size must fit, else it is not "our" message... */
+		ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE);
+	}
+
+	if(strncasecmp((char*) p2parse, ClosingText, lenMsg) != 0) {
+		/* wrong closing text */
+dbgprintf("strcasecmp: %d\n", strncasecmp((char*) p2parse, ClosingText, lenMsg));
+dbgprintf("pmlastmsg: closing msg to look at: [%d]'%s', (%s)\n", lenMsg, p2parse, ClosingText);
+dbgprintf("wrong closing text!\n");
+		ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE);
+	}
+
+	/* OK, now we know we need to process this message, so we do that
+	 * (and it is fairly simple in our case...)
+	 */
+	DBGPRINTF("pmlastmsg detected a \"last message repeated n times\" message\n");
+
+	setProtocolVersion(pMsg, 0);
+	memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
+	MsgSetMSGoffs(pMsg, pMsg->offAfterPRI); /* we don't have a header! */
+	MsgSetTAG(pMsg, (uchar*)"", 0);
+
+finalize_it:
+ENDparse
+
+
+BEGINmodExit
+CODESTARTmodExit
+	/* release what we no longer need */
+	objRelease(errmsg, CORE_COMPONENT);
+	objRelease(glbl, CORE_COMPONENT);
+	objRelease(parser, CORE_COMPONENT);
+	objRelease(datetime, CORE_COMPONENT);
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_PMOD_QUERIES
+CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES
+ENDqueryEtryPt
+
+
+BEGINmodInit()
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(glbl, CORE_COMPONENT));
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	CHKiRet(objUse(parser, CORE_COMPONENT));
+	CHKiRet(objUse(datetime, CORE_COMPONENT));
+
+	dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION);
+ 	bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */
+
+
+ENDmodInit
+
+/* vim:set ai:
+ */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index a0cc8c7d..7f9d8cd5 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -274,13 +274,13 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/execonlyonce.conf \
 	   testsuites/execonlyonce.data \
 	   execonlywhenprevsuspended.sh \
-	   testsuites/execonlywhenprevsuspended.sh \
+	   testsuites/execonlywhenprevsuspended.conf \
 	   execonlywhenprevsuspended2.sh \
-	   testsuites/execonlywhenprevsuspended2.sh \
+	   testsuites/execonlywhenprevsuspended2.conf \
 	   execonlywhenprevsuspended3.sh \
-	   testsuites/execonlywhenprevsuspended3.sh \
+	   testsuites/execonlywhenprevsuspended3.conf \
 	   execonlywhenprevsuspended4.sh \
-	   testsuites/execonlywhenprevsuspended4.sh \
+	   testsuites/execonlywhenprevsuspended4.conf \
 	   tabescape_dflt.sh \
 	   testsuites/tabescape_dflt.conf \
 	   testsuites/1.tabescape_dflt \
diff --git a/tests/pmlastmsg.sh b/tests/pmlastmsg.sh
new file mode 100755
index 00000000..9a14ce21
--- /dev/null
+++ b/tests/pmlastmsg.sh
@@ -0,0 +1,6 @@
+echo ==============================================================================
+echo \[pmlastmsg.sh\]: tests for pmlastmsg
+source $srcdir/diag.sh init
+source $srcdir/diag.sh nettester pmlastmsg udp
+source $srcdir/diag.sh nettester pmlastmsg tcp
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/master.pmlastmsg b/tests/testsuites/master.pmlastmsg
new file mode 100644
index 00000000..170538b0
--- /dev/null
+++ b/tests/testsuites/master.pmlastmsg
@@ -0,0 +1,28 @@
+# the following messages should be processed by pmlastmsg:
+<13>last message repeated 5 times
+last message repeated 5 times
+#
+<13>last message repeated 0090909787348927349875 times
+last message repeated 0090909787348927349875 times
+# now slightly malformed formats that should NOT be processed
+# by pmlasmsg:
+<13>last message  repeated 5 times
+  repeated 5 times
+#
+<13>last message repeated 5 times -- more data
+ repeated 5 times -- more data
+# message count invalid:
+<13>last message repeated 5.2 times
+ repeated 5.2 times
+#
+#
+# now follow samples of non-pmlastmsg messages:
+# 
+<167>Mar  6 16:57:54 172.20.245.8 TAG: Rest of message...
+ Rest of message...
+# Now exactly with 32 characters
+<167>Mar  6 16:57:54 172.20.245.8 TAG long message ================================================================================
+ long message ================================================================================
+# RFC5424 messages
+<34>1 2003-11-11T22:14:15.003Z mymachine.example.com su - ID47 last message repeated 5 times
+last message repeated 5 times
diff --git a/tests/testsuites/pmlastmsg.conf b/tests/testsuites/pmlastmsg.conf
new file mode 100644
index 00000000..59142dca
--- /dev/null
+++ b/tests/testsuites/pmlastmsg.conf
@@ -0,0 +1,16 @@
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$ModLoad ../plugins/pmlastmsg/.libs/pmlastmsg
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+
+# in this test, we use pmlastmsg, followed by the one-size-fits-all
+# pm3164 (built-in) module. So we can test if non-pmlastmsg messages
+# are also correctly detected.
+$RulesetParser rsyslog.lastline
+$RulesetParser rsyslog.rfc5424
+$RulesetParser rsyslog.rfc3164
+
+# use a special format
+$template fmt,"%msg%\n"
+*.* :omstdout:;fmt