diff options
author | Rainer Gerhards <rgerhards@adiscon.com> | 2010-07-13 16:15:02 +0200 |
---|---|---|
committer | Rainer Gerhards <rgerhards@adiscon.com> | 2010-07-13 16:15:02 +0200 |
commit | 73ebadd5980f91079416a14ba6463d576ecb6207 (patch) | |
tree | 09d1b1a5aff124d037468295fdacb2214c007547 | |
parent | 824a4b98d5fd96183aa2c0961df604fd5ef48915 (diff) | |
download | rsyslog-73ebadd5980f91079416a14ba6463d576ecb6207.tar.gz rsyslog-73ebadd5980f91079416a14ba6463d576ecb6207.tar.xz rsyslog-73ebadd5980f91079416a14ba6463d576ecb6207.zip |
added new parser modules
most importantly pmlastmsg, which handles the pathetic "last message
repeated n times" messages that some syslogd's emit.
Also some minor fixes, like wrong files names in make dist (for new
files introduced after last release)
-rw-r--r-- | ChangeLog | 5 | ||||
-rw-r--r-- | Makefile.am | 6 | ||||
-rw-r--r-- | configure.ac | 20 | ||||
-rw-r--r-- | doc/Makefile.am | 2 | ||||
-rw-r--r-- | doc/pmlastmsg.html | 62 | ||||
-rw-r--r-- | doc/rsyslog_conf_modules.html | 12 | ||||
-rw-r--r-- | plugins/pmlastmsg/Makefile.am | 8 | ||||
-rw-r--r-- | plugins/pmlastmsg/pmlastmsg.c | 176 | ||||
-rw-r--r-- | tests/Makefile.am | 8 | ||||
-rwxr-xr-x | tests/pmlastmsg.sh | 6 | ||||
-rw-r--r-- | tests/testsuites/master.pmlastmsg | 28 | ||||
-rw-r--r-- | tests/testsuites/pmlastmsg.conf | 16 |
12 files changed, 341 insertions, 8 deletions
@@ -1,5 +1,10 @@ --------------------------------------------------------------------------- Version 5.5.6 [DEVEL] (rgerhards), 2010-06-?? +- added parser modules + * pmlastmsg, which supports the notoriously malformed "last message + repeated n times" messages from some syslogd's (namely sysklogd) + * pmrfc3164sd (contributed), supports RFC5424 structured data in + RFC3164 messages [untested] - added new module type "string generator", used to speed up output processing. Expected speedup for (typical) rsyslog processing is roughly 5 to 6 percent compared to using string-based templates. diff --git a/Makefile.am b/Makefile.am index 2fc4b1d6..a7670f64 100644 --- a/Makefile.am +++ b/Makefile.am @@ -99,6 +99,10 @@ if ENABLE_OMSTDOUT SUBDIRS += plugins/omstdout endif +if ENABLE_PMLASTMSG +SUBDIRS += plugins/pmlastmsg +endif + if ENABLE_PMRFC3164SD SUBDIRS += plugins/pmrfc3164sd endif @@ -173,7 +177,7 @@ DISTCHECK_CONFIGURE_FLAGS= --enable-gssapi_krb5 \ --enable-diagtools \ --enable-gnutls \ --enable-omstdout \ - --enable-pmrfc3164sd \ + --enable-pmlastmsg \ --enable-omruleset \ --enable-omprog \ --enable-imdiag \ diff --git a/configure.ac b/configure.ac index 89a42aa3..bfcd3f0d 100644 --- a/configure.ac +++ b/configure.ac @@ -835,6 +835,20 @@ AC_ARG_ENABLE(omstdout, ) AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) + +# settings for pmlastmsg +AC_ARG_ENABLE(pmlastmsg, + [AS_HELP_STRING([--enable-pmlastmsg],[Compiles lastmsg parser module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_pmlastmsg="yes" ;; + no) enable_pmlastmsg="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-pmlastmsg) ;; + esac], + [enable_pmlastmsg=no] +) +AM_CONDITIONAL(ENABLE_PMLASTMSG, test x$enable_pmlastmsg = xyes) + + # settings for pmrfc3164sd AC_ARG_ENABLE(pmrfc3164sd, [AS_HELP_STRING([--enable-pmrfc3164sd],[Compiles rfc3164sd parser module @<:@default=no@:>@])], @@ -966,6 +980,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/pmrfc3164sd/Makefile \ + plugins/pmlastmsg/Makefile \ plugins/omruleset/Makefile \ plugins/omdbalerting/Makefile \ plugins/imfile/Makefile \ @@ -1011,12 +1026,15 @@ echo "---{ output plugins }---" echo " Mail support enabled: $enable_mail" echo " omprog module will be compiled: $enable_omprog" echo " omstdout module will be compiled: $enable_omstdout" -echo " pmrfc3164sd module will be compiled: $enable_pmrfc3164sd" echo " omruleset module will be compiled: $enable_omruleset" echo " omdbalerting module will be compiled: $enable_omdbalerting" echo " omudpspoof module will be compiled: $enable_omudpspoof" echo " output template module will be compiled: $enable_omtemplate" echo +echo "---{ parser modules }---" +echo " pmrfc3164sd module will be compiled: $enable_pmrfc3164sd" +echo " pmlastmsg module will be compiled: $enable_pmlastmsg" +echo echo "---{ database support }---" echo " MySql support enabled: $enable_mysql" echo " libdbi support enabled: $enable_libdbi" diff --git a/doc/Makefile.am b/doc/Makefile.am index 3d900cac..253fe998 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -33,6 +33,7 @@ html_files = \ dev_queue.html \ omstdout.html \ omudpspoof.html \ + omruleset.html \ omsnmp.html \ ommysql.html \ omoracle.html \ @@ -44,6 +45,7 @@ html_files = \ imsolaris.html \ imuxsock.html \ imklog.html \ + pmlastmsg.html \ professional_support.html \ queues.html \ src/queueWorkerLogic.dia \ diff --git a/doc/pmlastmsg.html b/doc/pmlastmsg.html new file mode 100644 index 00000000..397016df --- /dev/null +++ b/doc/pmlastmsg.html @@ -0,0 +1,62 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>parser module for "last message repeated n times" (pmlastmsg)</title> +</head> +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>parser module for "last message repeated n times" (pmlastmsg)</h1> +<p><b>Module Name: pmlastmsg</b></p> +<p><b>Module Type: parser module</b></p> +<p><b>Author: </b>Rainer Gerhards <rgerhards@adiscon.com></p> +<p><b>Available Since</b>: 5.5.6</p> +<p><b>Description</b>:</p> +<p>Some syslogds are known to emit severily malformed messages with content +"last message repeated n times". These messages can mess up message reception, as +they lead to wrong interpretation with the standard RFC3164 parser. Rather than +trying to fix this issue in pmrfc3164, we have created a new parser module +specifically for these messages. The reason is that some processing overhead is +involved in processing these messages (they must be recognized) and we would +not like to place this toll on every user but only on those actually in need +of the feature. Note that the performance toll is not large -- but if you expect +a very high message rate with tenthousands of messages per second, you will notice +a difference. +<p>This module should be loaded first inside <a href="messageparser.html">rsyslog's +parser chain</a>. It processes all those messages that contain a PRI, then none or +some spaces and then the exact text (case-insensitive) "last message repeated n times" +where n must be an integer. All other messages are left untouched. + +<p><b>Configuration Directives</b>:</p> +<p>There do not currently exist any configuration directives for this module. +<p><b>Examples:</b></p> +<p>This example is the typical use case, where some systems emit malformed +"repeated msg" messages. Other than that, the default RFC5424 and RFC3164 parsers +should be used. Note that when a parser is specified, the default parser chain +is removed, so we need to specify all three parsers. We use this together with the +default ruleset. +</p> +<textarea rows="15" cols="80">$ModLoad pmlastline # this parser is NOT a built-in module + +# note that parser are tried in the +# order they appear in rsyslog.conf, so put pmlastline first +$RulesetParser rsyslog.lastline +# as we have removed the default parser chain, we +# need to add the default parsers as well. +$RulesetParser rsyslog.rfc5424 +$RulesetParser rsyslog.rfc3164 + +# now come the typical rules, like... +*.* /path/to/file.log +</textarea> +<p><b>Caveats/Known Bugs:</b> +<p>currently none +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 57146e88..ef83d2ec 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -72,8 +72,16 @@ messages. For details, please see the <a href="messageparser.html">rsyslog message parser documentation</a>. <p>The current modules are currently provided as part of rsyslog: <ul> -<li>pmrfc5424 - parses RFC5424-formatted messages (the new syslog standard) -<li>pmrfc3164 - the traditional/legacy syslog parser +<li>pmrfc5424[builtin] - rsyslog.rfc5424 - +parses RFC5424-formatted messages (the new syslog standard) +<li>pmrfc3164[builtin] - rsyslog.rfc3164 - +the traditional/legacy syslog parser +<li>pmrfc3164sd - rsyslog.rfc3164sd - +a contributed module supporting RFC5424 structured data inside +RFC3164 messages (not supported by the rsyslog team) +<li><a href="pmlastmsg.html">pmlastmsg</a> - rsyslog.lastmsg - +a parser module that handles the typically malformed "last messages +repated n times" messages emitted by some syslogds. </ul> <a name="mm"></a><h2>Message Modification Modules</h2> diff --git a/plugins/pmlastmsg/Makefile.am b/plugins/pmlastmsg/Makefile.am new file mode 100644 index 00000000..f360243c --- /dev/null +++ b/plugins/pmlastmsg/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = pmlastmsg.la + +pmlastmsg_la_SOURCES = pmlastmsg.c +pmlastmsg_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -I ../../tools +pmlastmsg_la_LDFLAGS = -module -avoid-version +pmlastmsg_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/pmlastmsg/pmlastmsg.c b/plugins/pmlastmsg/pmlastmsg.c new file mode 100644 index 00000000..67a742c9 --- /dev/null +++ b/plugins/pmlastmsg/pmlastmsg.c @@ -0,0 +1,176 @@ +/* pmlastmsg.c + * This is a parser module specifically for those horrible + * "<PRI>last message repeated n times" messages notoriously generated + * by some syslog implementations. Note that this parser should be placed + * on top of the parser stack -- it takes out only these messages and + * leaves all others for processing by the other parsers. + * + * NOTE: read comments in module-template.h to understand how this file + * works! + * + * File begun on 2010-07-13 by RGerhards + * + * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see <http://www.gnu.org/licenses/>. + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <ctype.h> +#include "syslogd.h" +#include "conf.h" +#include "syslogd-types.h" +#include "template.h" +#include "msg.h" +#include "module-template.h" +#include "glbl.h" +#include "errmsg.h" +#include "parser.h" +#include "datetime.h" +#include "unicode-helper.h" + +MODULE_TYPE_PARSER +PARSER_NAME("rsyslog.lastline") + +/* internal structures + */ +DEF_PMOD_STATIC_DATA +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(parser) +DEFobjCurrIf(datetime) + + +/* static data */ +static int bParseHOSTNAMEandTAG; /* cache for the equally-named global param - performance enhancement */ + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATUREAutomaticSanitazion) + iRet = RS_RET_OK; + if(eFeat == sFEATUREAutomaticPRIParsing) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +/* parse a legay-formatted syslog message. + */ +BEGINparse + uchar *p2parse; + int lenMsg; +#define OpeningText "last message repeated " +#define ClosingText " times" +CODESTARTparse + dbgprintf("Message will now be parsed by \"last message repated n times\" parser.\n"); + assert(pMsg != NULL); + assert(pMsg->pszRawMsg != NULL); + lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */ + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ + + /* check if this message is of the type we handle in this (very limited) parser */ + /* first, we permit SP */ + while(lenMsg && *p2parse == ' ') { + --lenMsg; + ++p2parse; + } +dbgprintf("pmlastmsg: msg to look at: [%d]'%s'\n", lenMsg, p2parse); + if((unsigned) lenMsg < sizeof(OpeningText)-1 + sizeof(ClosingText)-1 + 1) { + /* too short, can not be "our" message */ +dbgprintf("msg too short!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + if(strncasecmp((char*) p2parse, OpeningText, sizeof(OpeningText)-1) != 0) { + /* wrong opening text */ +dbgprintf("wrong opening text!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + lenMsg -= sizeof(OpeningText) - 1; + p2parse += sizeof(OpeningText) - 1; + + /* now we need an integer --> digits */ + while(lenMsg && isdigit(*p2parse)) { + --lenMsg; + ++p2parse; + } + + if(lenMsg != sizeof(ClosingText)-1) { + /* size must fit, else it is not "our" message... */ + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + if(strncasecmp((char*) p2parse, ClosingText, lenMsg) != 0) { + /* wrong closing text */ +dbgprintf("strcasecmp: %d\n", strncasecmp((char*) p2parse, ClosingText, lenMsg)); +dbgprintf("pmlastmsg: closing msg to look at: [%d]'%s', (%s)\n", lenMsg, p2parse, ClosingText); +dbgprintf("wrong closing text!\n"); + ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE); + } + + /* OK, now we know we need to process this message, so we do that + * (and it is fairly simple in our case...) + */ + DBGPRINTF("pmlastmsg detected a \"last message repeated n times\" message\n"); + + setProtocolVersion(pMsg, 0); + memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); + MsgSetMSGoffs(pMsg, pMsg->offAfterPRI); /* we don't have a header! */ + MsgSetTAG(pMsg, (uchar*)"", 0); + +finalize_it: +ENDparse + + +BEGINmodExit +CODESTARTmodExit + /* release what we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_PMOD_QUERIES +CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + dbgprintf("lastmsg parser init called, compiled with version %s\n", VERSION); + bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */ + + +ENDmodInit + +/* vim:set ai: + */ diff --git a/tests/Makefile.am b/tests/Makefile.am index a0cc8c7d..7f9d8cd5 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -274,13 +274,13 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/execonlyonce.conf \ testsuites/execonlyonce.data \ execonlywhenprevsuspended.sh \ - testsuites/execonlywhenprevsuspended.sh \ + testsuites/execonlywhenprevsuspended.conf \ execonlywhenprevsuspended2.sh \ - testsuites/execonlywhenprevsuspended2.sh \ + testsuites/execonlywhenprevsuspended2.conf \ execonlywhenprevsuspended3.sh \ - testsuites/execonlywhenprevsuspended3.sh \ + testsuites/execonlywhenprevsuspended3.conf \ execonlywhenprevsuspended4.sh \ - testsuites/execonlywhenprevsuspended4.sh \ + testsuites/execonlywhenprevsuspended4.conf \ tabescape_dflt.sh \ testsuites/tabescape_dflt.conf \ testsuites/1.tabescape_dflt \ diff --git a/tests/pmlastmsg.sh b/tests/pmlastmsg.sh new file mode 100755 index 00000000..9a14ce21 --- /dev/null +++ b/tests/pmlastmsg.sh @@ -0,0 +1,6 @@ +echo ============================================================================== +echo \[pmlastmsg.sh\]: tests for pmlastmsg +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester pmlastmsg udp +source $srcdir/diag.sh nettester pmlastmsg tcp +source $srcdir/diag.sh exit diff --git a/tests/testsuites/master.pmlastmsg b/tests/testsuites/master.pmlastmsg new file mode 100644 index 00000000..170538b0 --- /dev/null +++ b/tests/testsuites/master.pmlastmsg @@ -0,0 +1,28 @@ +# the following messages should be processed by pmlastmsg: +<13>last message repeated 5 times +last message repeated 5 times +# +<13>last message repeated 0090909787348927349875 times +last message repeated 0090909787348927349875 times +# now slightly malformed formats that should NOT be processed +# by pmlasmsg: +<13>last message repeated 5 times + repeated 5 times +# +<13>last message repeated 5 times -- more data + repeated 5 times -- more data +# message count invalid: +<13>last message repeated 5.2 times + repeated 5.2 times +# +# +# now follow samples of non-pmlastmsg messages: +# +<167>Mar 6 16:57:54 172.20.245.8 TAG: Rest of message... + Rest of message... +# Now exactly with 32 characters +<167>Mar 6 16:57:54 172.20.245.8 TAG long message ================================================================================ + long message ================================================================================ +# RFC5424 messages +<34>1 2003-11-11T22:14:15.003Z mymachine.example.com su - ID47 last message repeated 5 times +last message repeated 5 times diff --git a/tests/testsuites/pmlastmsg.conf b/tests/testsuites/pmlastmsg.conf new file mode 100644 index 00000000..59142dca --- /dev/null +++ b/tests/testsuites/pmlastmsg.conf @@ -0,0 +1,16 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/pmlastmsg/.libs/pmlastmsg +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# in this test, we use pmlastmsg, followed by the one-size-fits-all +# pm3164 (built-in) module. So we can test if non-pmlastmsg messages +# are also correctly detected. +$RulesetParser rsyslog.lastline +$RulesetParser rsyslog.rfc5424 +$RulesetParser rsyslog.rfc3164 + +# use a special format +$template fmt,"%msg%\n" +*.* :omstdout:;fmt |