From 96ac7824cd3bc33db9ff004ad7a7db0377ead200 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 May 2011 11:56:40 +0200 Subject: doc: added merged important patch to ChangeLog --- ChangeLog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ChangeLog b/ChangeLog index db5d17e3..acc8d5f3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,15 @@ --------------------------------------------------------------------------- Version 5.8.1 [V5-stable] (rgerhards), 2011-04-?? +- bugfix: invalid processing in QUEUE_FULL condition + If the the multi-submit interface was used and a QUEUE_FULL condition + occured, the failed message was properly destructed. However, the + rest of the input batch, if it existed, was not processed. So this + lead to potential loss of messages and a memory leak. The potential + loss of messages was IMHO minor, because they would have been dropped + in most cases due to the queue remaining full, but very few lucky ones + from the batch may have made it. Anyhow, this has now been changed so + that the rest of the batch is properly tried to be enqueued and, if + not possible, destructed. - new module mmsnmptrapd, a sample message modification module This can be useful to reformat snmptrapd messages and also serves as a sample for how to write message modification modules using the -- cgit From 687aaaab178fb098fa27a5921762deaa0ac4728c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 12 May 2011 01:58:46 +0200 Subject: mmsnmptrapd: removed unnecessary includes --- plugins/mmsnmptrapd/mmsnmptrapd.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/mmsnmptrapd/mmsnmptrapd.c b/plugins/mmsnmptrapd/mmsnmptrapd.c index 8f996832..767829d6 100644 --- a/plugins/mmsnmptrapd/mmsnmptrapd.c +++ b/plugins/mmsnmptrapd/mmsnmptrapd.c @@ -37,9 +37,6 @@ #include #include #include -#include -#include -#include #include "conf.h" #include "msg.h" #include "syslogd-types.h" -- cgit From 8ca5c6b27feb46efbf6d3e3b8a78b0f839bfe613 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 12 May 2011 02:00:13 +0200 Subject: omlibdbi: added necessary include --- plugins/omlibdbi/omlibdbi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c index 7fcf9631..e6f3fbd9 100644 --- a/plugins/omlibdbi/omlibdbi.c +++ b/plugins/omlibdbi/omlibdbi.c @@ -43,6 +43,7 @@ #include "dirty.h" #include "syslogd-types.h" #include "cfsysline.h" +#include "conf.h" #include "srUtils.h" #include "template.h" #include "module-template.h" -- cgit From c399118345dff8568613133054e98ddf46d67aa5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 May 2011 12:44:28 +0200 Subject: preparing for 5.8.1 stable --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index acc8d5f3..665ed184 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 5.8.1 [V5-stable] (rgerhards), 2011-04-?? +Version 5.8.1 [V5-stable] (rgerhards), 2011-05-19 - bugfix: invalid processing in QUEUE_FULL condition If the the multi-submit interface was used and a QUEUE_FULL condition occured, the failed message was properly destructed. However, the diff --git a/configure.ac b/configure.ac index 6ac6ff65..1faaff89 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.8.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.8.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/manual.html b/doc/manual.html index e1f0ebf9..5c656752 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

-

This documentation is for version 5.8.0 (stable branch) of rsyslog. +

This documentation is for version 5.8.1 (stable branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

If you like rsyslog, you might -- cgit From 59056371156616274d9970300b3ab02432201422 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 24 May 2011 09:34:14 +0200 Subject: enhanced imfile to support input batching --- ChangeLog | 1 + plugins/imfile/imfile.c | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 665ed184..0d35ce0f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +- enhanced imfile to support input batching --------------------------------------------------------------------------- Version 5.8.1 [V5-stable] (rgerhards), 2011-05-19 - bugfix: invalid processing in QUEUE_FULL condition diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index cac3a55d..415eb137 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -64,6 +64,7 @@ DEFobjCurrIf(strm) DEFobjCurrIf(prop) DEFobjCurrIf(ruleset) +#define NUM_MULTISUB 1024 /* max number of submits -- TODO: make configurable */ typedef struct fileInfo_s { uchar *pszFileName; uchar *pszTag; @@ -76,6 +77,7 @@ typedef struct fileInfo_s { strm_t *pStrm; /* its stream (NULL if not assigned) */ int readMode; /* which mode to use in ReadMulteLine call? */ ruleset_t *pRuleset; /* ruleset to bind listener to (use system default if unspecified) */ + multi_submit_t multiSub; } fileInfo_t; @@ -122,7 +124,9 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); MsgSetRuleset(pMsg, pInfo->pRuleset); - CHKiRet(submitMsg(pMsg)); + pInfo->multiSub.ppMsgs[pInfo->multiSub.nElem++] = pMsg; + if(pInfo->multiSub.nElem == pInfo->multiSub.maxElem) + CHKiRet(multiSubmitMsg(&pInfo->multiSub)); finalize_it: RETiRet; } @@ -229,8 +233,13 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) pThis->nRecords = 0; } } + /* NOTE: This is usually not reached due to loop exit via CHKiRet() only! */ finalize_it: + if(pThis->multiSub.nElem > 0) { + /* submit everything that was not yet submitted */ + CHKiRet(multiSubmitMsg(&pThis->multiSub)); + } ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ /* Note: the problem above is that pthread:cleanup_pop() is a macro which * evaluates to something like "} while(0);". So the code would become @@ -513,6 +522,9 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->pszStateFile = (uchar*) strdup((char*) pszStateFile); } + CHKmalloc(pThis->multiSub.ppMsgs = MALLOC(NUM_MULTISUB * sizeof(msg_t*))); + pThis->multiSub.maxElem = NUM_MULTISUB; + pThis->multiSub.nElem = 0; pThis->iSeverity = iSeverity; pThis->iFacility = iFacility; pThis->iPersistStateInterval = iPersistStateInterval; -- cgit From ffdc33e3f178ad85ba5c2c9f7fcee98b743e9d5e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 24 May 2011 10:09:43 +0200 Subject: imfile: added $InputFileMaxLinesAtOnce directive --- ChangeLog | 1 + doc/imfile.html | 11 +++++++++-- plugins/imfile/imfile.c | 11 ++++++++++- tests/inputfilegen.c | 1 + 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0d35ce0f..d139f5b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ +- imfile: added $InputFileMaxLinesAtOnce directive - enhanced imfile to support input batching --------------------------------------------------------------------------- Version 5.8.1 [V5-stable] (rgerhards), 2011-05-19 diff --git a/doc/imfile.html b/doc/imfile.html index 60726ceb..60bbbeea 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -98,9 +98,16 @@ performance, especially when set to a low value. Frequently writing the state file is very time consuming.

  • $InputFileReadMode [mode]
    Available in 5.7.5+ +
  • $InputFileMaxLinesAtOnce [number]
    +Available in 5.9.0+
    -Mode to be used when reading lines. 0 (the default) means that each line is forwarded -as its own log message. +This is useful if multiple files need to be monitored. If set to 0, each file +will be fully processed and then processing switches to the next file +(this was the default in previous versions). If it is set, a maximum of +[number] lines is processed in sequence for each file, and then the file is +switched. This provides a kind of mutiplexing the load of multiple files and +probably leads to a more natural distribution of events when multiple busy files +are monitored. The default is 10240.
  • $InputFileBindRuleset <ruleset>
    Available in 5.7.5+, 6.1.5+ Binds the listener to a specific ruleset.
    • diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 415eb137..9bc84220 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -72,6 +72,7 @@ typedef struct fileInfo_s { uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; + int maxLinesAtOnce; int nRecords; /**< How many records did we process before persisting the stream? */ int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ @@ -93,6 +94,7 @@ static int iPersistStateInterval = 0; /* how often if state file to be persisted static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ static int readMode = 0; /* mode to use for ReadMultiLine call */ +static int maxLinesAtOnce = 10240; /* how many lines to process in a row? */ static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ static int iFilPtr = 0; /* number of files to be monitored; pointer to next free spot during config */ @@ -210,6 +212,7 @@ static void pollFileCancelCleanup(void *pArg) static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) { cstr_t *pCStr = NULL; + int nProcessed = 0; DEFiRet; ASSERT(pbHadFileData != NULL); @@ -224,7 +227,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(glbl.GetGlobalInputTermState() == 0) { + if(pThis->maxLinesAtOnce != 0 && nProcessed >= pThis->maxLinesAtOnce) + break; CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr, pThis->readMode)); + ++nProcessed; *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ @@ -233,7 +239,6 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) pThis->nRecords = 0; } } - /* NOTE: This is usually not reached due to loop exit via CHKiRet() only! */ finalize_it: if(pThis->multiSub.nElem > 0) { @@ -484,6 +489,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iSeverity = 5; /* notice, as of rfc 3164 */ readMode = 0; pBindRuleset = NULL; + maxLinesAtOnce = 10240; RETiRet; } @@ -527,6 +533,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->multiSub.nElem = 0; pThis->iSeverity = iSeverity; pThis->iFacility = iFacility; + pThis->maxLinesAtOnce = maxLinesAtOnce; pThis->iPersistStateInterval = iPersistStateInterval; pThis->nRecords = 0; pThis->readMode = readMode; @@ -604,6 +611,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilereadmode", 0, eCmdHdlrInt, NULL, &readMode, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilemaxlinesatonce", 0, eCmdHdlrSize, + NULL, &maxLinesAtOnce, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilebindruleset", 0, eCmdHdlrGetWord, diff --git a/tests/inputfilegen.c b/tests/inputfilegen.c index 26fb79af..0ff8d049 100644 --- a/tests/inputfilegen.c +++ b/tests/inputfilegen.c @@ -1,5 +1,6 @@ /* generate an input file suitable for use by the testbench * Copyright (C) 2011 by Rainer Gerhards and Adiscon GmbH. + * usage: ./inputfilegen num-lines > file * Part of rsyslog, licensed under GPLv3 */ #include -- cgit