From 40814bb0307f9d8536b3dc4a28b94f80a361b41d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 Oct 2010 10:10:47 +0200 Subject: imfile improvements - added the $InputFilePersistStateInterval config directive to imfile - changed imfile so that the state file is never deleted (makes imfile more robust in regard to fatal failures) --- ChangeLog | 6 ++++++ configure.ac | 2 +- doc/imfile.html | 10 ++++++++++ plugins/imfile/imfile.c | 20 +++++++++++++++++--- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index cf752df5..0dea5306 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,11 @@ --------------------------------------------------------------------------- Version 5.6.0 [V5-STABLE] (rgerhards), 2010-10-19 +- private build +- added the $InputFilePersistStateInterval config directive to imfile +- changed imfile so that the state file is never deleted (makes imfile + more robust in regard to fatal failures) +--------------------------------------------------------------------------- +Version 5.6.0 [V5-STABLE] (rgerhards), 2010-10-19 This release brings all changes and enhancements of the 5.5.x series to the v5-stable branch. diff --git a/configure.ac b/configure.ac index 7b9a1949..410046f5 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.6.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.6.0a],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) diff --git a/doc/imfile.html b/doc/imfile.html index af0413dd..a7ac5a49 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -86,6 +86,16 @@ level may be needed. Even if you need quick response, 1 seconds should be well enough. Please note that imfile keeps reading files as long as there is any data in them. So a "polling sleep" will only happen when nothing is left to be processed. +
  • $InputFilePersistStateInterval [lines]
    +Available in 4.7.3+
    +Specifies how often the state file shall be written when processing the input +file. The default value is 0, which means a new state file is only written when +the monitored files is being closed (end of rsyslogd execution). Any other +value n means that the state file is written every time n file lines have +been processed. This setting can be used to guard against message duplication due +to fatal errors (like power fail). Note that this setting affects imfile +performance, especially when set to a low value. Frequently writing the state +file is very time consuming. Caveats/Known Bugs:

    So far, only 100 files can be monitored. If more are needed, diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 6c2a64de..686fa048 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -47,6 +47,7 @@ #include "datetime.h" #include "unicode-helper.h" #include "prop.h" +#include "stringbuf.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -67,15 +68,21 @@ typedef struct fileInfo_s { uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; + int nRecords; /**< How many records did we process before persisting the stream? */ + int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */ strm_t *pStrm; /* its stream (NULL if not assigned) */ } fileInfo_t; +/* forward definitions */ +static rsRetVal persistStrmState(fileInfo_t *pInfo); + /* config variables */ static uchar *pszFileName = NULL; static uchar *pszFileTag = NULL; static uchar *pszStateFile = NULL; static int iPollInterval = 10; /* number of seconds to sleep when there was no file activity */ +static int iPersistStateInterval = 0; /* how often if state file to be persisted? (default 0->never) */ static int iFacility = 128; /* local0 */ static int iSeverity = 5; /* notice, as of rfc 3164 */ @@ -153,10 +160,9 @@ openFile(fileInfo_t *pThis) CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); - /* OK, we could successfully read the file, so we now can request that it be deleted. - * If we need it again, it will be written on the next shutdown. + /* note: we do not delete the state file, so that the last position remains + * known even in the case that rsyslogd aborts for some reason (like powerfail) */ - psSF->bDeleteOnClose = 1; finalize_it: if(psSF != NULL) @@ -210,6 +216,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ + if(pThis->iPersistStateInterval > 0 && pThis->nRecords++ >= pThis->iPersistStateInterval) { + persistStrmState(pThis); + pThis->nRecords = 0; + } } finalize_it: @@ -477,6 +487,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) pThis->iSeverity = iSeverity; pThis->iFacility = iFacility; + pThis->iPersistStateInterval = iPersistStateInterval; } else { errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one"); ABORT_FINALIZE(RS_RET_OUT_OF_DESRIPTORS); @@ -522,6 +533,9 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iFacility, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt, NULL, &iPollInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt, + + NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID)); /* that command ads a new file! */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord, addMonitor, NULL, STD_LOADABLE_MODULE_ID)); -- cgit