imfile improvements

- added the $InputFilePersistStateInterval config directive to imfile
- changed imfile so that the state file is never deleted (makes imfile
  more robust in regard to fatal failures)

3 files changed, 30 insertions, 18 deletions

diff --git a/ChangeLog b/ChangeLog
index a4ac46b9..acb0ef5e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -9,6 +9,9 @@ Version 4.7.3  [v4-devel] (rgerhards), 2010-??-??
 - bugfix: imfile utilizes 32 bit to track offset. Most importantly,
   this problem can not experienced on Fedora 64 bit OS (which has
   64 bit long's!)
+- added the $InputFilePersistStateInterval config directive to imfile
+- changed imfile so that the state file is never deleted (makes imfile
+  more robust in regard to fatal failures)
 ---------------------------------------------------------------------------
 Version 4.7.2  [v4-devel] (rgerhards), 2010-05-03
 - bugfix: problems with atomic operations emulaton
diff --git a/doc/imfile.html b/doc/imfile.html
index af0413dd..3687302b 100644
--- a/doc/imfile.html
+++ b/doc/imfile.html
@@ -86,6 +86,16 @@ level may be needed. Even if you need quick response, 1 seconds should
 be well enough. Please note that imfile keeps reading files as long as
 there is any data in them. So a "polling sleep" will only happen when
 nothing is left to be processed.</li>
+<li><b>$InputFilePollInterval</b> [lines]</b><br>
+Available in 4.7.3+<br>
+Specifies how often the state file shall be written when processing the input
+file. The default value is 0, which means a new state file is only written when 
+the monitored files is being closed (end of rsyslogd execution). Any other
+value n means that the state file is written every time n file lines have
+been processed. This setting can be used to guard against message duplication due
+to fatal errors (like power fail). Note that this setting affects imfile
+performance, especially when set to a low value. Frequently writing the state
+file is very time consuming.
 </ul>
 <b>Caveats/Known Bugs:</b>
 <p>So far, only 100 files can be monitored. If more are needed,
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index 3981f9f7..91493bb1 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -67,15 +67,21 @@ typedef struct fileInfo_s {
 	uchar *pszStateFile; /* file in which state between runs is to be stored */
 	int iFacility;
 	int iSeverity;
+	int nRecords; /**< How many records did we process before persisting the stream? */
+	int iPersistStateInterval; /**< how often should state be persisted? (0=on close only) */
 	strm_t *pStrm;	/* its stream (NULL if not assigned) */
 } fileInfo_t;
 
 
+/* forward definitions */
+static rsRetVal persistStrmState(fileInfo_t *pInfo);
+
 /* config variables */
 static uchar *pszFileName = NULL;
 static uchar *pszFileTag = NULL;
 static uchar *pszStateFile = NULL;
 static int iPollInterval = 10;	/* number of seconds to sleep when there was no file activity */
+static int iPersistStateInterval = 0;	/* how often if state file to be persisted? (default 0->never) */
 static int iFacility = 128; /* local0 */
 static int iSeverity = 5;  /* notice, as of rfc 3164 */
 
@@ -154,10 +160,9 @@ openFile(fileInfo_t *pThis)
 
 	CHKiRet(strm.SeekCurrOffs(pThis->pStrm));
 
-	/* OK, we could successfully read the file, so we now can request that it be deleted.
-	 * If we need it again, it will be written on the next shutdown.
+	/* note: we do not delete the state file, so that the last position remains
+	 * known even in the case that rsyslogd aborts for some reason (like powerfail)
 	 */
-	psSF->bDeleteOnClose = 1;
 
 finalize_it:
 	if(psSF != NULL)
@@ -211,6 +216,10 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData)
 		*pbHadFileData = 1; /* this is just a flag, so set it and forget it */
 		CHKiRet(enqLine(pThis, pCStr)); /* process line */
 		rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */
+		if(pThis->iPersistStateInterval > 0 && pThis->nRecords++ >= pThis->iPersistStateInterval) {
+			persistStrmState(pThis);
+			pThis->nRecords = 0;
+		}
 	}
 
 finalize_it:
@@ -244,27 +253,12 @@ finalize_it:
  * IMPORTANT: the calling interface of this function can NOT be modified. It actually is
  * called by pthreads. The provided argument is currently not being used.
  */
-/* ------------------------------------------------------------------------------------------ *
- * DO NOT TOUCH the following code - it will soon be part of the module generation macros!    */
 static void
 inputModuleCleanup(void __attribute__((unused)) *arg)
 {
 	BEGINfunc
-/* END no-touch zone                                                                          *
- * ------------------------------------------------------------------------------------------ */
-
-
-
-	/* so far not needed */
-
-
-
-/* ------------------------------------------------------------------------------------------ *
- * DO NOT TOUCH the following code - it will soon be part of the module generation macros!    */
 	ENDfunc
 }
-/* END no-touch zone                                                                          *
- * ------------------------------------------------------------------------------------------ */
 
 
 /* This function is called by the framework to gather the input. The module stays
@@ -499,6 +493,9 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal)
 
 		pThis->iSeverity = iSeverity;
 		pThis->iFacility = iFacility;
+		pThis->iPersistStateInterval = iPersistStateInterval;
+		pThis->nRecords = 0;
+		iPersistStateInterval = 0;
 	} else {
 		errmsg.LogError(0, RS_RET_OUT_OF_DESRIPTORS, "Too many file monitors configured - ignoring this one");
 		ABORT_FINALIZE(RS_RET_OUT_OF_DESRIPTORS);
@@ -544,6 +541,8 @@ CODEmodInit_QueryRegCFSLineHdlr
 	  	NULL, &iFacility, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt,
 	  	NULL, &iPollInterval, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepersiststateinterval", 0, eCmdHdlrInt,
+	  	NULL, &iPersistStateInterval, STD_LOADABLE_MODULE_ID));
 	/* that command ads a new file! */
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord,
 		addMonitor, NULL, STD_LOADABLE_MODULE_ID));