- enhanced file monitor doc

- implemented $InputFilePollInterval config directive

3 files changed, 99 insertions, 42 deletions

diff --git a/ChangeLog b/ChangeLog
index e637c580..d945fe44 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 3.11.2 (rgerhards), 2008-02-??
+- added the capability to monitor text files and process their content
+  as syslog messages (including forwarding)
 - removed some left-over unnecessary dbgprintf's (cluttered screen,
   cosmetic)
 - doc bugfix: html documentation for omsnmp was missing
diff --git a/doc/imfile.html b/doc/imfile.html
index 6942b366..15ace7c1 100644
--- a/doc/imfile.html
+++ b/doc/imfile.html
@@ -13,46 +13,95 @@
 <p>Provides the ability to convert any standard text file into
 a syslog message. A standard
 text file is a file consisting of printable characters with lines
-being&nbsp;delimited by LF.</p><p>The
-file is read line-by-line and any line read is passed to rsyslog's rule
-engine. The rule engine applies filter conditons and selects which
-actions needs to be carried out.</p><p>As new lines are written they
-are taken from the file and processed. Please note that this happens
-based on a polling interval and not immediately. The file monitor
-support file rotation. To fully work, rsyslogd must run while the file
-is rotated. Then, any remaining lines from the old file are read and
-processed and when done with that, the new file is being processed from
-the beginning. If rsyslogd is stopped during rotation, the new file is
-read, but any not-yet-reported lines from the previous file can no
-longer be obtained.</p><p>When rsyslogd is stopped while monitoring a
-text file, it records the last processed location and continues to work
-from there upon restart. So no data is lost during a restart (except,
-as noted above, if the file is rotated just in this very moment).</p><p>Currently,
-the file must have a fixed name and location (directory). It is planned
-to add support for dynamically generating file names in the future.</p><p>Multiple files may be monitored by specifying $InputRunFileMonitor multiple times.
+being&nbsp;delimited by LF.</p>
+<p>The file is read line-by-line and any line read is passed to
+rsyslog's rule engine. The rule engine applies filter conditons and
+selects which actions needs to be carried out.</p>
+<p>As new lines are written they are taken from the file and
+processed. Please note that this happens based on a polling interval
+and not immediately. The file monitor support file rotation. To fully
+work, rsyslogd must run while the file is rotated. Then, any remaining
+lines from the old file are read and processed and when done with that,
+the new file is being processed from the beginning. If rsyslogd is
+stopped during rotation, the new file is read, but any not-yet-reported
+lines from the previous file can no longer be obtained.</p>
+<p>When rsyslogd is stopped while monitoring a text file, it
+records the last processed location and continues to work from there
+upon restart. So no data is lost during a restart (except, as noted
+above, if the file is rotated just in this very moment).</p>
+<p>Currently, the file must have a fixed name and location
+(directory). It is planned to add support for dynamically generating
+file names in the future.</p>
+<p>Multiple files may be monitored by specifying
+$InputRunFileMonitor multiple times.
 </p>
 <p><b>Configuration Directives</b>:</p>
 <ul>
 <li><strong>$InputFileName&nbsp;/path/to/file</strong><br>
-The file being monitored. So far, this must be an absolute name (no macros or templates)</li><li><span style="font-weight: bold;">$InputFileTag tag:</span><br>The
-tag to be used for messages that originate from this file. If you would
-like to see the colon after the tag, you need to specify it here (as
-shown above).</li><li><span style="font-weight: bold;">$InputFileStateFile &lt;name-of-state-file&gt;</span><br>Rsyslog
-must keep track of which parts of the to be monitored file it already
-processed. This is done in the state file. This file always is created
-in the rsyslog working directory (configurable via $WorkDirectory). Be
-careful to use unique names for different files being monitored. If
-there are duplicates, all sorts of "interesting" things may happen.
-Rsyslog currently does not check if a name is specified multiple times.</li><li><span style="font-weight: bold;">$InputRunFileMonitor</span><br>This <span style="font-weight: bold;">activates</span> the current monitor. It has no parameters. If you forget this directive, no file monitoring will take place.
-</li></ul><b>Caveats/Known Bugs:</b>
+The file being monitored. So far, this must be an absolute name (no
+macros or templates)</li>
+<li><span style="font-weight: bold;">$InputFileTag
+tag:</span><br>
+The tag to be used for messages that originate from this file. If you
+would like to see the colon after the tag, you need to specify it here
+(as shown above).</li>
+<li><span style="font-weight: bold;">$InputFileStateFile
+&lt;name-of-state-file&gt;</span><br>
+Rsyslog must keep track of which parts of the to be monitored file it
+already processed. This is done in the state file. This file always is
+created in the rsyslog working directory (configurable via
+$WorkDirectory). Be careful to use unique names for different files
+being monitored. If there are duplicates, all sorts of "interesting"
+things may happen. Rsyslog currently does not check if a name is
+specified multiple times.</li>
+<li><span style="font-weight: bold;">$InputRunFileMonitor</span><br>
+This <span style="font-weight: bold;">activates</span>
+the current monitor. It has no parameters. If you forget this
+directive, no file monitoring will take place.</li>
+<li><span style="font-weight: bold;">$InputFilePollInterval
+seconds</span><br>
+This is a global setting. It specifies how often files are to be polled
+for new data. The time specified is in seconds. The <span style="font-weight: bold;">default value</span> is 10 seconds. Please note that future
+releases of imfile may support per-file polling intervals, but
+currently this is not the case. If multiple $InputFilePollInterval
+statements are present in rsyslog.conf, only the last one is used.<br>
+A short poll interval provides more rapid message forwarding, but
+requires more system ressources. While it is possible, we stongly
+recommend not to set the polling interval to 0 seconds. That will make
+rsyslogd become a CPU hog, taking up considerable ressources. It is
+supported, however, for the few very unusual situations where this
+level may be needed. Even if you need quick response, 1 seconds should
+be well enough. Please note that imfile keeps reading files as long as
+there is any data in them. So a "polling sleep" will only happen when
+nothing is left to be processed.</li>
+</ul>
+<b>Caveats/Known Bugs:</b>
+<p>So far, only 100 files can be monitored. If more are needed,
+the source needs to be patched. See define MAX_INPUT_FILES in imfile.c
+</p>
 <p><b>Sample:</b></p>
-<p>The following sample monitors a single file.<br></p>
-<textarea rows="10" cols="60">
-ModLoad imfile.so
-$InputFileName /path/to/file
-$InputFileTag tag:
-$InputFileStateFile file-in-workdir to keep status in
-$InputRunFileMonitor </textarea>
+<p>The following sample monitors two files. If you need just one,
+remove the second one. If you need more, add them according to the
+sample ;). This code must be placed in /etc/rsyslog.conf (or wherever
+your distro puts rsyslog's config files).<br>
+</p>
+<textarea rows="15" cols="60">$ModLoad imfile.so #
+needs to be done just once
+# File 1
+$InputFileName /path/to/file1
+$InputFileTag tag1:
+$InputFileStateFile stat-file1
+$InputRunFileMonitor
+# File 2
+$InputFileName /path/to/file2
+$InputFileTag tag2:
+$InputFileStateFile stat-file2
+$InputRunFileMonitor
+# ... and so on ...
+#
+# check for new lines every 10 seconds
+$InputFilePollingInterval 10
+</textarea>
 <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
 [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index ce7134d2..b8198a4c 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -68,6 +68,7 @@ typedef struct fileInfo_s {
 static uchar *pszFileName = NULL;
 static uchar *pszFileTag = NULL;
 static uchar *pszStateFile = NULL;
+static int iPollInterval = 10;	/* number of seconds to sleep when there was no file activity */
 static int iFacility;
 static int iSeverity;
 
@@ -127,10 +128,10 @@ openFile(fileInfo_t *pThis)
 	/* check if the file exists */
 	if(stat((char*) pszSFNam, &stat_buf) == -1) {
 		if(errno == ENOENT) {
-			dbgoprint((obj_t*) pThis, "clean startup, no .si file found\n");
+			/* currently no object! dbgoprint((obj_t*) pThis, "clean startup, no .si file found\n"); */
 			ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
 		} else {
-			dbgoprint((obj_t*) pThis, "error %d trying to access .si file\n", errno);
+			/* currently no object! dbgoprint((obj_t*) pThis, "error %d trying to access .si file\n", errno); */
 			ABORT_FINALIZE(RS_RET_IO_ERROR);
 		}
 	}
@@ -158,7 +159,6 @@ finalize_it:
 		strmDestruct(&psSF);
 
 	if(iRet != RS_RET_OK) {
-		dbgoprint((obj_t*) pThis, "error %d reading .si file - can not read persisted info (not necessarily an error)\n", iRet);
 		CHKiRet(strmConstruct(&pThis->pStrm));
 		CHKiRet(strmSettOperationsMode(pThis->pStrm, STREAMMODE_READ));
 		CHKiRet(strmSetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR));
@@ -252,7 +252,12 @@ CODESTARTrunInput
 		pollFile(&files[i]);
 	}
 
-	srSleep(3, 0);
+RUNLOG_VAR("%d", iPollInterval);
+	/* Note: the 10ns additional wait is vitally important. It guards rsyslog against totally
+	 * hogging the CPU if the users selects a polling interval of 0 seconds. It doesn't hurt any
+	 * other valid scenario. So do not remove. -- rgerhards, 2008-02-14
+	 */
+	srSleep(iPollInterval, 10);
 
 	/* ------------------------------------------------------------------------------------------ *
 	 * DO NOT TOUCH the following code - it will soon be part of the module generation macros!    */
@@ -398,6 +403,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 
 
 	/* set defaults... */
+	iPollInterval = 10;
 	iFacility = 12; /* see RFC 3164 for values */
 	iSeverity = 4;
 
@@ -411,7 +417,6 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar __attribute
 	DEFiRet;
 	fileInfo_t *pThis;
 
-RUNLOG_VAR("%d", iFilPtr);
 	if(iFilPtr < MAX_INPUT_FILES) {
 		pThis = &files[iFilPtr];
 		++iFilPtr;
@@ -453,9 +458,10 @@ CODEmodInit_QueryRegCFSLineHdlr
 	 /* use numerical values as of RFC 3164 for the time being... */
 	 CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfileseverity", 0, eCmdHdlrInt,
 	  	NULL, &iSeverity, STD_LOADABLE_MODULE_ID));
-	 CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilesfacility", 0, eCmdHdlrInt,
+	 CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilefacility", 0, eCmdHdlrInt,
 	  	NULL, &iFacility, STD_LOADABLE_MODULE_ID));
-	/* things missing, e.g. polling intervall... */
+	 CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilepollinterval", 0, eCmdHdlrInt,
+	  	NULL, &iPollInterval, STD_LOADABLE_MODULE_ID));
 	/* that command ads a new file! */
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputrunfilemonitor", 0, eCmdHdlrGetWord,
 		addMonitor, NULL, STD_LOADABLE_MODULE_ID));