Merge branch 'v4-stable' into v4-devel

Conflicts:
	tcpsrv.c
	tests/Makefile.am
	tests/tcpflood.c

15 files changed, 143 insertions, 21 deletions

diff --git a/ChangeLog b/ChangeLog
index c838f6d6..197c8bc2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -74,11 +74,48 @@ Version 4.7.0  [v4-devel] (rgerhards), 2010-04-14
   Thanks to varmojfekoj for the patch [imported from 4.5.8]
 ---------------------------------------------------------------------------
 Version 4.6.6  [v4-stable] (rgerhards), 2010-11-??
+- bugfix: invalid storage type for config variables
+- bugfix: stream driver mode was not correctly set on tcp ouput on big
+  endian systems.
+  thanks varmojfekoj for the patch
+- bugfix: IPv6-address could not be specified in omrelp
+  this was due to improper parsing of ":"
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=250
+- bugfix: memory and file descriptor leak in stream processing
+  Leaks could occur under some circumstances if the file stream handler
+  errored out during the open call. Among others, this could cause very
+  big memory leaks if there were a problem with unreadable disk queue
+  files. In regard to the memory leak, this
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=256
+- bugfix: imfile potentially duplicates lines
+  This can happen when 0 bytes are read from the input file, and some
+  writer appends data to the file BEFORE we check if a rollover happens.
+  The check for rollover uses the inode and size as a criterion. So far,
+  we checked for equality of sizes, which is not given in this scenario,
+  but that does not indicate a rollover. From the source code comments:
+     Note that when we check the size, we MUST NOT check for equality.
+     The reason is that the file may have been written right after we
+     did try to read (so the file size has increased). That is NOT in
+     indicator of a rollover (this is an actual bug scenario we 
+     experienced). So we need to check if the new size is smaller than
+     what we already have seen!
+  Also, under some circumstances an invalid truncation was detected. This
+  code has now been removed, a file change (and thus resent) is only
+  detected if the inode number changes.
 - bugfix: a couple of problems that imfile had on some platforms, namely
   Ubuntu (not their fault, but occured there)
 - bugfix: imfile utilizes 32 bit to track offset. Most importantly,
   this problem can not experienced on Fedora 64 bit OS (which has
   64 bit long's!)
+- bugfix: abort if imfile reads file line of more than 64KiB
+  Thanks to Peter Eisentraut for reporting and analysing this problem.
+  bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=221
+- bugfix: omlibdbi did not use password from rsyslog.con
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203
+- bugfix: TCP connection invalidly aborted when messages needed to be
+  discarded (due to QUEUE_FULL or similar problem)
+- bugfix: a slightly more informative error message when a TCP
+  connections is aborted
 - some improvements thanks to clang's static code analyzer
   o overall cleanup (mostly unnecessary writes and otherwise unused stuff)
   o bugfix: fixed a very remote problem in msg.c which could occur when
diff --git a/doc/omrelp.html b/doc/omrelp.html
index b3132d78..22e6845f 100644
--- a/doc/omrelp.html
+++ b/doc/omrelp.html
@@ -44,6 +44,7 @@ special "RSYSLOG_ForwardFormat" (case sensitive!) template is used.<br>
 # port 2514
 *.* :omrelp:centralserv:2514;RSYSLOG_ForwardFormat
 </textarea>
+Note: to use IPv6 addresses, encode them in [::1] format.
 <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
 [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
diff --git a/plugins/omlibdbi/omlibdbi.c b/plugins/omlibdbi/omlibdbi.c
index 6f130f54..67b1edf9 100644
--- a/plugins/omlibdbi/omlibdbi.c
+++ b/plugins/omlibdbi/omlibdbi.c
@@ -287,7 +287,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	if(dbName  != NULL)
 		if((pData->dbName   = (uchar*) strdup((char*)dbName))   == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
 	if(pwd     != NULL)
-		if((pData->pwd      = (uchar*) strdup((char*)""))       == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+		if((pData->pwd      = (uchar*) strdup((char*)pwd))       == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
 
 	CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, (uchar*) " StdDBFmt"));
 
diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c
index d5ef8b4f..4a21627d 100644
--- a/plugins/omrelp/omrelp.c
+++ b/plugins/omrelp/omrelp.c
@@ -249,8 +249,18 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	/* extract the host first (we do a trick - we replace the ';' or ':' with a '\0')
 	 * now skip to port and then template name. rgerhards 2005-07-06
 	 */
-	for(q = p ; *p && *p != ';' && *p != ':' ; ++p)
-		/* JUST SKIP */;
+	if(*p == '[') { /* everything is hostname upto ']' */
+		++p; /* skip '[' */
+		for(q = p ; *p && *p != ']' ; ++p)
+			/* JUST SKIP */;
+		if(*p == ']') {
+			*p = '\0'; /* trick to obtain hostname (later)! */
+			++p; /* eat it */
+		}
+	} else { /* traditional view of hostname */
+		for(q = p ; *p && *p != ';' && *p != ':' && *p != '#' ; ++p)
+			/* JUST SKIP */;
+	}
 
 	pData->port = NULL;
 	if(*p == ':') { /* process port */
diff --git a/runtime/stream.c b/runtime/stream.c
index ce1f5ec2..44b24ee2 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -259,6 +259,7 @@ static rsRetVal strmOpenFile(strm_t *pThis)
 
 	if(pThis->fd != -1)
 		ABORT_FINALIZE(RS_RET_OK);
+	pThis->pszCurrFName = NULL; /* used to prevent mem leak in case of error */
 
 	if(pThis->pszFName == NULL)
 		ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING);
@@ -290,6 +291,16 @@ static rsRetVal strmOpenFile(strm_t *pThis)
 		  (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", pThis->fd);
 
 finalize_it:
+	if(iRet != RS_RET_OK) {
+		if(pThis->pszCurrFName != NULL) {
+			free(pThis->pszCurrFName);
+			pThis->pszCurrFName = NULL; /* just to prevent mis-adressing down the road... */
+		}
+		if(pThis->fd != -1) {
+			close(pThis->fd);
+			pThis->fd = -1;
+		}
+	}
 	RETiRet;
 }
 
@@ -401,6 +412,12 @@ finalize_it:
  * If we are monitoring a file, someone may have rotated it. In this case, we
  * also need to close it and reopen it under the same name.
  * rgerhards, 2008-02-13
+ * The previous code also did a check for file truncation, in which case the
+ * file was considered rewritten. However, this potential border case turned
+ * out to be a big trouble spot on busy systems. It caused massive message
+ * duplication (I guess stat() can return a too-low number under some
+ * circumstances). So starting as of now, we only check the inode number and
+ * a file change is detected only if the inode changes. -- rgerhards, 2011-01-10
  */
 static rsRetVal
 strmHandleEOFMonitor(strm_t *pThis)
@@ -410,23 +427,18 @@ strmHandleEOFMonitor(strm_t *pThis)
 	struct stat statName;
 
 	ISOBJ_TYPE_assert(pThis, strm);
-	/* find inodes of both current descriptor as well as file now in file
-	 * system. If they are different, the file has been rotated (or
-	 * otherwise rewritten). We also check the size, because the inode
-	 * does not change if the file is truncated (this, BTW, is also a case
-	 * where we actually loose log lines, because we can not do anything
-	 * against truncation...). We do NOT rely on the time of last
-	 * modificaton because that may not be available under all
-	 * circumstances. -- rgerhards, 2008-02-13
-	 */
 	if(fstat(pThis->fd, &statOpen) == -1)
 		ABORT_FINALIZE(RS_RET_IO_ERROR);
 	if(stat((char*) pThis->pszCurrFName, &statName) == -1)
 		ABORT_FINALIZE(RS_RET_IO_ERROR);
-	if(statOpen.st_ino == statName.st_ino && pThis->iCurrOffs == statName.st_size) {
+	DBGPRINTF("stream checking for file change on '%s', inode %u/%u",
+	  pThis->pszCurrFName, (unsigned) statOpen.st_ino,
+	  (unsigned) statName.st_ino);
+	if(statOpen.st_ino == statName.st_ino) {
 		ABORT_FINALIZE(RS_RET_EOF);
 	} else {
 		/* we had a file change! */
+		DBGPRINTF("we had a file change on '%s'\n", pThis->pszCurrFName);
 		CHKiRet(strmCloseFile(pThis));
 		CHKiRet(strmOpenFile(pThis));
 	}
diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c
index 93995b38..8b2fe455 100644
--- a/runtime/stringbuf.c
+++ b/runtime/stringbuf.c
@@ -156,7 +156,7 @@ rsRetVal
 rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded)
 {
 	uchar *pNewBuf;
-	unsigned short iNewSize;
+	size_t iNewSize;
 	DEFiRet;
 
 	/* first compute the new size needed */
diff --git a/tcpsrv.c b/tcpsrv.c
index 5de805b2..0581828e 100644
--- a/tcpsrv.c
+++ b/tcpsrv.c
@@ -547,7 +547,7 @@ Run(tcpsrv_t *pThis)
 				case RS_RET_OK:
 					/* valid data received, process it! */
 					localRet = tcps_sess.DataRcvd(pThis->pSessions[iTCPSess], buf, iRcvd);
-					if(localRet != RS_RET_OK) {
+					if(localRet != RS_RET_OK && localRet != RS_RET_QUEUE_FULL) {
 						/* in this case, something went awfully wrong.
 						 * We are instructed to terminate the session.
 						 */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 5914f94b..d3871e3a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,6 +1,6 @@
 if ENABLE_TESTBENCH
 TESTRUNS = rt_init rscript
-check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr
+check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq msleep randomgen diagtalker uxsockrcvr inputfilegen
 TESTS = $(TESTRUNS) cfg.sh \
 	validation-run.sh \
 	imtcp-multiport.sh \
@@ -52,6 +52,10 @@ if ENABLE_EXTENDED_TESTS
 TESTS += random.sh 
 endif
 
+if ENABLE_IMFILE
+TESTS += imfile-basic.sh
+endif
+
 endif # if ENABLE_TESTBENCH
 
 TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/
@@ -209,6 +213,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/complex1.conf \
 	   random.sh \
 	   testsuites/random.conf \
+	   imfile-basic.sh \
+	   testsuites/imfile-basic.conf \
 	   dynfile_invld_async.sh \
 	   dynfile_invld_sync.sh \
 	   dynfile_cachemiss.sh \
@@ -239,6 +245,9 @@ diagtalker_LDADD = $(SOL_LIBS)
 randomgen_SOURCES = randomgen.c
 randomgen_LDADD = $(SOL_LIBS)
 
+inputfilegen_SOURCES = inputfilegen.c
+inputfilegen_LDADD = $(SOL_LIBS)
+
 nettester_SOURCES = nettester.c getline.c
 nettester_LDADD = $(SOL_LIBS)
 
diff --git a/tests/diag.sh b/tests/diag.sh
index 1caf529f..3360e328 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -22,6 +22,7 @@ case $1 in
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
 		rm -f rsyslog.out.*.log work-presort
 		rm -rf test-spool
+		rm -f rsyslog.input
 		rm -f core.* vgcore.*
 		mkdir test-spool
 		;;
@@ -30,6 +31,7 @@ case $1 in
 		rm -f work rsyslog.out.log rsyslog.out.log.save # common work files
 		rm -f rsyslog.out.*.log rsyslog.random.data work-presort
 		rm -rf test-spool
+		rm -f rsyslog.input
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
    		# returns only after successful startup, $3 is the instance (blank or 2!)
diff --git a/tests/imfile-basic.sh b/tests/imfile-basic.sh
new file mode 100755
index 00000000..ca6a5d3a
--- /dev/null
+++ b/tests/imfile-basic.sh
@@ -0,0 +1,14 @@
+# This is part of the rsyslog testbench, licensed under GPLv3
+echo [imfile-basic.sh]
+source $srcdir/diag.sh init
+# generate input file first. Note that rsyslog processes it as
+# soon as it start up (so the file should exist at that point).
+./inputfilegen 50000 > rsyslog.input
+ls -l rsyslog.input
+source $srcdir/diag.sh startup imfile-basic.conf
+# sleep a little to give rsyslog a chance to begin processing
+sleep 1
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown	# we need to wait until rsyslogd is finished!
+source $srcdir/diag.sh seq-check 0 49999
+source $srcdir/diag.sh exit
diff --git a/tests/inputfilegen.c b/tests/inputfilegen.c
new file mode 100644
index 00000000..26fb79af
--- /dev/null
+++ b/tests/inputfilegen.c
@@ -0,0 +1,23 @@
+/* generate an input file suitable for use by the testbench
+ * Copyright (C) 2011 by Rainer Gerhards and Adiscon GmbH.
+ * Part of rsyslog, licensed under GPLv3
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char* argv[])
+{
+	int nmsgs;
+	int i;
+
+	if(argc != 2) {
+		fprintf(stderr, "usage: inputfilegen num-messages\n");
+		return 1;
+	}
+
+	nmsgs = atoi(argv[1]);
+	for(i = 0 ; i < nmsgs ; ++i) {
+		printf("msgnum:%8.8d:\n", i);
+	}
+	return 0;
+}
diff --git a/tests/random.sh b/tests/random.sh
index d1f392d3..969d720c 100755
--- a/tests/random.sh
+++ b/tests/random.sh
@@ -5,9 +5,6 @@
 echo ===============================================================================
 echo TEST: \[random.sh\]: testing random data
 source $srcdir/diag.sh init
-# uncomment for debugging support:
-#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
-#export RSYSLOG_DEBUGLOG="log"
 source $srcdir/diag.sh startup random.conf
 # generate random data
 ./randomgen -f rsyslog.random.data -s 100000
@@ -17,4 +14,5 @@ source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done proces
 source $srcdir/diag.sh wait-shutdown       # and wait for it to terminate
 # we do not check anything yet, the point is if rsyslog survived ;)
 # TODO: check for exit message, but we'll notice an abort anyhow, so not that important
+rm -f random.data
 source $srcdir/diag.sh exit
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 3020f389..c34f87c9 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -354,6 +354,7 @@ int main(int argc, char *argv[])
 	int ret = 0;
 	int opt;
 	struct sigaction sigAct;
+	struct rlimit maxFiles;
 	static char buf[1024];
 
 	srand(time(NULL));	/* seed is good enough for our needs */
@@ -430,6 +431,9 @@ int main(int argc, char *argv[])
 		maxFiles.rlim_max = numConnections + 20;
 		if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) {
 			perror("setrlimit to increase file handles failed");
+			fprintf(stderr,
+			        "could net set sufficiently large number of "
+			        "open files for required connection count!\n");
 			exit(1);
 		}
 	}
diff --git a/tests/testsuites/imfile-basic.conf b/tests/testsuites/imfile-basic.conf
new file mode 100644
index 00000000..9fb9b5ca
--- /dev/null
+++ b/tests/testsuites/imfile-basic.conf
@@ -0,0 +1,12 @@
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imfile/.libs/imfile
+$InputFileName ./rsyslog.input
+$InputFileTag file:
+$InputFileStateFile stat-file1
+$InputFileSeverity error
+$InputFileFacility local7
+$InputRunFileMonitor
+
+$template outfmt,"%msg:F,58:2%\n"
+:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt
diff --git a/tools/omfwd.c b/tools/omfwd.c
index 96b365a0..aadeec6a 100644
--- a/tools/omfwd.c
+++ b/tools/omfwd.c
@@ -101,8 +101,8 @@ typedef struct _instanceData {
 /* config data */
 static uchar *pszTplName = NULL; /* name of the default template to use */
 static uchar *pszStrmDrvr = NULL; /* name of the stream driver to use */
-static short iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */
-static short bResendLastOnRecon = 0; /* should the last message be re-sent on a successful reconnect? */
+static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */
+static int bResendLastOnRecon = 0; /* should the last message be re-sent on a successful reconnect? */
 static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */
 static int iUDPRebindInterval = 0;	/* support for automatic re-binding (load balancers!). 0 - no rebind */
 static int iTCPRebindInterval = 0;	/* support for automatic re-binding (load balancers!). 0 - no rebind */