From 7f0c927623b6425fee2c39b2d5207d7c36e32acd Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 2 Jun 2011 11:50:15 +0200
Subject: bugfix: $ActionFileDefaultTemplate did not work

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=262
---
 ChangeLog      |  4 ++++
 tools/omfile.c | 10 +++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 45ed6501..46a41574 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,8 @@
 ---------------------------------------------------------------------------
+Version 6.1.9  [BETA] (rgerhards), 2011-0?-??
+- bugfix: $ActionFileDefaultTemplate did not work
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=262
+---------------------------------------------------------------------------
 Version 6.1.8  [BETA] (rgerhards), 2011-05-20
 - official new beta version (note that in a sense 6.1.7 was already beta,
   so we may release the first stable v6 earlier than usual)
diff --git a/tools/omfile.c b/tools/omfile.c
index 8526cb74..eb9933cd 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -177,15 +177,15 @@ typedef struct configSettings_s {
 	int64	iIOBufSize;	/* size of an io buffer */
 	int	iFlushInterval; 	/* how often flush the output buffer on inactivity? */
 	int	bUseAsyncWriter;	/* should we enable asynchronous writing? */
-	uchar	*pszFileDfltTplName; /* name of the default template to use */
 	EMPTY_STRUCT
 } configSettings_t;
+uchar	*pszFileDfltTplName; /* name of the default template to use */
 
 SCOPING_SUPPORT; /* must be set AFTER configSettings_t is defined */
 
 BEGINinitConfVars		/* (re)set config variables to default values */
 CODESTARTinitConfVars 
-	cs.pszFileDfltTplName = NULL; /* make sure this can be free'ed! */
+	pszFileDfltTplName = NULL; /* make sure this can be free'ed! */
 	iRet = resetConfigVariables(NULL, NULL); /* params are dummies */
 ENDinitConfVars
 
@@ -849,8 +849,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	cs.iIOBufSize = IOBUF_DFLT_SIZE;
 	cs.iFlushInterval = FLUSH_INTRVL_DFLT;
 	cs.bUseAsyncWriter = USE_ASYNCWRITER_DFLT;
-	free(cs.pszFileDfltTplName);
-	cs.pszFileDfltTplName = NULL;
+	free(pszFileDfltTplName);
+	pszFileDfltTplName = NULL;
 
 	return RS_RET_OK;
 }
@@ -914,7 +914,7 @@ SCOPINGmodInit
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"failonchownfailure", 0, eCmdHdlrBinary, NULL, &cs.bFailOnChown, STD_LOADABLE_MODULE_ID, eConfObjAction));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileForceChown", 0, eCmdHdlrBinary, NULL, &cs.bForceChown, STD_LOADABLE_MODULE_ID, eConfObjAction));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionfileenablesync", 0, eCmdHdlrBinary, NULL, &cs.bEnableSync, STD_LOADABLE_MODULE_ID, eConfObjAction));
-	CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &cs.pszFileDfltTplName, NULL, eConfObjAction));
+	CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszFileDfltTplName, NULL, eConfObjGlobal));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID, eConfObjAction));
 ENDmodInit
 /* vi:set ai:
-- 
cgit 


From 2b5a7f10f5cde91a6463a23daeeb7fbab051e642 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Fri, 10 Jun 2011 19:49:42 +0200
Subject: somewhat improved debug logging

---
 runtime/nsd_ptcp.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c
index c8915231..69eb7684 100644
--- a/runtime/nsd_ptcp.c
+++ b/runtime/nsd_ptcp.c
@@ -334,6 +334,12 @@ AcceptConnReq(nsd_t *pNsd, nsd_t **ppNew)
 
 	iNewSock = accept(pThis->sock, (struct sockaddr*) &addr, &addrlen);
 	if(iNewSock < 0) {
+		if(Debug) {
+			char errStr[1024];
+			rs_strerror_r(errno, errStr, sizeof(errStr));
+			dbgprintf("nds_ptcp: error accepting connection on socket %d, errno %d: %s\n",
+				  pThis->sock, errno, errStr);
+		}
 		ABORT_FINALIZE(RS_RET_ACCEPT_ERR);
 	}
 
-- 
cgit 


From 2c81df12bcbe85d819a43227cc9adb05d8d0fe29 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Fri, 10 Jun 2011 22:50:25 +0200
Subject: bugfix: memory leak in imtcp & subsystems under some circumstances

This leak is tied to error conditions which lead to incorrect cleanup
of some data structures.

Note: this is a backport from v6. In v5, we currently do not have
the toolchain to verify the original problem and that it is solved.
So this patch is preliminary and subject to change as work progresses.
---
 ChangeLog             | 5 +++++
 runtime/nsdsel_gtls.c | 1 +
 tcpsrv.c              | 9 ++++++---
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 665ed184..d28a6bf0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,9 @@
 ---------------------------------------------------------------------------
+Version 5.8.2  [V5-stable] (rgerhards), 2011-06-??
+- bugfix: memory leak in imtcp & subsystems under some circumstances
+  This leak is tied to error conditions which lead to incorrect cleanup
+  of some data structures. [backport from v6]
+---------------------------------------------------------------------------
 Version 5.8.1  [V5-stable] (rgerhards), 2011-05-19
 - bugfix: invalid processing in QUEUE_FULL condition
   If the the multi-submit interface was used and a QUEUE_FULL condition
diff --git a/runtime/nsdsel_gtls.c b/runtime/nsdsel_gtls.c
index 1a389a00..aff55af2 100644
--- a/runtime/nsdsel_gtls.c
+++ b/runtime/nsdsel_gtls.c
@@ -177,6 +177,7 @@ doRetry(nsd_gtls_t *pNsd)
 finalize_it:
 	if(iRet != RS_RET_OK && iRet != RS_RET_CLOSED && iRet != RS_RET_RETRY)
 		pNsd->bAbortConn = 1; /* request abort */
+dbgprintf("XXXXXX: doRetry: iRet %d, pNsd->bAbortConn %d\n", iRet, pNsd->bAbortConn);
 	RETiRet;
 }
 
diff --git a/tcpsrv.c b/tcpsrv.c
index 9972a135..0b822511 100644
--- a/tcpsrv.c
+++ b/tcpsrv.c
@@ -556,6 +556,7 @@ RunSelect(tcpsrv_t *pThis)
 	int bIsReady;
 	tcps_sess_t *pNewSess;
 	nssel_t *pSel = NULL;
+	rsRetVal localRet;
 
 	ISOBJ_TYPE_assert(pThis, tcpsrv);
 
@@ -604,8 +605,8 @@ RunSelect(tcpsrv_t *pThis)
 		while(nfds && iTCPSess != -1) {
 			if(glbl.GetGlobalInputTermState() == 1)
 				ABORT_FINALIZE(RS_RET_FORCE_TERM);
-			CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds));
-			if(bIsReady) {
+			localRet = nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds);
+			if(bIsReady || localRet != RS_RET_OK) {
 				doReceive(pThis, &pThis->pSessions[iTCPSess], NULL);
 				--nfds; /* indicate we have processed one */
 			}
@@ -618,7 +619,9 @@ finalize_it: /* this is a very special case - this time only we do not exit the
 	      * crashed, which made sense (the rest of the engine was not prepared for
 	      * that) -- rgerhards, 2008-05-19
 	      */
-		/*EMPTY*/;
+		if(pSel != NULL) { /* cleanup missing? happens during err exit! */
+			nssel.Destruct(&pSel);
+		}
 	}
 
 	/* note that this point is usually not reached */
-- 
cgit 


From d5906846156e49cad90736b949712fe17eb8edba Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 14 Jun 2011 11:47:17 +0200
Subject: backported current tcpflood testing tool

so that we can improve v5's testbench as well
---
 tests/Makefile.am |   3 +-
 tests/tcpflood.c  | 609 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 557 insertions(+), 55 deletions(-)

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 930aa304..8e178ab8 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -426,7 +426,8 @@ uxsockrcvr_SOURCES = uxsockrcvr.c
 uxsockrcvr_LDADD = $(SOL_LIBS)
 
 tcpflood_SOURCES = tcpflood.c
-tcpflood_LDADD = $(SOL_LIBS)
+tcpflood_CPPFLAGS =  $(PTHREADS_CFLAGS)
+tcpflood_LDADD = $(SOL_LIBS) $(PTHREADS_LIBS)
 
 syslog_caller_SOURCES = syslog_caller.c
 syslog_caller_LDADD = $(SOL_LIBS)
diff --git a/tests/tcpflood.c b/tests/tcpflood.c
index 3e7053c9..49b1e9e6 100644
--- a/tests/tcpflood.c
+++ b/tests/tcpflood.c
@@ -16,6 +16,7 @@
  *      bytes as forth. Add -r to randomize the amount of extra
  *      data included in the range 1..(value of -d).
  * -r	randomize amount of extra data added (-d must be > 0)
+ * -s	(silent) do not show progress indicator (never done on non-tty)
  * -f	support for testing dynafiles. If given, include a dynafile ID
  *      in the range 0..(f-1) as the SECOND field, shifting all field values
  *      one field to the right. Zero (default) disables this functionality.
@@ -32,6 +33,24 @@
  * -D	randomly drop and re-establish connections. Useful for stress-testing
  *      the TCP receiver.
  * -F	USASCII value for frame delimiter (in octet-stuffing mode), default LF
+ * -R	number of times the test shall be run (very useful for gathering performance
+ *      data and other repetitive things). Default: 1
+ * -S   number of seconds to sleep between different runs (-R) Default: 30
+ * -X   generate sTats data records. Default: off
+ * -e   encode output in CSV (not yet everywhere supported)
+ *      for performance data:
+ *      each inidividual line has the runtime of one test
+ *      the last line has 0 in field 1, followed by numberRuns,TotalRuntime,
+ *      Average,min,max
+ * -T   transport to use. Currently supported: "udp", "tcp" (default)
+ *      Note: UDP supports a single target port, only
+ * -W	wait time between sending batches of messages, in microseconds (Default: 0)
+ * -b   number of messages within a batch (default: 100,000,000 millions)
+ * -Y	use multiple threads, one per connection (which means 1 if one only connection
+ *  	is configured!)
+ * -z	private key file for TLS mode
+ * -Z	cert (public key) file for TLS mode
+ * -L	loglevel to use for GnuTLS troubleshooting (0-off to 10-all, 0 default)
  *
  * Part of the testbench for rsyslog.
  *
@@ -66,7 +85,15 @@
 #include <unistd.h>
 #include <string.h>
 #include <netinet/in.h>
+#include <pthread.h>
 #include <sys/resource.h>
+#include <sys/time.h>
+#include <errno.h>
+#ifdef ENABLE_GNUTLS
+#	include <gnutls/gnutls.h>
+#	include <gcrypt.h>
+	GCRY_THREAD_OPTION_PTHREAD_IMPL;
+#endif
 
 #define EXIT_FAILURE 1
 #define INVALID_SOCKET -1
@@ -74,6 +101,7 @@
 #define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */
 
 #define MAX_EXTRADATA_LEN 100*1024
+#define MAX_SENDBUF 2 * MAX_EXTRADATA_LEN
 
 static char *targetIP = "127.0.0.1";
 static char *msgPRI = "167";
@@ -83,10 +111,11 @@ static int dynFileIDs = 0;
 static int extraDataLen = 0; /* amount of extra data to add to message */
 static int bRandomizeExtraData = 0; /* randomize amount of extra data added */
 static int numMsgsToSend; /* number of messages to send */
-static int numConnections = 1; /* number of connections to create */
+static unsigned numConnections = 1; /* number of connections to create */
 static int *sockArray;  /* array of sockets to use */
 static int msgNum = 0;	/* initial message number to start with */
 static int bShowProgress = 1; /* show progress messages */
+static int bSilent = 0; /* completely silent operation */
 static int bRandConnDrop = 0; /* randomly drop connections? */
 static char *MsgToSend = NULL; /* if non-null, this is the actual message to send */
 static int bBinaryFile = 0;	/* is -I file binary */
@@ -95,6 +124,75 @@ static int numFileIterations = 1;/* how often is file data to be sent? */
 static char frameDelim = '\n';	/* default frame delimiter */
 FILE *dataFP = NULL;		/* file pointer for data file, if used */
 static long nConnDrops = 0;	/* counter: number of time connection was dropped (-D option) */
+static int numRuns = 1;		/* number of times the test shall be run */
+static int sleepBetweenRuns = 30; /* number of seconds to sleep between runs */
+static int bStatsRecords = 0;	/* generate stats records */
+static int bCSVoutput = 0;	/* generate output in CSV (where applicable) */
+static long long batchsize = 100000000ll;
+static int waittime = 0;
+static int runMultithreaded = 0; /* run tests in multithreaded mode */
+static int numThrds = 1;	/* number of threads to use */
+static char *tlsCertFile = NULL;
+static char *tlsKeyFile = NULL;
+static int tlsLogLevel = 0;
+
+#ifdef ENABLE_GNUTLS
+static gnutls_session_t *sessArray;	/* array of TLS sessions to use */
+static gnutls_certificate_credentials tlscred;
+#endif
+
+/* variables for managing multi-threaded operations */
+int runningThreads;		/* number of threads currently running */
+int doRun;			/* shall sender thread begin to run? */
+pthread_mutex_t thrdMgmt;	/* mutex for controling startup/shutdown */
+pthread_cond_t condStarted;
+pthread_cond_t condDoRun;
+
+/* the following struct provides information for a generator instance (thread) */
+struct instdata {
+	/* lower and upper bounds for the thread in question */
+	unsigned long long lower;
+	unsigned long long numMsgs; /* number of messages to send */
+	unsigned long long numSent; /* number of messages already sent */
+	unsigned idx;	/**< index of fd to be used for sending */
+	pthread_t thread; /**< thread processing this instance */
+} *instarray = NULL;
+
+/* the following structure is used to gather performance data */
+struct runstats {
+	unsigned long long totalRuntime;
+	unsigned long minRuntime;
+	unsigned long maxRuntime;
+	int numRuns;
+};
+
+static int udpsock;			/* socket for sending in UDP mode */
+static struct sockaddr_in udpRcvr;	/* remote receiver in UDP mode */
+
+static enum { TP_UDP, TP_TCP, TP_TLS } transport = TP_TCP;
+
+/* forward definitions */
+static void initTLSSess(int);
+static int sendTLS(int i, char *buf, int lenBuf);
+static void closeTLSSess(int __attribute__((unused)) i);
+
+/* prepare send subsystem for UDP send */
+static inline int
+setupUDP(void)
+{
+	if((udpsock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1)
+		return 1;
+
+	memset((char *) &udpRcvr, 0, sizeof(udpRcvr));
+	udpRcvr.sin_family = AF_INET;
+	udpRcvr.sin_port = htons(targetPort);
+	if(inet_aton(targetIP, &udpRcvr.sin_addr)==0) {
+		fprintf(stderr, "inet_aton() failed\n");
+		return(1);
+	}
+
+	return 0;
+}
 
 
 /* open a single tcp connection
@@ -150,12 +248,18 @@ int openConn(int *fd)
  */
 int openConnections(void)
 {
-	int i;
+	unsigned i;
 	char msgBuf[128];
 	size_t lenMsg;
 
+	if(transport == TP_UDP)
+		return setupUDP();
+
 	if(bShowProgress)
 		write(1, "      open connections", sizeof("      open connections")-1);
+#	ifdef ENABLE_GNUTLS
+	sessArray = calloc(numConnections, sizeof(gnutls_session_t));
+#	endif
 	sockArray = calloc(numConnections, sizeof(int));
 	for(i = 0 ; i < numConnections ; ++i) {
 		if(i % 10 == 0) {
@@ -166,9 +270,14 @@ int openConnections(void)
 			printf("error in trying to open connection i=%d\n", i);
 			return 1;
 		}
+		if(transport == TP_TLS) {
+			initTLSSess(i);
+		}
+	}
+	if(bShowProgress) {
+		lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i);
+		write(1, msgBuf, lenMsg);
 	}
-	lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i);
-	write(1, msgBuf, lenMsg);
 
 	return 0;
 }
@@ -183,11 +292,14 @@ int openConnections(void)
  */
 void closeConnections(void)
 {
-	int i;
+	unsigned i;
 	size_t lenMsg;
 	struct linger ling;
 	char msgBuf[128];
 
+	if(transport == TP_UDP)
+		return;
+
 	if(bShowProgress)
 		write(1, "      close connections", sizeof("      close connections")-1);
 	for(i = 0 ; i < numConnections ; ++i) {
@@ -204,11 +316,15 @@ void closeConnections(void)
 			ling.l_onoff = 1;
 			ling.l_linger = 1;
 			setsockopt(sockArray[i], SOL_SOCKET, SO_LINGER, &ling, sizeof(ling));
+			if(transport == TP_TLS)
+				closeTLSSess(i);
 			close(sockArray[i]);
 		}
 	}
-	lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i);
-	write(1, msgBuf, lenMsg);
+	if(bShowProgress) {
+		lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i);
+		write(1, msgBuf, lenMsg);
+	}
 
 }
 
@@ -218,12 +334,11 @@ void closeConnections(void)
  * of constructing test messages. -- rgerhards, 2010-03-31
  */
 static inline void
-genMsg(char *buf, size_t maxBuf, int *pLenBuf)
+genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst)
 {
 	int edLen; /* actual extra data length to use */
 	char extraData[MAX_EXTRADATA_LEN + 1];
 	char dynFileIDBuf[128] = "";
-	static int numMsgsGen = 0;
 	int done;
 
 	if(dataFP != NULL) {
@@ -262,11 +377,9 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf)
 		/* use fixed message format from command line */
 		*pLenBuf = snprintf(buf, maxBuf, "%s\n", MsgToSend);
 	}
+	++inst->numSent;
 
-	if(numMsgsGen++ >= numMsgsToSend)
-		*pLenBuf = 0; /* indicate end of run */
-
-finalize_it: ;
+finalize_it: /*EMPTY to keep the compiler happy */;
 }
 
 /* send messages to the tcp connections we keep open. We use
@@ -277,50 +390,72 @@ finalize_it: ;
  * last. All messages in between are sent over random connections.
  * Note that message numbers start at 0.
  */
-int sendMessages(void)
+int sendMessages(struct instdata *inst)
 {
-	int i = 0;
+	unsigned i = 0;
 	int socknum;
 	int lenBuf;
-	int lenSend;
-	char *statusText;
+	int lenSend = 0;
+	char *statusText = "";
 	char buf[MAX_EXTRADATA_LEN + 1024];
+	char sendBuf[MAX_SENDBUF];
+	int offsSendBuf = 0;
 
-	if(dataFile == NULL) {
-		printf("Sending %d messages.\n", numMsgsToSend);
-		statusText = "messages";
-	} else {
-		printf("Sending file '%s' %d times.\n", dataFile, numFileIterations);
-		statusText = "kb";
+	if(!bSilent) {
+		if(dataFile == NULL) {
+			printf("Sending %llu messages.\n", inst->numMsgs);
+			statusText = "messages";
+		} else {
+			printf("Sending file '%s' %d times.\n", dataFile,
+			       numFileIterations);
+			statusText = "kb";
+		}
 	}
 	if(bShowProgress)
 		printf("\r%8.8d %s sent", 0, statusText);
-	while(1) { /* broken inside loop! */
-		if(i < numConnections)
-			socknum = i;
-		else if(i >= numMsgsToSend - numConnections)
-			socknum = i - (numMsgsToSend - numConnections);
-		else {
-			int rnd = rand();
-			socknum = rnd % numConnections;
+	while(i < inst->numMsgs) {
+		if(runMultithreaded) {
+			socknum = inst->idx;
+		} else {
+			if(i < numConnections)
+				socknum = i;
+			else if(i >= inst->numMsgs - numConnections) {
+				socknum = i - (inst->numMsgs - numConnections);
+			} else {
+				int rnd = rand();
+				socknum = rnd % numConnections;
+			}
 		}
-		genMsg(buf, sizeof(buf), &lenBuf); /* generate the message to send according to params */
-		if(lenBuf == 0)
-			break; /* end of processing! */
-		if(sockArray[socknum] == -1) {
-			/* connection was dropped, need to re-establish */
-			if(openConn(&(sockArray[socknum])) != 0) {
-				printf("error in trying to re-open connection %d\n", socknum);
-				exit(1);
+		genMsg(buf, sizeof(buf), &lenBuf, inst); /* generate the message to send according to params */
+		if(transport == TP_TCP) {
+			if(sockArray[socknum] == -1) {
+				/* connection was dropped, need to re-establish */
+				if(openConn(&(sockArray[socknum])) != 0) {
+					printf("error in trying to re-open connection %d\n", socknum);
+					exit(1);
+				}
+			}
+			lenSend = send(sockArray[socknum], buf, lenBuf, 0);
+		} else if(transport == TP_UDP) {
+			lenSend = sendto(udpsock, buf, lenBuf, 0, &udpRcvr, sizeof(udpRcvr));
+		} else if(transport == TP_TLS) {
+			if(offsSendBuf + lenBuf < MAX_SENDBUF) {
+				memcpy(sendBuf+offsSendBuf, buf, lenBuf);
+				offsSendBuf += lenBuf;
+				lenSend = lenBuf; /* simulate "good" call */
+			} else {
+				lenSend = sendTLS(socknum, sendBuf, offsSendBuf);
+				lenSend = (lenSend == offsSendBuf) ? lenBuf : -1;
+				memcpy(sendBuf, buf, lenBuf);
+				offsSendBuf = lenBuf;
 			}
 		}
-		lenSend = send(sockArray[socknum], buf, lenBuf, 0);
 		if(lenSend != lenBuf) {
 			printf("\r%5.5d\n", i);
 			fflush(stdout);
 			perror("send test data");
-			printf("send() failed at socket %d, index %d, msgNum %d\n",
-				sockArray[socknum], i, msgNum);
+			printf("send() failed at socket %d, index %d, msgNum %lld\n",
+				sockArray[socknum], i, inst->numSent);
 			fflush(stderr);
 			return(1);
 		}
@@ -328,7 +463,7 @@ int sendMessages(void)
 			if(bShowProgress)
 				printf("\r%8.8d", i);
 		}
-		if(bRandConnDrop) {
+		if(!runMultithreaded && bRandConnDrop) {
 			/* if we need to randomly drop connections, see if we 
 			 * are a victim
 			 */
@@ -338,14 +473,333 @@ int sendMessages(void)
 				sockArray[socknum] = -1;
 			}
 		}
+		if(inst->numSent % batchsize == 0) {
+			usleep(waittime);
+		}
 		++msgNum;
 		++i;
 	}
-	printf("\r%8.8d %s sent\n", i, statusText);
+	if(transport == TP_TLS && offsSendBuf != 0) {
+		/* send remaining buffer */
+		lenSend = sendTLS(socknum, sendBuf, offsSendBuf);
+	}
+	if(!bSilent)
+		printf("\r%8.8d %s sent\n", i, statusText);
+
+	return 0;
+}
+
+
+/* this is the thread that starts a generator
+ */
+static void *
+thrdStarter(void *arg)
+{
+	struct instdata *inst = (struct instdata*) arg;
+	pthread_mutex_lock(&thrdMgmt);
+	runningThreads++;
+	pthread_cond_signal(&condStarted);
+	while(doRun == 0) {
+		pthread_cond_wait(&condDoRun, &thrdMgmt);
+	}
+	pthread_mutex_unlock(&thrdMgmt);
+	if(sendMessages(inst) != 0) {
+		printf("error sending messages\n");
+	}
+	return NULL;
+}
+
+
+/* This function initializes the actual traffic generators. The function sets up all required
+ * parameter blocks and starts threads. It returns when all threads are ready to run
+ * and the main task must just enable them.
+ */
+static inline void
+prepareGenerators()
+{
+	int i;
+	long long msgsThrd;
+	long long starting = 0;
+	
+	if(runMultithreaded) {
+		bSilent = 1;
+		numThrds = numConnections;
+	} else {
+		numThrds = 1;
+	}
+
+	runningThreads = 0;
+	doRun = 0;
+	pthread_mutex_init(&thrdMgmt, NULL);
+	pthread_cond_init(&condStarted, NULL);
+	pthread_cond_init(&condDoRun, NULL);
+
+	if(instarray != NULL) {
+		free(instarray);
+	}
+	instarray = calloc(numThrds, sizeof(struct instdata));
+	msgsThrd = numMsgsToSend / numThrds;
+
+	for(i = 0 ; i < numThrds ; ++i)  {
+		instarray[i].lower = starting;
+		instarray[i].numMsgs = msgsThrd;
+		instarray[i].numSent = 0;
+		instarray[i].idx = i;
+		pthread_create(&(instarray[i].thread), NULL, thrdStarter, instarray + i); 
+		/*printf("started thread %x\n", (unsigned) instarray[i].thread);*/
+		starting += msgsThrd;
+	}
+}
+
+/* Let all generators run. Threads must have been started. Here we wait until
+ * all threads are initialized and then broadcast that they can begin to run.
+ */
+static inline void
+runGenerators()
+{
+	pthread_mutex_lock(&thrdMgmt);
+	while(runningThreads != numThrds){
+		pthread_cond_wait(&condStarted, &thrdMgmt);
+	}
+	doRun = 1;
+	pthread_cond_broadcast(&condDoRun);
+	pthread_mutex_unlock(&thrdMgmt);
+}
+
+
+/* Wait for all traffic generators to stop.
+ */
+static inline void
+waitGenerators()
+{
+	int i;
+	for(i = 0 ; i < numThrds ; ++i)  {
+		pthread_join(instarray[i].thread, NULL);
+		/*printf("thread %x stopped\n", (unsigned) instarray[i].thread);*/
+	}
+	pthread_mutex_destroy(&thrdMgmt);
+	pthread_cond_destroy(&condStarted);
+	pthread_cond_destroy(&condDoRun);
+}
+
+/* functions related to computing statistics on the runtime of a test. This is
+ * a separate function primarily not to mess up the test driver.
+ * rgerhards, 2010-12-08
+ */
+static inline void
+endTiming(struct timeval *tvStart, struct runstats *stats)
+{
+	long sec, usec;
+	unsigned long runtime;
+	struct timeval tvEnd;
+
+	gettimeofday(&tvEnd, NULL);
+	if(tvStart->tv_usec > tvEnd.tv_usec) {
+		tvEnd.tv_sec--;
+		tvEnd.tv_usec += 1000000;
+	}
+
+	sec = tvEnd.tv_sec - tvStart->tv_sec;
+	usec = tvEnd.tv_usec - tvStart->tv_usec;
+
+	runtime = sec * 1000 + (usec / 1000);
+	stats->totalRuntime += runtime;
+	if(runtime < stats->minRuntime)
+		stats->minRuntime = runtime;
+	if(runtime > stats->maxRuntime)
+		stats->maxRuntime = runtime;
+
+	if(!bSilent || bStatsRecords) {
+		if(bCSVoutput) {
+			printf("%ld.%3.3ld\n", runtime / 1000, runtime % 1000);
+		} else {
+			printf("runtime: %ld.%3.3ld\n", runtime / 1000, runtime % 1000);
+		}
+	}
+}
+
+
+/* generate stats summary record at end of run
+ */
+static inline void
+genStats(struct runstats *stats)
+{
+	long unsigned avg;
+	avg = stats->totalRuntime / stats->numRuns;
+
+	if(bCSVoutput) {
+		printf("#numRuns,TotalRuntime,AvgRuntime,MinRuntime,MaxRuntime\n");
+		printf("%d,%llu.%3.3d,%lu.%3.3lu,%lu.%3.3lu,%lu.%3.3lu\n",
+			stats->numRuns,
+		        stats->totalRuntime / 1000, (int) stats->totalRuntime % 1000,
+		        avg / 1000, avg % 1000,
+		        stats->minRuntime / 1000, stats->minRuntime % 1000,
+		        stats->maxRuntime / 1000, stats->maxRuntime % 1000);
+	} else {
+		printf("Runs:     %d\n",   stats->numRuns);
+		printf("Runtime:\n");
+		printf("  total:  %llu.%3.3d\n", stats->totalRuntime / 1000,
+						 (int) stats->totalRuntime % 1000);
+		printf("  avg:    %lu.%3.3lu\n",  avg / 1000, avg % 1000);
+		printf("  min:    %lu.%3.3lu\n",  stats->minRuntime / 1000, stats->minRuntime % 1000);
+		printf("  max:    %lu.%3.3lu\n",  stats->maxRuntime / 1000, stats->maxRuntime % 1000);
+		printf("All times are wallclock time.\n");
+	}
+}
+
+
+/* Run the actual test. This function handles various meta-parameters, like
+ * a specified number of iterations, performance measurement and so on...
+ * rgerhards, 2010-12-08
+ */
+static int
+runTests(void)
+{
+	struct timeval tvStart;
+	struct runstats stats;
+	int run;
+
+	stats.totalRuntime = 0;
+	stats.minRuntime = (unsigned long long) 0xffffffffffffffffll;
+	stats.maxRuntime = 0;
+	stats.numRuns = numRuns;
+	run = 1;
+	while(1) { /* loop broken inside */
+		if(!bSilent)
+			printf("starting run %d\n", run);
+		prepareGenerators();
+		gettimeofday(&tvStart, NULL);
+		runGenerators();
+		waitGenerators();
+		endTiming(&tvStart, &stats);
+		if(run == numRuns)
+			break;
+		if(!bSilent)
+			printf("sleeping %d seconds before next run\n", sleepBetweenRuns);
+		sleep(sleepBetweenRuns);
+		++run;
+	}
+
+	if(bStatsRecords) {
+		genStats(&stats);
+	}
 
 	return 0;
 }
 
+#	if defined(ENABLE_GNUTLS)
+/* This defines a log function to be provided to GnuTLS. It hopefully
+ * helps us track down hard to find problems.
+ * rgerhards, 2008-06-20
+ */
+static void tlsLogFunction(int level, const char *msg)
+{
+	printf("GnuTLS (level %d): %s", level, msg);
+
+}
+
+
+/* global init GnuTLS
+ */
+static void
+initTLS(void)
+{
+	int r;
+
+	/* order of gcry_control and gnutls_global_init matters! */
+	gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
+	gnutls_global_init();
+	/* set debug mode, if so required by the options */
+	if(tlsLogLevel > 0) {
+		gnutls_global_set_log_function(tlsLogFunction);
+		gnutls_global_set_log_level(tlsLogLevel);
+	}
+
+	r = gnutls_certificate_allocate_credentials(&tlscred);
+	if(r != GNUTLS_E_SUCCESS) {
+		printf("error allocating credentials\n");
+		gnutls_perror(r);
+		exit(1);
+	}
+	r = gnutls_certificate_set_x509_key_file(tlscred, tlsCertFile, tlsKeyFile, GNUTLS_X509_FMT_PEM);
+	if(r != GNUTLS_E_SUCCESS) {
+		printf("error setting certificate files -- have you mixed up key and certificate?\n");
+		printf("If in doubt, try swapping the files in -z/-Z\n");
+		printf("Certifcate is: '%s'\n", tlsCertFile);
+		printf("Key        is: '%s'\n", tlsKeyFile);
+		gnutls_perror(r);
+		r = gnutls_certificate_set_x509_key_file(tlscred, tlsKeyFile, tlsCertFile,
+							 GNUTLS_X509_FMT_PEM);
+		if(r == GNUTLS_E_SUCCESS) {
+			printf("Tried swapping files, this seems to work "
+			       "(but results may be unpredictable!)\n");
+		} else {
+			exit(1);
+		}
+	}
+}
+
+
+static void
+initTLSSess(int i)
+{
+	int r;
+	gnutls_init(sessArray + i, GNUTLS_CLIENT);
+
+	/* Use default priorities */
+	gnutls_set_default_priority(sessArray[i]);
+
+	/* put our credentials to the current session */
+	r = gnutls_credentials_set(sessArray[i], GNUTLS_CRD_CERTIFICATE, tlscred);
+	if(r != GNUTLS_E_SUCCESS) {
+		fprintf (stderr, "Setting credentials failed\n");
+		gnutls_perror(r);
+		exit(1);
+	}
+
+	/* NOTE: the following statement generates a cast warning, but there seems to
+	 * be no way around it with current GnuTLS. Do NOT try to "fix" the situation!
+	 */
+	gnutls_transport_set_ptr(sessArray[i], (gnutls_transport_ptr_t) sockArray[i]);
+
+	/* Perform the TLS handshake */
+	r = gnutls_handshake(sessArray[i]);
+	if(r < 0) {
+		fprintf (stderr, "TLS Handshake failed\n");
+		gnutls_perror(r);
+		exit(1);
+	}
+}
+
+static int
+sendTLS(int i, char *buf, int lenBuf)
+{
+	int lenSent;
+	int r;
+
+	lenSent = 0;
+	while(lenSent != lenBuf) {
+		r = gnutls_record_send(sessArray[i], buf + lenSent, lenBuf - lenSent);
+		if(r < 0)
+			break;
+		lenSent += r;
+	}
+
+	return lenSent;
+}
+
+static void
+closeTLSSess(int i)
+{
+	gnutls_bye(sessArray[i], GNUTLS_SHUT_RDWR);
+	gnutls_deinit(sessArray[i]);
+}
+#	else	/* NO TLS available */
+static void initTLS(void) {}
+static void initTLSSess(int __attribute__((unused)) i) {}
+static int sendTLS(int i, char *buf, int lenBuf) { return 0; }
+static void closeTLSSess(int __attribute__((unused)) i) {}
+#	endif
 
 /* Run the test.
  * rgerhards, 2009-04-03
@@ -370,18 +824,17 @@ int main(int argc, char *argv[])
 
 	setvbuf(stdout, buf, _IONBF, 48);
 	
-	if(!isatty(1))
-		bShowProgress = 0;
-
-	while((opt = getopt(argc, argv, "f:F:t:p:c:C:m:i:I:P:d:Dn:M:rB")) != -1) {
+	while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:L:M:rsBR:S:T:XW:Yz:Z:")) != -1) {
 		switch (opt) {
+		case 'b':	batchsize = atoll(optarg);
+				break;
 		case 't':	targetIP = optarg;
 				break;
 		case 'p':	targetPort = atoi(optarg);
 				break;
 		case 'n':	numTargetPorts = atoi(optarg);
 				break;
-		case 'c':	numConnections = atoi(optarg);
+		case 'c':	numConnections = (unsigned) atoi(optarg);
 				break;
 		case 'C':	numFileIterations = atoi(optarg);
 				break;
@@ -406,6 +859,8 @@ int main(int argc, char *argv[])
 				break;
 		case 'F':	frameDelim = atoi(optarg);
 				break;
+		case 'L':	tlsLogLevel = atoi(optarg);
+				break;
 		case 'M':	MsgToSend = optarg;
 				break;
 		case 'I':	dataFile = optarg;
@@ -414,20 +869,61 @@ int main(int argc, char *argv[])
 				 */
 				numMsgsToSend = 1000000;
 				break;
+		case 's':	bSilent = 1;
+				break;
 		case 'B':	bBinaryFile = 1;
 				break;
+		case 'R':	numRuns = atoi(optarg);
+				break;
+		case 'S':	sleepBetweenRuns = atoi(optarg);
+				break;
+		case 'X':	bStatsRecords = 1;
+				break;
+		case 'e':	bCSVoutput = 1;
+				break;
+		case 'T':	if(!strcmp(optarg, "udp")) {
+					transport = TP_UDP;
+				} else if(!strcmp(optarg, "tcp")) {
+					transport = TP_TCP;
+				} else if(!strcmp(optarg, "tls")) {
+#					if defined(ENABLE_GNUTLS)
+						transport = TP_TLS;
+#					else
+						fprintf(stderr, "compiled without TLS support!\n", optarg);
+						exit(1);
+#					endif
+				} else {
+					fprintf(stderr, "unknown transport '%s'\n", optarg);
+					exit(1);
+				}
+				break;
+		case 'W':	waittime = atoi(optarg);
+				break;
+		case 'Y':	runMultithreaded = 1;
+				break;
+		case 'z':	tlsKeyFile = optarg;
+				break;
+		case 'Z':	tlsCertFile = optarg;
+				break;
 		default:	printf("invalid option '%c' or value missing - terminating...\n", opt);
 				exit (1);
 				break;
 		}
 	}
 
+	if(bStatsRecords && waittime) {
+		fprintf(stderr, "warning: generating performance stats and using a waittime "
+				"is somewhat contradictory!\n");
+	}
+
+	if(!isatty(1) || bSilent)
+		bShowProgress = 0;
+
 	if(numConnections > 20) {
 		/* if we use many (whatever this means, 20 is randomly picked)
 		 * connections, we need to make sure we have a high enough
 		 * limit. -- rgerhards, 2010-03-25
 		 */
-		struct rlimit maxFiles;
 		maxFiles.rlim_cur = numConnections + 20;
 		maxFiles.rlim_max = numConnections + 20;
 		if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) {
@@ -446,22 +942,27 @@ int main(int argc, char *argv[])
 		}
 	}
 
+	if(transport == TP_TLS) {
+		initTLS();
+	}
+
 	if(openConnections() != 0) {
 		printf("error opening connections\n");
 		exit(1);
 	}
 
-	if(sendMessages() != 0) {
-		printf("error sending messages\n");
+	if(runTests() != 0) {
+		printf("error running tests\n");
 		exit(1);
 	}
 
 	closeConnections(); /* this is important so that we do not finish too early! */
 
-	if(nConnDrops > 0)
+	if(nConnDrops > 0 && !bSilent)
 		printf("-D option initiated %ld connection closures\n", nConnDrops);
 
-	printf("End of tcpflood Run\n");
+	if(!bSilent)
+		printf("End of tcpflood Run\n");
 
 	exit(ret);
 }
-- 
cgit 


From f50f24a7750bc3ad66b339ed3fcc0fdc544a1e15 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 14 Jun 2011 12:18:39 +0200
Subject: backporting TLS-based test

---
 tests/Makefile.am                         |  6 ++++++
 tests/manytcp-too-few-tls.sh              | 15 +++++++++++++++
 tests/testsuites/manytcp-too-few-tls.conf | 22 ++++++++++++++++++++++
 3 files changed, 43 insertions(+)
 create mode 100755 tests/manytcp-too-few-tls.sh
 create mode 100644 tests/testsuites/manytcp-too-few-tls.conf

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 8e178ab8..3ee69413 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -96,6 +96,12 @@ TESTS +=  \
 	imptcp_conndrop.sh 
 endif
 
+if ENABLE_GNUTLS
+if HAVE_VALGRIND
+TESTS += manytcp-too-few-tls.sh
+endif
+endif
+
 if ENABLE_OMUXSOCK
 TESTS += uxsock_simple.sh
 endif
diff --git a/tests/manytcp-too-few-tls.sh b/tests/manytcp-too-few-tls.sh
new file mode 100755
index 00000000..899a87dc
--- /dev/null
+++ b/tests/manytcp-too-few-tls.sh
@@ -0,0 +1,15 @@
+# test many concurrent tcp connections
+echo \[manytcp-too-few-tls.sh\]: test concurrent tcp connections
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup-vg manytcp-too-few-tls.conf
+echo wait for DH param generation -- NOT needed in v6!
+sleep 15
+# the config file specifies exactly 1100 connections
+source $srcdir/diag.sh tcpflood -c1000 -m40000
+# the sleep below is needed to prevent too-early termination of the tcp listener
+sleep 1
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown-vg	# we need to wait until rsyslogd is finished!
+source $srcdir/diag.sh check-exit-vg
+source $srcdir/diag.sh seq-check 0 39999
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/manytcp-too-few-tls.conf b/tests/testsuites/manytcp-too-few-tls.conf
new file mode 100644
index 00000000..5269e73b
--- /dev/null
+++ b/tests/testsuites/manytcp-too-few-tls.conf
@@ -0,0 +1,22 @@
+# Test for tcp "flood" testing
+# rgerhards, 2009-04-08
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+$MainMsgQueueTimeoutShutdown 10000
+$MaxOpenFiles 200
+$InputTCPMaxSessions 1100
+# certificates
+$DefaultNetstreamDriverCAFile testsuites/x.509/ca.pem
+$DefaultNetstreamDriverCertFile testsuites/x.509/client-cert.pem
+$DefaultNetstreamDriverKeyFile testsuites/x.509/client-key.pem
+
+$DefaultNetstreamDriver gtls # use gtls netstream driver
+
+$InputTCPServerStreamDriverMode 1
+$InputTCPServerStreamDriverAuthMode anon
+$InputTCPServerRun 13514
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+:msg, contains, "msgnum:" ?dynfile;outfmt
-- 
cgit 


From fdc25fb14b6acc1484a59f55746bd4041e0103ff Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 14 Jun 2011 12:23:33 +0200
Subject: bugfix: TLS-induced smaller memory still existed, now fixed

---
 runtime/nsd_gtls.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c
index 152dc8de..ca4b2928 100644
--- a/runtime/nsd_gtls.c
+++ b/runtime/nsd_gtls.c
@@ -1121,6 +1121,7 @@ gtlsEndSess(nsd_gtls_t *pThis)
 			}
 		}
 		gnutls_deinit(pThis->sess);
+		pThis->bHaveSess = 0;
 	}
 	RETiRet;
 }
@@ -1174,6 +1175,8 @@ CODESTARTobjDestruct(nsd_gtls)
 		gnutls_x509_crt_deinit(pThis->ourCert);
 	if(pThis->bOurKeyIsInit)
 		gnutls_x509_privkey_deinit(pThis->ourKey);
+	if(pThis->bHaveSess)
+		gnutls_deinit(pThis->sess);
 ENDobjDestruct(nsd_gtls)
 
 
-- 
cgit 


From bcd956d4d500040808b920e468529da94a1e33c8 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 14 Jun 2011 12:52:02 +0200
Subject: bugfix: memory leak in imtcp & subsystems under some circumstances

This leak is tied to error conditions which lead to incorrect cleanup
of some data structures. [backport from v6, limited testing under v4]
---
 ChangeLog          |  3 +++
 runtime/nsd_gtls.c |  3 +++
 tcpsrv.c           | 10 ++++++----
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9bc775a4..39a66f5d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,8 @@
 ---------------------------------------------------------------------------
 Version 4.6.6  [v4-stable] (rgerhards), 2010-11-??
+- bugfix: memory leak in imtcp & subsystems under some circumstances
+  This leak is tied to error conditions which lead to incorrect cleanup
+  of some data structures. [backport from v6, limited testing under v4]
 - bugfix: invalid processing in QUEUE_FULL condition
   If the the multi-submit interface was used and a QUEUE_FULL condition
   occured, the failed message was properly destructed. However, the
diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c
index fb2e219d..e1dcf870 100644
--- a/runtime/nsd_gtls.c
+++ b/runtime/nsd_gtls.c
@@ -1118,6 +1118,7 @@ gtlsEndSess(nsd_gtls_t *pThis)
 			}
 		}
 		gnutls_deinit(pThis->sess);
+		pThis->bHaveSess = 0;
 	}
 	RETiRet;
 }
@@ -1171,6 +1172,8 @@ CODESTARTobjDestruct(nsd_gtls)
 		gnutls_x509_crt_deinit(pThis->ourCert);
 	if(pThis->bOurKeyIsInit)
 		gnutls_x509_privkey_deinit(pThis->ourKey);
+	if(pThis->bHaveSess)
+		gnutls_deinit(pThis->sess);
 ENDobjDestruct(nsd_gtls)
 
 
diff --git a/tcpsrv.c b/tcpsrv.c
index 0581828e..8c992f79 100644
--- a/tcpsrv.c
+++ b/tcpsrv.c
@@ -477,7 +477,7 @@ Run(tcpsrv_t *pThis)
 	int iTCPSess;
 	int bIsReady;
 	tcps_sess_t *pNewSess;
-	nssel_t *pSel;
+	nssel_t *pSel = NULL;
 	ssize_t iRcvd;
 
 	ISOBJ_TYPE_assert(pThis, tcpsrv);
@@ -521,8 +521,8 @@ Run(tcpsrv_t *pThis)
 		/* now check the sessions */
 		iTCPSess = TCPSessGetNxtSess(pThis, -1);
 		while(nfds && iTCPSess != -1) {
-			CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds));
-			if(bIsReady) {
+			localRet = nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds);
+			if(bIsReady || localRet != RS_RET_OK) {
 				char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */
 				dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm);
 
@@ -576,7 +576,9 @@ finalize_it: /* this is a very special case - this time only we do not exit the
 	      * crashed, which made sense (the rest of the engine was not prepared for
 	      * that) -- rgerhards, 2008-05-19
 	      */
-		/*EMPTY*/;
+		if(pSel != NULL) { /* cleanup missing? happens during err exit! */
+			nssel.Destruct(&pSel);
+		}
 	}
 
 	/* note that this point is usually not reached */
-- 
cgit 


From b1a905c5eb7833d661be4a910697f49deb34c640 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 14 Jun 2011 15:45:52 +0200
Subject: preparing for 6.1.9 release

---
 ChangeLog       | 4 ++--
 configure.ac    | 2 +-
 doc/manual.html | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d34ab025..cd7890d8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,8 @@
 ---------------------------------------------------------------------------
-Version 6.1.9  [BETA] (rgerhards), 2011-0?-??
+Version 6.1.9  [BETA] (rgerhards), 2011-06-14
 - bugfix: memory leak in imtcp & subsystems under some circumstances
   This leak is tied to error conditions which lead to incorrect cleanup
-  of some data structures. [backport from v6].3
+  of some data structures. [backport from v6.3]
 - bugfix: $ActionFileDefaultTemplate did not work
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=262
 ---------------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index f6a09fa7..a7c7cf42 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[6.1.8],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[6.1.9],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
diff --git a/doc/manual.html b/doc/manual.html
index 798680ba..6a2a401d 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 6.1.8 (devel branch) of rsyslog.</b>
+<p><b>This documentation is for version 6.1.9 (devel branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
-- 
cgit 


From a201de795ad630ddf81fea9f7bbb13ad79bdfff4 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 15 Jun 2011 12:02:39 +0200
Subject: doc: clarified requirements for $WorkDirectory directive

---
 doc/rsyslog_conf_global.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html
index ce46bac2..9e85ae1e 100644
--- a/doc/rsyslog_conf_global.html
+++ b/doc/rsyslog_conf_global.html
@@ -249,7 +249,8 @@ default may change as uniprocessor systems become less common. [available since
 <li>$PreserveFQDN [on/<b>off</b>) - if set to off (legacy default to remain compatible
 to sysklogd), the domain part from a name that is within the same domain as the receiving
 system is stripped. If set to on, full names are always used.</li>
-<li>$WorkDirectory &lt;name&gt; (directory for spool and other work files)</li>
+<li>$WorkDirectory &lt;name&gt; (directory for spool and other work files.
+Do <b>not</b> use trailing slashes)</li>
 <li>$UDPServerAddress &lt;IP&gt; (imudp) -- local IP
 address (or name) the UDP listens should bind to</li>
 <li>$UDPServerRun &lt;port&gt; (imudp) -- former
-- 
cgit 


From 41d8672524b93aaf7b7835336ac9b92428b90fed Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 15 Jun 2011 12:20:12 +0200
Subject: bugfix/improvement:$WorkDirectory now gracefully handles trailing
 slashes

---
 ChangeLog         |  1 +
 runtime/glbl.c    | 23 ++++++++++++++++++++++-
 runtime/rsyslog.h |  1 +
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 8867e04e..042aeae3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,7 @@ Version 5.8.2  [V5-stable] (rgerhards), 2011-06-??
 - bugfix: memory leak in imtcp & subsystems under some circumstances
   This leak is tied to error conditions which lead to incorrect cleanup
   of some data structures. [backport from v6]
+- bugfix/improvement:$WorkDirectory now gracefully handles trailing slashes
 ---------------------------------------------------------------------------
 Version 5.8.1  [V5-stable] (rgerhards), 2011-05-19
 - bugfix: invalid processing in QUEUE_FULL condition
diff --git a/runtime/glbl.c b/runtime/glbl.c
index ec4992cf..dea5a17b 100644
--- a/runtime/glbl.c
+++ b/runtime/glbl.c
@@ -159,8 +159,29 @@ static void SetGlobalInputTermination(void)
  */
 static rsRetVal setWorkDir(void __attribute__((unused)) *pVal, uchar *pNewVal)
 {
-	DEFiRet;
+	size_t lenDir;
+	int i;
 	struct stat sb;
+	DEFiRet;
+
+	/* remove trailing slashes */
+	lenDir = ustrlen(pNewVal);
+	i = lenDir - 1;
+	while(i > 0 && pNewVal[i] == '/') {
+		--i;
+	}
+
+	if(i < 0) {
+		errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: empty value "
+				"- directive ignored");
+		ABORT_FINALIZE(RS_RET_ERR_WRKDIR);
+	}
+
+	if(i != (int) lenDir - 1) {
+		pNewVal[i+1] = '\0';
+		errmsg.LogError(0, RS_RET_WRN_WRKDIR, "$WorkDirectory: trailing slashes "
+			"removed, new value is '%s'", pNewVal);
+	}
 
 	if(stat((char*) pNewVal, &sb) != 0) {
 		errmsg.LogError(0, RS_RET_ERR_WRKDIR, "$WorkDirectory: %s can not be "
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index d63dbe4f..52b29ac4 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -342,6 +342,7 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_ERR_HDFS_OPEN = -2179, /**< error during hdfsOpen (e.g. file does not exist) */
 	RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */
 	RS_RET_ERR_WRKDIR = -2181, /**< problems with the rsyslog working directory */
+	RS_RET_WRN_WRKDIR = -2182, /**< correctable problems with the rsyslog working directory */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
-- 
cgit 


From 90f8c7300495da9b61e4706be652c612fccc084f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 16 Jun 2011 15:17:48 +0200
Subject: bugfix: problems in failover action handling

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270 (not yet confirmed!)
---
 ChangeLog       |   2 +
 action.c        | 140 +++++++++++++++++++++++++++++++++++++-------------------
 action.h        |   2 +-
 runtime/rule.c  |  12 +++--
 tools/syslogd.c |   2 +-
 5 files changed, 106 insertions(+), 52 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 042aeae3..0d7bf02a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.8.2  [V5-stable] (rgerhards), 2011-06-??
+- bugfix: problems in failover action handling
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270 (not yet confirmed!)
 - bugfix: memory leak in imtcp & subsystems under some circumstances
   This leak is tied to error conditions which lead to incorrect cleanup
   of some data structures. [backport from v6]
diff --git a/action.c b/action.c
index c5bd03cb..b1986c38 100644
--- a/action.c
+++ b/action.c
@@ -39,7 +39,35 @@
  * - processAction
  * - submitBatch
  * - tryDoAction
- * - 
+ * - ...
+ *
+ * MORE ON PROCESSING, QUEUES and FILTERING
+ * All filtering needs to be done BEFORE messages are enqueued to an
+ * action. In previous code, part of the filtering was done at the
+ * "remote end" of the action queue, which lead to problems in
+ * non-direct mode (because then things run asynchronously). In order
+ * to solve this problem once and for all, I have changed the code so
+ * that all filtering is done before enq, and processing on the
+ * dequeue side of action processing now always executes whatever is
+ * enqueued. This is the only way to handle things consistently and
+ * (as much as possible) in a queue-type agnostic way. However, it is
+ * a rather radical change, which I unfortunately needed to make from
+ * stable version 5.8.1 to 5.8.2. If new problems pop up, you now know
+ * what may be their cause. In any case, the way it is done now is the
+ * only correct one.
+ * A problem is that, under fortunate conditions, we use the current
+ * batch for the output system as well. This is very good from a performance
+ * point of view, but makes the distinction between enq and deq side of
+ * the queue a bit hard. The current idea is that the filter condition
+ * alone is checked at the deq side of the queue (seems to be unavoidable
+ * to do it that way), but all other complex conditons (like failover
+ * handling) go into the computation of the filter condition. For
+ * non-direct queues, we still enqueue only what is acutally necessary.
+ * Note that in this case the rest of the code must ensure that the filter
+ * is set to "true". While this is not perfect and not as simple as
+ * we would like to see it, it looks like the best way to tackle that
+ * beast.
+ * rgerhards, 2011-06-15
  *
  * Copyright 2007-2011 Rainer Gerhards and Adiscon GmbH.
  *
@@ -611,8 +639,8 @@ static rsRetVal actionTryResume(action_t *pThis, int *pbShutdownImmediate)
 	}
 
 	if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) {
-		DBGPRINTF("actionTryResume: action state: %s, next retry (if applicable): %u [now %u]\n",
-			getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow);
+		DBGPRINTF("actionTryResume: action %p state: %s, next retry (if applicable): %u [now %u]\n",
+			pThis, getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow);
 	}
 
 finalize_it:
@@ -932,16 +960,19 @@ tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem)
 	i = pBatch->iDoneUpTo;	/* all messages below that index are processed */
 	iElemProcessed = 0;
 	iCommittedUpTo = i;
+dbgprintf("XXXXX:     tryDoAction %p, pnElem %d, nElem %d\n", pAction, *pnElem, pBatch->nElem);
 	while(iElemProcessed <= *pnElem && i < pBatch->nElem) {
 		if(*(pBatch->pbShutdownImmediate))
 			ABORT_FINALIZE(RS_RET_FORCE_TERM);
+		/* NOTE: do NOT extend the filter below! Anything else must be done on the
+		 * enq side of the queue (see file header comment)! -- rgerhards, 2011-06-15
+		 */
 		if(   pBatch->pElem[i].bFilterOK
-		   && pBatch->pElem[i].state != BATCH_STATE_DISC//) {
-		   && ((pAction->bExecWhenPrevSusp  == 0) || pBatch->pElem[i].bPrevWasSuspended) ) {
+		    && pBatch->pElem[i].state != BATCH_STATE_DISC) {
 			pMsg = (msg_t*) pBatch->pElem[i].pUsrp;
 			localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams,
-						        pBatch->pbShutdownImmediate);
-			DBGPRINTF("action call returned %d\n", localRet);
+							pBatch->pbShutdownImmediate);
+			DBGPRINTF("action %p call returned %d\n", pAction, localRet);
 			/* Note: we directly modify the batch object state, because we know that
 			 * wo do not overwrite BATCH_STATE_DISC indicators!
 			 */
@@ -1035,6 +1066,8 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem)
 				bDone = 1;
 			} else {
 				/* retry with half as much. Depth is log_2 batchsize, so recursion is not too deep */
+				DBGPRINTF("submitBatch recursing trying to find and exclude the culprit "
+				          "for iRet %d\n", localRet);
 				submitBatch(pAction, pBatch, nElem / 2);
 				submitBatch(pAction, pBatch, nElem - (nElem / 2));
 				bDone = 1;
@@ -1224,11 +1257,13 @@ doSubmitToActionQ(action_t *pAction, msg_t *pMsg)
 
 /* This function builds up a batch of messages to be (later)
  * submitted to the action queue.
- * Note: this function is also called from syslogd itself as part of its
- * flush processing. If so, pBatch will be NULL and idxBtch undefined.
+ * Important: this function MUST not be called with messages that are to
+ * be discarded due to their "prevWasSuspended" state. It will not check for
+ * this and submit all messages to the queue for execution. So these must
+ * be filtered out before calling us (what is done currently!).
  */
 rsRetVal
-actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch)
+actionWriteToAction(action_t *pAction)
 {
 	msg_t *pMsgSave;	/* to save current message pointer, necessary to restore
 				   it in case it needs to be updated (e.g. repeated msgs) */
@@ -1325,35 +1360,7 @@ actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 	/* When we reach this point, we have a valid, non-disabled action.
 	 * So let's enqueue our message for execution. -- rgerhards, 2007-07-24
 	 */
-	if(   pBatch != NULL
-	   && (pAction->bExecWhenPrevSusp  == 1 && pBatch->pElem[idxBtch].bPrevWasSuspended)) {
-		/* in that case, we need to create a special batch which reflects the
-		 * suspended state. Otherwise, that information would be dropped inside
-		 * the queue engine. TODO: in later releases (v6?) create a better
-		 * solution than what we do here. However, for v5 this sounds much too
-		 * intrusive. -- rgerhardsm, 2011-03-16
-		 * (Code is copied over from queue.c and slightly modified)
-		 */
-		batch_t singleBatch;
-		batch_obj_t batchObj;
-		int i;
-		memset(&batchObj, 0, sizeof(batch_obj_t));
-		memset(&singleBatch, 0, sizeof(batch_t));
-		batchObj.state = BATCH_STATE_RDY;
-		batchObj.pUsrp = (obj_t*) pAction->f_pMsg;
-		batchObj.bPrevWasSuspended = 1;
-		batchObj.bFilterOK = 1;
-		singleBatch.nElem = 1; /* there always is only one in direct mode */
-		singleBatch.pElem = &batchObj;
-
-		iRet = qqueueEnqObjDirectBatch(pAction->pQueue, &singleBatch);
-
-		for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) {
-			free(batchObj.staticActStrings[i]);
-		}
-	} else { /* standard case, just submit */
-		iRet = doSubmitToActionQ(pAction, pAction->f_pMsg);
-	}
+	iRet = doSubmitToActionQ(pAction, pAction->f_pMsg);
 
 	if(iRet == RS_RET_OK)
 		pAction->f_prevcount = 0; /* message processed, so we start a new cycle */
@@ -1413,7 +1420,7 @@ doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 		 * isolated messages), but back off so we'll flush less often in the future.
 		 */
 		if(getActNow(pAction) > REPEATTIME(pAction)) {
-			iRet = actionWriteToAction(pAction, pBatch, idxBtch);
+			iRet = actionWriteToAction(pAction);
 			BACKOFF(pAction);
 		}
 	} else {/* new message, save it */
@@ -1422,7 +1429,7 @@ doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 		 */
 		if(pAction->f_pMsg != NULL) {
 			if(pAction->f_prevcount > 0)
-				actionWriteToAction(pAction, pBatch, idxBtch);
+				actionWriteToAction(pAction);
 				/* we do not care about iRet above - I think it's right but if we have
 				 * some troubles, you know where to look at ;) -- rgerhards, 2007-08-01
 				 */
@@ -1430,7 +1437,7 @@ doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 		}
 		pAction->f_pMsg = MsgAddRef(pMsg);
 		/* call the output driver */
-		iRet = actionWriteToAction(pAction, pBatch, idxBtch);
+		iRet = actionWriteToAction(pAction);
 	}
 
 finalize_it:
@@ -1528,16 +1535,51 @@ static rsRetVal
 doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch)
 {
 	int i;
+	sbool bNeedSubmit;
 	DEFiRet;
 
+	/* TODO
+	ich arbeite an dieser funktion, es müssen die verscheidenen modi geprüft werden. ausserdem
+	muss geschaut werden, in welche anderen funktionen die neue Funktionalität noch eingebaut
+	werden muss, bzw. ob man das an zentralerer stelle machen kann. Am besten die gesamte
+	filter evaluation nochmal druchgehen (also das füllen des arrays).
+	*/
+
 	DBGPRINTF("Called action(Batch), logging to %s\n", module.GetStateName(pAction->pMod));
-	if(pAction->pQueue->qType == QUEUETYPE_DIRECT)
-		iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
-	else {  /* in this case, we do single submits to the queue. 
+	/* if necessary, take care of failover cases. We do this by simply
+	 * changing the filter setting, which is perfectly legal.
+	 */
+	if(pAction->pQueue->qType == QUEUETYPE_DIRECT) {
+		/* note: for direct mode, we need to adjust the filter property. For non-direct
+		 * this is not necessary, because in that case we enqueue only what actually needs
+		 * to be processed.
+		 */
+		if(pAction->bExecWhenPrevSusp) {
+			bNeedSubmit = 0;
+			for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+				if(!pBatch->pElem[i].bPrevWasSuspended) {
+					DBGPRINTF("action enq stage: change bFilterOK to 0 due to "
+						  "failover case in elem %d\n", i);
+					pBatch->pElem[i].bFilterOK = 0;
+				}
+				if(pBatch->pElem[i].bFilterOK)
+					bNeedSubmit = 1;
+			}
+			if(bNeedSubmit) {
+				iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
+			} else {
+				DBGPRINTF("no need to submit batch, all bFilterOK==0\n");
+			}
+		} else {
+			iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
+		}
+	} else {  /* in this case, we do single submits to the queue. 
 		 * TODO: optimize this, we may do at least a multi-submit!
 		 */
 		for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
-			if(pBatch->pElem[i].bFilterOK) {
+			if(   pBatch->pElem[i].bFilterOK
+			   && pBatch->pElem[i].state != BATCH_STATE_DISC
+			   && (pAction->bExecWhenPrevSusp == 0 || pBatch->pElem[i].bPrevWasSuspended == 1)) {
 				doSubmitToActionQ(pAction, (msg_t*)(pBatch->pElem[i].pUsrp));
 			}
 		}
@@ -1558,8 +1600,12 @@ helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch)
 	int i;
 	DEFiRet;
 
-	DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod));
+	DBGPRINTF("Called action %p (complex case), logging to %s\n",
+		  pAction, module.GetStateName(pAction->pMod));
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+		DBGPRINTF("action %p: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n",
+		           pAction, pBatch->pElem[i].bFilterOK,  pBatch->pElem[i].state,
+			   pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended);
 		if(   pBatch->pElem[i].bFilterOK
                    && pBatch->pElem[i].state != BATCH_STATE_DISC
 		   && ((pAction->bExecWhenPrevSusp  == 0) || pBatch->pElem[i].bPrevWasSuspended) ) {
diff --git a/action.h b/action.h
index 0ab8062a..bae64d31 100644
--- a/action.h
+++ b/action.h
@@ -100,7 +100,7 @@ rsRetVal actionDestruct(action_t *pThis);
 rsRetVal actionDbgPrint(action_t *pThis);
 rsRetVal actionSetGlobalResumeInterval(int iNewVal);
 rsRetVal actionDoAction(action_t *pAction);
-rsRetVal actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch);
+rsRetVal actionWriteToAction(action_t *pAction);
 rsRetVal actionCallHUPHdlr(action_t *pAction);
 rsRetVal actionClassInit(void);
 rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended);
diff --git a/runtime/rule.c b/runtime/rule.c
index d5f18e71..19239d61 100644
--- a/runtime/rule.c
+++ b/runtime/rule.c
@@ -266,6 +266,7 @@ static rsRetVal
 processBatch(rule_t *pThis, batch_t *pBatch)
 {
 	int i;
+	rsRetVal localRet;
 	DEFiRet;
 
 	ISOBJ_TYPE_assert(pThis, rule);
@@ -273,9 +274,14 @@ processBatch(rule_t *pThis, batch_t *pBatch)
 
 	/* first check the filters and reset status variables */
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
-		CHKiRet(shouldProcessThisMessage(pThis, (msg_t*)(pBatch->pElem[i].pUsrp),
-						 &(pBatch->pElem[i].bFilterOK)));
-		// TODO: really abort on error? 2010-06-10
+		localRet = shouldProcessThisMessage(pThis, (msg_t*)(pBatch->pElem[i].pUsrp),
+						    &(pBatch->pElem[i].bFilterOK));
+		if(localRet != RS_RET_OK) {
+			DBGPRINTF("processBatch: iRet %d returned from shouldProcessThisMessage, "
+			          "ignoring message\n", localRet);
+			
+			pBatch->pElem[i].bFilterOK = 0;
+		}
 		if(pBatch->pElem[i].bFilterOK) {
 			/* re-init only when actually needed (cache write cost!) */
 			pBatch->pElem[i].bPrevWasSuspended = 0;
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 487ab364..096f9309 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -799,7 +799,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions)
 		DBGPRINTF("flush %s: repeated %d times, %d sec.\n",
 		    module.GetStateName(pAction->pMod), pAction->f_prevcount,
 		    repeatinterval[pAction->f_repeatcount]);
-		actionWriteToAction(pAction, NULL, 0);
+		actionWriteToAction(pAction);
 		BACKOFF(pAction);
 	}
 	UnlockObj(pAction);
-- 
cgit 


From da52cbae520e747568162ad558bf01d40658c745 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 16 Jun 2011 17:35:45 +0200
Subject: bugfix: timestamp was incorrectly calculated for timezones with
 minute offset

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=271
---
 ChangeLog          | 3 +++
 runtime/datetime.c | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index a50c4ee0..b547f0ea 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,8 @@
 ---------------------------------------------------------------------------
 Version 3.22.4 [v3-stable] (rgerhards), 2010-??-??
+- bugfix: timestamp was incorrectly calculated for timezones with minute
+  offset
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=271
 - improved some code based on clang static analyzer results
 ---------------------------------------------------------------------------
 Version 3.22.3 [v3-stable] (rgerhards), 2010-11-24
diff --git a/runtime/datetime.c b/runtime/datetime.c
index 20ca6191..bed33127 100644
--- a/runtime/datetime.c
+++ b/runtime/datetime.c
@@ -112,7 +112,7 @@ static void getCurrTime(struct syslogTime *t)
 	else
 		t->OffsetMode = '+';
 	t->OffsetHour = lBias / 3600;
-	t->OffsetMinute = lBias % 3600;
+	t->OffsetMinute = (lBias % 3600) / 60;
 }
 
 
-- 
cgit 


From 257b06aac8222cdea231a95cbbe659679a2d417e Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 16 Jun 2011 17:52:16 +0200
Subject: failover problem was not totally solved, now (hopefully ;))

I overlooked a border case, which came up on a larger testbench run.
---
 action.c                     | 86 ++++++++++++++++++++++++++++++--------------
 tests/manytcp-too-few-tls.sh |  6 +++-
 2 files changed, 64 insertions(+), 28 deletions(-)

diff --git a/action.c b/action.c
index b1986c38..7909d8e3 100644
--- a/action.c
+++ b/action.c
@@ -1526,6 +1526,63 @@ finalize_it:
 }
 
 
+/* enqueue a batch in direct mode. We have put this into its own function just to avoid
+ * cluttering the actual submit function.
+ * rgerhards, 2011-06-16
+ */
+static inline rsRetVal
+doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch)
+{
+	sbool FilterSave[1024];
+	sbool *pFilterSave;
+	sbool bNeedSubmit;
+	sbool bModifiedFilter;
+	int i;
+	DEFiRet;
+
+	if(batchNumMsgs(pBatch) <= (int) (sizeof(FilterSave)/sizeof(sbool))) {
+		pFilterSave = FilterSave;
+	} else {
+		CHKmalloc(pFilterSave = malloc(batchNumMsgs(pBatch) * sizeof(sbool)));
+	}
+
+	/* note: for direct mode, we need to adjust the filter property. For non-direct
+	 * this is not necessary, because in that case we enqueue only what actually needs
+	 * to be processed.
+	 */
+	if(pAction->bExecWhenPrevSusp) {
+		bNeedSubmit = 0;
+		bModifiedFilter = 0;
+		for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+			pFilterSave[i] = pBatch->pElem[i].bFilterOK;
+			if(!pBatch->pElem[i].bPrevWasSuspended) {
+				DBGPRINTF("action enq stage: change bFilterOK to 0 due to "
+					  "failover case in elem %d\n", i);
+				pBatch->pElem[i].bFilterOK = 0;
+				bModifiedFilter = 1;
+			}
+			if(pBatch->pElem[i].bFilterOK)
+				bNeedSubmit = 1;
+		}
+		if(bNeedSubmit) {
+			iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
+		} else {
+			DBGPRINTF("no need to submit batch, all bFilterOK==0\n");
+		}
+		if(bModifiedFilter) {
+			for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) {
+				/* note: clang static code analyzer reports a false positive below */
+				pBatch->pElem[i].bFilterOK = pFilterSave[i];
+			}
+		}
+	} else {
+		iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
+	}
+
+finalize_it:
+	RETiRet;
+}
+
 /* This submits the message to the action queue in case we do NOT need to handle repeat
  * message processing. That case permits us to gain lots of freedom during processing
  * and thus speed.
@@ -1535,7 +1592,6 @@ static rsRetVal
 doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch)
 {
 	int i;
-	sbool bNeedSubmit;
 	DEFiRet;
 
 	/* TODO
@@ -1546,33 +1602,9 @@ doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch)
 	*/
 
 	DBGPRINTF("Called action(Batch), logging to %s\n", module.GetStateName(pAction->pMod));
-	/* if necessary, take care of failover cases. We do this by simply
-	 * changing the filter setting, which is perfectly legal.
-	 */
+
 	if(pAction->pQueue->qType == QUEUETYPE_DIRECT) {
-		/* note: for direct mode, we need to adjust the filter property. For non-direct
-		 * this is not necessary, because in that case we enqueue only what actually needs
-		 * to be processed.
-		 */
-		if(pAction->bExecWhenPrevSusp) {
-			bNeedSubmit = 0;
-			for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
-				if(!pBatch->pElem[i].bPrevWasSuspended) {
-					DBGPRINTF("action enq stage: change bFilterOK to 0 due to "
-						  "failover case in elem %d\n", i);
-					pBatch->pElem[i].bFilterOK = 0;
-				}
-				if(pBatch->pElem[i].bFilterOK)
-					bNeedSubmit = 1;
-			}
-			if(bNeedSubmit) {
-				iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
-			} else {
-				DBGPRINTF("no need to submit batch, all bFilterOK==0\n");
-			}
-		} else {
-			iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
-		}
+		iRet = doQueueEnqObjDirectBatch(pAction, pBatch);
 	} else {  /* in this case, we do single submits to the queue. 
 		 * TODO: optimize this, we may do at least a multi-submit!
 		 */
diff --git a/tests/manytcp-too-few-tls.sh b/tests/manytcp-too-few-tls.sh
index 899a87dc..970a572d 100755
--- a/tests/manytcp-too-few-tls.sh
+++ b/tests/manytcp-too-few-tls.sh
@@ -11,5 +11,9 @@ sleep 1
 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
 source $srcdir/diag.sh wait-shutdown-vg	# we need to wait until rsyslogd is finished!
 source $srcdir/diag.sh check-exit-vg
-source $srcdir/diag.sh seq-check 0 39999
+# we do not do a seq check, as of the design of this test some messages
+# will be lost. So there is no point in checking if all were received. The
+# point is that we look at the valgrind result, to make sure we do not
+# have a mem leak in those error cases (we had in the past, thus the test
+# to prevent that in the future).
 source $srcdir/diag.sh exit
-- 
cgit 


From 656740b66307af24d1318d389c3ef0e7095460bc Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 20 Jun 2011 15:01:11 +0200
Subject: bugfix: problems in failover action handling

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=254
---
 ChangeLog      |  3 ++-
 action.c       | 20 ++++++++++++--------
 runtime/rule.c |  1 -
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 4d1fe09f..6cdfb670 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,8 @@
 ---------------------------------------------------------------------------
 Version 5.8.2  [V5-stable] (rgerhards), 2011-06-??
 - bugfix: problems in failover action handling
-  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270 (not yet confirmed!)
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=254
 - bugfix: memory leak in imtcp & subsystems under some circumstances
   This leak is tied to error conditions which lead to incorrect cleanup
   of some data structures. [backport from v6]
diff --git a/action.c b/action.c
index 7909d8e3..54a05fc3 100644
--- a/action.c
+++ b/action.c
@@ -979,11 +979,13 @@ dbgprintf("XXXXX:     tryDoAction %p, pnElem %d, nElem %d\n", pAction, *pnElem,
 			if(localRet == RS_RET_OK) {
 				/* mark messages as committed */
 				while(iCommittedUpTo <= i) {
+					pBatch->pElem[iCommittedUpTo].bPrevWasSuspended = 0; /* we had success! */
 					pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM;
 				}
 			} else if(localRet == RS_RET_PREVIOUS_COMMITTED) {
 				/* mark messages as committed */
 				while(iCommittedUpTo < i) {
+					pBatch->pElem[iCommittedUpTo].bPrevWasSuspended = 0; /* we had success! */
 					pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM;
 				}
 				pBatch->pElem[i].state = BATCH_STATE_SUB;
@@ -1563,6 +1565,9 @@ doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch)
 			}
 			if(pBatch->pElem[i].bFilterOK)
 				bNeedSubmit = 1;
+			DBGPRINTF("action %p[%d]: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n",
+				   pAction, i, pBatch->pElem[i].bFilterOK,  pBatch->pElem[i].state,
+				   pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended);
 		}
 		if(bNeedSubmit) {
 			iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch);
@@ -1571,6 +1576,9 @@ doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch)
 		}
 		if(bModifiedFilter) {
 			for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) {
+				DBGPRINTF("action %p: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n",
+					   pAction, pBatch->pElem[i].bFilterOK,  pBatch->pElem[i].state,
+					   pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended);
 				/* note: clang static code analyzer reports a false positive below */
 				pBatch->pElem[i].bFilterOK = pFilterSave[i];
 			}
@@ -1594,21 +1602,17 @@ doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch)
 	int i;
 	DEFiRet;
 
-	/* TODO
-	ich arbeite an dieser funktion, es müssen die verscheidenen modi geprüft werden. ausserdem
-	muss geschaut werden, in welche anderen funktionen die neue Funktionalität noch eingebaut
-	werden muss, bzw. ob man das an zentralerer stelle machen kann. Am besten die gesamte
-	filter evaluation nochmal druchgehen (also das füllen des arrays).
-	*/
-
 	DBGPRINTF("Called action(Batch), logging to %s\n", module.GetStateName(pAction->pMod));
 
 	if(pAction->pQueue->qType == QUEUETYPE_DIRECT) {
 		iRet = doQueueEnqObjDirectBatch(pAction, pBatch);
-	} else {  /* in this case, we do single submits to the queue. 
+	} else {/* in this case, we do single submits to the queue. 
 		 * TODO: optimize this, we may do at least a multi-submit!
 		 */
 		for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+			DBGPRINTF("action %p: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n",
+		           pAction, pBatch->pElem[i].bFilterOK,  pBatch->pElem[i].state,
+			   pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended);
 			if(   pBatch->pElem[i].bFilterOK
 			   && pBatch->pElem[i].state != BATCH_STATE_DISC
 			   && (pAction->bExecWhenPrevSusp == 0 || pBatch->pElem[i].bPrevWasSuspended == 1)) {
diff --git a/runtime/rule.c b/runtime/rule.c
index 19239d61..0776e2dc 100644
--- a/runtime/rule.c
+++ b/runtime/rule.c
@@ -279,7 +279,6 @@ processBatch(rule_t *pThis, batch_t *pBatch)
 		if(localRet != RS_RET_OK) {
 			DBGPRINTF("processBatch: iRet %d returned from shouldProcessThisMessage, "
 			          "ignoring message\n", localRet);
-			
 			pBatch->pElem[i].bFilterOK = 0;
 		}
 		if(pBatch->pElem[i].bFilterOK) {
-- 
cgit 


From 9fcc6b7285c316b97d4178c9e264d6de432f88b6 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 20 Jun 2011 15:57:15 +0200
Subject: testbench: added new tests for failover cases

---
 tests/Makefile.am                            | 11 ++++++
 tests/diag.sh                                |  4 +--
 tests/failover-async.sh                      |  2 +-
 tests/failover-double.sh                     | 12 +++++++
 tests/sndrcv_drvr.sh                         | 50 +---------------------------
 tests/sndrcv_drvr_noexit.sh                  | 49 +++++++++++++++++++++++++++
 tests/sndrcv_failover.sh                     | 21 ++++++++++++
 tests/testsuites/failover-double.conf        |  9 +++++
 tests/testsuites/sndrcv_failover_rcvr.conf   | 11 ++++++
 tests/testsuites/sndrcv_failover_sender.conf | 13 ++++++++
 10 files changed, 130 insertions(+), 52 deletions(-)
 create mode 100755 tests/failover-double.sh
 create mode 100755 tests/sndrcv_drvr_noexit.sh
 create mode 100755 tests/sndrcv_failover.sh
 create mode 100644 tests/testsuites/failover-double.conf
 create mode 100644 tests/testsuites/sndrcv_failover_rcvr.conf
 create mode 100644 tests/testsuites/sndrcv_failover_sender.conf

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 3ee69413..eec7b16b 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -18,6 +18,7 @@ TESTS +=  \
 	imtcp_conndrop.sh \
 	imtcp_addtlframedelim.sh \
 	sndrcv.sh \
+	sndrcv_failover.sh \
 	sndrcv_gzip.sh \
 	sndrcv_udp.sh \
 	sndrcv_udp_nonstdpt.sh \
@@ -52,6 +53,8 @@ TESTS +=  \
 	discard-rptdmsg.sh \
 	discard-allmark.sh \
 	discard.sh \
+	failover-async.sh \
+	failover-double.sh \
 	failover-basic.sh \
 	failover-rptd.sh \
 	failover-no-rptd.sh \
@@ -285,6 +288,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   failover-basic.sh \
 	   failover-basic-vg.sh \
 	   testsuites/failover-basic.conf \
+	   failover-async.sh \
+	   testsuites/failover-async.conf \
+	   failover-double.sh \
+	   testsuites/failover-double.conf \
 	   discard-rptdmsg.sh \
 	   discard-rptdmsg-vg.sh \
 	   testsuites/discard-rptdmsg.conf \
@@ -307,6 +314,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   threadingmqaq.sh \
 	   testsuites/threadingmqaq.conf \
 	   sndrcv_drvr.sh \
+	   sndrcv_drvr_noexit.sh \
+	   sndrcv_failover.sh \
+	   testsuites/sndrcv_failover_sender.conf \
+	   testsuites/sndrcv_failover_rcvr.conf \
 	   sndrcv.sh \
 	   testsuites/sndrcv_sender.conf \
 	   testsuites/sndrcv_rcvr.conf \
diff --git a/tests/diag.sh b/tests/diag.sh
index d1242fb1..7651b1de 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -22,7 +22,7 @@ case $1 in
 		rm -f work rsyslog.out.log rsyslog2.out.log rsyslog.out.log.save # common work files
 		rm -rf test-spool test-logdir
 		rm -f rsyslog.out.*.log work-presort rsyslog.pipe
-		rm -f rsyslog.input
+		rm -f rsyslog.input rsyslog.empty
 		rm -f core.* vgcore.*
 		mkdir test-spool
 		;;
@@ -31,7 +31,7 @@ case $1 in
 		rm -f work rsyslog.out.log rsyslog2.out.log rsyslog.out.log.save # common work files
 		rm -rf test-spool test-logdir 
 		rm -f rsyslog.out.*.log rsyslog.random.data work-presort rsyslog.pipe
-		rm -f rsyslog.input stat-file1
+		rm -f rsyslog.input stat-file1 #rsyslog.empty
 		echo  -------------------------------------------------------------------------------
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
diff --git a/tests/failover-async.sh b/tests/failover-async.sh
index f17bc0ff..5fc393de 100755
--- a/tests/failover-async.sh
+++ b/tests/failover-async.sh
@@ -3,7 +3,7 @@ echo ===========================================================================
 echo \[failover-async.sh\]: async test for failover functionality
 source $srcdir/diag.sh init
 source $srcdir/diag.sh startup failover-async.conf
-source $srcdir/diag.sh injectmsg  0 5
+source $srcdir/diag.sh injectmsg  0 5000
 echo doing shutdown
 source $srcdir/diag.sh shutdown-when-empty
 echo wait on shutdown
diff --git a/tests/failover-double.sh b/tests/failover-double.sh
new file mode 100755
index 00000000..172b91de
--- /dev/null
+++ b/tests/failover-double.sh
@@ -0,0 +1,12 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-double.sh\]: test for double failover functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup failover-double.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown 
+source $srcdir/diag.sh seq-check  0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/sndrcv_drvr.sh b/tests/sndrcv_drvr.sh
index 3d613069..f9092647 100755
--- a/tests/sndrcv_drvr.sh
+++ b/tests/sndrcv_drvr.sh
@@ -1,50 +1,2 @@
-# This is test driver for testing two rsyslog instances. It can be
-# utilized by any test that just needs two instances with different
-# config files, where messages are injected in instance TWO and 
-# (with whatever rsyslog mechanism) being relayed over to instance ONE,
-# where they are written to the log file. After the run, the completeness
-# of that log file is checked.
-# The code is almost the same, but the config files differ (probably greatly)
-# for different test cases. As such, this driver needs to be called with the
-# config file name ($2). From that name, the sender and receiver config file
-# names are automatically generated. 
-# So: $1 config file name, $2 number of messages
-#
-# A note on TLS testing: the current testsuite (in git!) already contains
-# TLS test cases. However, getting these test cases correct is not simple.
-# That's not a problem with the code itself, but rater a problem with
-# synchronization in the test environment. So I have deciced to keep the
-# TLS tests in, but not yet actually utilize them. This is most probably
-# left as an excercise for future (devel) releases. -- rgerhards, 2009-11-11
-#
-# added 2009-11-11 by Rgerhards
-# This file is part of the rsyslog project, released  under GPLv3
-# uncomment for debugging support:
-source $srcdir/diag.sh init
-# start up the instances
-#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
-#export RSYSLOG_DEBUGLOG="log"
-source $srcdir/diag.sh startup $1_rcvr.conf 
-source $srcdir/diag.sh wait-startup
-#export RSYSLOG_DEBUGLOG="log2"
-#valgrind="valgrind"
-source $srcdir/diag.sh startup $1_sender.conf 2
-source $srcdir/diag.sh wait-startup 2
-# may be needed by TLS (once we do it): sleep 30
-
-# now inject the messages into instance 2. It will connect to instance 1,
-# and that instance will record the data.
-source $srcdir/diag.sh tcpflood -m$2 -i1
-sleep 2 # make sure all data is received in input buffers
-# shut down sender when everything is sent, receiver continues to run concurrently
-# may be needed by TLS (once we do it): sleep 60
-source $srcdir/diag.sh shutdown-when-empty 2
-source $srcdir/diag.sh wait-shutdown 2
-# now it is time to stop the receiver as well
-source $srcdir/diag.sh shutdown-when-empty
-source $srcdir/diag.sh wait-shutdown
-
-# may be needed by TLS (once we do it): sleep 60
-# do the final check
-source $srcdir/diag.sh seq-check 1 $2
+source $srcdir/sndrcv_drvr_noexit.sh $1 $2
 source $srcdir/diag.sh exit
diff --git a/tests/sndrcv_drvr_noexit.sh b/tests/sndrcv_drvr_noexit.sh
new file mode 100755
index 00000000..899eace3
--- /dev/null
+++ b/tests/sndrcv_drvr_noexit.sh
@@ -0,0 +1,49 @@
+# This is test driver for testing two rsyslog instances. It can be
+# utilized by any test that just needs two instances with different
+# config files, where messages are injected in instance TWO and 
+# (with whatever rsyslog mechanism) being relayed over to instance ONE,
+# where they are written to the log file. After the run, the completeness
+# of that log file is checked.
+# The code is almost the same, but the config files differ (probably greatly)
+# for different test cases. As such, this driver needs to be called with the
+# config file name ($2). From that name, the sender and receiver config file
+# names are automatically generated. 
+# So: $1 config file name, $2 number of messages
+#
+# A note on TLS testing: the current testsuite (in git!) already contains
+# TLS test cases. However, getting these test cases correct is not simple.
+# That's not a problem with the code itself, but rater a problem with
+# synchronization in the test environment. So I have deciced to keep the
+# TLS tests in, but not yet actually utilize them. This is most probably
+# left as an excercise for future (devel) releases. -- rgerhards, 2009-11-11
+#
+# added 2009-11-11 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+# uncomment for debugging support:
+source $srcdir/diag.sh init
+# start up the instances
+#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction"
+#export RSYSLOG_DEBUGLOG="log"
+source $srcdir/diag.sh startup $1_rcvr.conf 
+source $srcdir/diag.sh wait-startup
+#export RSYSLOG_DEBUGLOG="log2"
+#valgrind="valgrind"
+source $srcdir/diag.sh startup $1_sender.conf 2
+source $srcdir/diag.sh wait-startup 2
+# may be needed by TLS (once we do it): sleep 30
+
+# now inject the messages into instance 2. It will connect to instance 1,
+# and that instance will record the data.
+source $srcdir/diag.sh tcpflood -m$2 -i1
+sleep 2 # make sure all data is received in input buffers
+# shut down sender when everything is sent, receiver continues to run concurrently
+# may be needed by TLS (once we do it): sleep 60
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+# now it is time to stop the receiver as well
+source $srcdir/diag.sh shutdown-when-empty
+source $srcdir/diag.sh wait-shutdown
+
+# may be needed by TLS (once we do it): sleep 60
+# do the final check
+source $srcdir/diag.sh seq-check 1 $2
diff --git a/tests/sndrcv_failover.sh b/tests/sndrcv_failover.sh
new file mode 100755
index 00000000..4c5e1831
--- /dev/null
+++ b/tests/sndrcv_failover.sh
@@ -0,0 +1,21 @@
+# This tests failover capabilities. Data is sent to local port 13516, where
+# no process shall listen. Then it fails over to a second instance, then to
+# a file. The second instance is started. So all data should be received
+# there and none be logged to the file.
+# This builds on the basic sndrcv.sh test, but adds a first, failing,
+# location to the conf file.
+# added 2011-06-20 by Rgerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo \[sndrcv_failover.sh\]: testing failover capabilities for tcp sending
+source $srcdir/sndrcv_drvr_noexit.sh sndrcv_failover 50000
+ls -l rsyslog.empty
+if [[ -s rsyslog.empty ]] ; then
+  echo "FAIL: rsyslog.empty has data. Failover handling failed. Data is written"
+  echo "      even though the previous action (in a failover chain!) properly"
+  echo "      worked."
+  exit 1
+else
+  echo "rsyslog.empty is empty - OK"
+fi ;
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/failover-double.conf b/tests/testsuites/failover-double.conf
new file mode 100644
index 00000000..a9991321
--- /dev/null
+++ b/tests/testsuites/failover-double.conf
@@ -0,0 +1,9 @@
+$IncludeConfig diag-common.conf
+
+$template outfmt,"%msg:F,58:2%\n"
+
+:msg, contains, "msgnum:" @@127.0.0.1:13516
+$ActionExecOnlyWhenPreviousIsSuspended on
+&	@@127.0.0.1:1234
+&	./rsyslog.out.log;outfmt
+$ActionExecOnlyWhenPreviousIsSuspended off
diff --git a/tests/testsuites/sndrcv_failover_rcvr.conf b/tests/testsuites/sndrcv_failover_rcvr.conf
new file mode 100644
index 00000000..6f7ce34b
--- /dev/null
+++ b/tests/testsuites/sndrcv_failover_rcvr.conf
@@ -0,0 +1,11 @@
+# see equally-named shell file for details
+# rgerhards, 2009-11-11
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# then SENDER sends to this port (not tcpflood!)
+$InputTCPServerRun 13515
+
+$template outfmt,"%msg:F,58:2%\n"
+$template dynfile,"rsyslog.out.log" # trick to use relative path names!
+:msg, contains, "msgnum:" ?dynfile;outfmt
diff --git a/tests/testsuites/sndrcv_failover_sender.conf b/tests/testsuites/sndrcv_failover_sender.conf
new file mode 100644
index 00000000..b8e7c186
--- /dev/null
+++ b/tests/testsuites/sndrcv_failover_sender.conf
@@ -0,0 +1,13 @@
+# see tcpsndrcv.sh for details
+# rgerhards, 2009-11-11
+$IncludeConfig diag-common2.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# this listener is for message generation by the test framework!
+$InputTCPServerRun 13514
+
+*.*	@@127.0.0.1:13516 # this must be DEAD
+$ActionExecOnlyWhenPreviousIsSuspended on
+&	@@127.0.0.1:13515
+&	./rsyslog.empty
+$ActionExecOnlyWhenPreviousIsSuspended off
-- 
cgit 


From de33ec026e505658d613b372ea9d32656ebb86d8 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 21 Jun 2011 10:03:47 +0200
Subject: preparing for 5.8.2

---
 ChangeLog         | 2 +-
 configure.ac      | 2 +-
 doc/manual.html   | 2 +-
 tests/Makefile.am | 3 ++-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6cdfb670..9cef03e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,5 @@
 ---------------------------------------------------------------------------
-Version 5.8.2  [V5-stable] (rgerhards), 2011-06-??
+Version 5.8.2  [V5-stable] (rgerhards), 2011-06-21
 - bugfix: problems in failover action handling
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=254
diff --git a/configure.ac b/configure.ac
index 1faaff89..5873c661 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[5.8.1],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[5.8.2],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
diff --git a/doc/manual.html b/doc/manual.html
index 5c656752..db9eb907 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 5.8.1 (stable branch) of rsyslog.</b>
+<p><b>This documentation is for version 5.8.2 (stable branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
diff --git a/tests/Makefile.am b/tests/Makefile.am
index eec7b16b..77381d2f 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -101,7 +101,8 @@ endif
 
 if ENABLE_GNUTLS
 if HAVE_VALGRIND
-TESTS += manytcp-too-few-tls.sh
+# This test does not work on v5 as we keep DH params
+#TESTS += manytcp-too-few-tls.sh
 endif
 endif
 
-- 
cgit 


From 86225089f2d0e82deb368e1688464e8ba84d24cf Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 21 Jun 2011 12:35:14 +0200
Subject: bugfix: mutex was invalidly left unlocked during action processing

At least one case where this can occur is during thread shutdown, which
may be initiated by lower activity. In most cases, this is quite
unlikely to happen. However, if it does, data structures may be
corrupted which could lead to fatal failure and segfault. I detected
this via a testbench test, not a user report. But I assume that some
users may have had unreproducable aborts that were cause by this bug.
---
 ChangeLog       |  7 +++++++
 runtime/queue.c | 12 ++++++++----
 tests/diag.sh   |  2 +-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 9cef03e6..e438ac61 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,13 @@ Version 5.8.2  [V5-stable] (rgerhards), 2011-06-21
 - bugfix: problems in failover action handling
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=270
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=254
+- bugfix: mutex was invalidly left unlocked during action processing
+  At least one case where this can occur is during thread shutdown, which
+  may be initiated by lower activity. In most cases, this is quite
+  unlikely to happen. However, if it does, data structures may be 
+  corrupted which could lead to fatal failure and segfault. I detected
+  this via a testbench test, not a user report. But I assume that some
+  users may have had unreproducable aborts that were cause by this bug.
 - bugfix: memory leak in imtcp & subsystems under some circumstances
   This leak is tied to error conditions which lead to incorrect cleanup
   of some data structures. [backport from v6]
diff --git a/runtime/queue.c b/runtime/queue.c
index 88e01a7a..00eb76c7 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1678,6 +1678,7 @@ static rsRetVal
 ConsumerReg(qqueue_t *pThis, wti_t *pWti)
 {
 	int iCancelStateSave;
+	int bNeedReLock = 0;	/**< do we need to lock the mutex again? */
 	DEFiRet;
 
 	ISOBJ_TYPE_assert(pThis, qqueue);
@@ -1687,6 +1688,7 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti)
 
 	/* we now have a non-idle batch of work, so we can release the queue mutex and process it */
 	d_pthread_mutex_unlock(pThis->mut);
+	bNeedReLock = 1;
 
 	/* at this spot, we may be cancelled */
 	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave);
@@ -1706,12 +1708,14 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti)
 	/* but now cancellation is no longer permitted */
 	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
 
-	/* now we are done, but need to re-aquire the mutex */
-	d_pthread_mutex_lock(pThis->mut);
-
 finalize_it:
-	dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet,
+	DBGPRINTF("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet,
 	          getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
+
+	/* now we are done, but potentially need to re-aquire the mutex */
+	if(bNeedReLock)
+		d_pthread_mutex_lock(pThis->mut);
+
 	RETiRet;
 }
 
diff --git a/tests/diag.sh b/tests/diag.sh
index 7651b1de..617118ba 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -10,7 +10,7 @@
 #valgrind="valgrind --tool=helgrind --log-fd=1"
 #valgrind="valgrind --tool=exp-ptrcheck --log-fd=1"
 #set -o xtrace
-#export RSYSLOG_DEBUG="debug nologfuncflow nostdout"
+#export RSYSLOG_DEBUG="debug nologfuncflow noprintmutexaction stdout"
 #export RSYSLOG_DEBUGLOG="log"
 case $1 in
    'init')	$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
-- 
cgit