From ce1860a0bc77ec2ba7992547776a46df6b3d0e8a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 14:40:36 +0100
Subject: bugfix: failover did not work correctly if repeated msg reduction was
 on   affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on

---
 ChangeLog                              |   2 +
 action.c                               | 149 ++++++++++++++++++++++++++-------
 action.h                               |  19 +----
 tests/Makefile.am                      |   9 ++
 tests/diag.sh                          |   2 +-
 tests/failover-async.sh                |  12 +++
 tests/failover-basic.sh                |  12 +++
 tests/failover-no-rptd.sh              |  19 +++++
 tests/failover-rptd.sh                 |  12 +++
 tests/testsuites/failover-async.conf   |   9 ++
 tests/testsuites/failover-basic.conf   |   8 ++
 tests/testsuites/failover-no-rptd.conf |   9 ++
 tests/testsuites/failover-rptd.conf    |  10 +++
 13 files changed, 223 insertions(+), 49 deletions(-)
 create mode 100755 tests/failover-async.sh
 create mode 100755 tests/failover-basic.sh
 create mode 100755 tests/failover-no-rptd.sh
 create mode 100755 tests/failover-rptd.sh
 create mode 100644 tests/testsuites/failover-async.conf
 create mode 100644 tests/testsuites/failover-basic.conf
 create mode 100644 tests/testsuites/failover-no-rptd.conf
 create mode 100644 tests/testsuites/failover-rptd.conf

diff --git a/ChangeLog b/ChangeLog
index 74fd299f..c9bf22b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.6.5  [V5-STABLE] (rgerhards), 2011-03-??
+- bugfix: failover did not work correctly if repeated msg reduction was on
+  affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on
 - bugfix: omlibdbi did not use password from rsyslog.con
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203
 - bugfix(kind of): tell users that config graph can currently not be
diff --git a/action.c b/action.c
index d41449d0..4312c377 100644
--- a/action.c
+++ b/action.c
@@ -4,7 +4,44 @@
  *
  * File begun on 2007-08-06 by RGerhards (extracted from syslogd.c)
  *
- * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH.
+ * Some notes on processing (this hopefully makes it easier to find
+ * the right code in question): For performance reasons, this module
+ * uses different methods of message submission based on the user-selected
+ * configuration. This code is similar, but can not be abstracted because
+ * of the performanse-affecting differences in it. As such, it is often
+ * necessary to triple-check that everything works well in *all* modes.
+ * The different modes (and calling sequence) are:
+ *
+ * if set iExecEveryNthOccur > 1 || f_ReduceRepeated || iSecsExecOnceInterval
+ * - doSubmitToActionQComplexBatch
+ * - helperSubmitToActionQComplexBatch
+ * - doActionCallAction
+ *   handles duplicate message processing, but in essence calls
+ * - actionWriteToAction
+ * - qqueueEnqObj
+ *   (now queue engine processing)
+ * if(pThis->bWriteAllMarkMsgs == FALSE) - this is the DEFAULT
+ * - doSubmitToActionQNotAllMarkBatch
+ * - doSubmitToActionQBatch (and from here like in the else case below!)
+ * else
+ * - doSubmitToActionQBatch
+ * - doSubmitToActionQ
+ * - qqueueEnqObj
+ *   (now queue engine processing)
+ *
+ * Note that bWriteAllMakrMsgs on or off creates almost the same processing.
+ * The difference ist that if WriteAllMarkMsgs is not set, we need to
+ * preprocess the batch and drop mark messages which are not yet due for
+ * writing.
+ *
+ * After dequeue, processing is as follows:
+ * - processBatchMain
+ * - processAction
+ * - submitBatch
+ * - tryDoAction
+ * - 
+ *
+ * Copyright 2007-2011 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -710,6 +747,7 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch)
 	ASSERT(pAction != NULL);
 
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+dbgprintf("XXXX: releaseBatch %d: bPrevWasSuspended %d\n", i, pBatch->pElem[i].bPrevWasSuspended);
 		pElem = &(pBatch->pElem[i]);
 		if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) {
 			switch(pAction->eParamPassing) {
@@ -897,8 +935,9 @@ tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem)
 	while(iElemProcessed <= *pnElem && i < pBatch->nElem) {
 		if(*(pBatch->pbShutdownImmediate))
 			ABORT_FINALIZE(RS_RET_FORCE_TERM);
+dbgprintf("XXXX:tryDoAction %d: bExecWhenPrevSusp: %d, bPrevWasSuspended %d ptr %p\n", i, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended, &(pBatch->pElem[i].bPrevWasSuspended));
 		if(   pBatch->pElem[i].bFilterOK
-		   && pBatch->pElem[i].state != BATCH_STATE_DISC
+		   && pBatch->pElem[i].state != BATCH_STATE_DISC//) {
 		   && ((pAction->bExecWhenPrevSusp  == 0) || pBatch->pElem[i].bPrevWasSuspended) ) {
 			pMsg = (msg_t*) pBatch->pElem[i].pUsrp;
 			localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams);
@@ -962,6 +1001,7 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem)
 	bDone = 0;
 	do {
 		localRet = tryDoAction(pAction, pBatch, &nElem);
+dbgprintf("XXXX: tryDoAction returns %d\n", localRet);
 		if(localRet == RS_RET_FORCE_TERM) {
 			ABORT_FINALIZE(RS_RET_FORCE_TERM);
 		}
@@ -981,12 +1021,14 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem)
 		} else if(localRet == RS_RET_SUSPENDED) {
 			; /* do nothing, this will retry the full batch */
 		} else if(localRet == RS_RET_ACTION_FAILED) {
+dbgprintf("XXXX: in ACTION_FAILED branch, doneUpTo %d\n", pBatch->iDoneUpTo);
 			/* in this case, everything not yet committed is BAD */
 			for(i = pBatch->iDoneUpTo ; i < wasDoneTo + nElem ; ++i) {
 				if(   pBatch->pElem[i].state != BATCH_STATE_DISC
 				   && pBatch->pElem[i].state != BATCH_STATE_COMM ) {
 					pBatch->pElem[i].state = BATCH_STATE_BAD;
 					pBatch->pElem[i].bPrevWasSuspended = 1;
+dbgprintf("XXXX: setting susps for item %d ptr %p\n", i, &(pBatch->pElem[i].bPrevWasSuspended));
 				}
 			}
 			bDone = 1;
@@ -1029,6 +1071,7 @@ prepareBatch(action_t *pAction, batch_t *pBatch)
 
 	pBatch->iDoneUpTo = 0;
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+dbgprintf("XXXX: prepareBatch %d: bPrevWasSuspended %d\n", i, pBatch->pElem[i].bPrevWasSuspended);
 		pElem = &(pBatch->pElem[i]);
 		if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) {
 			pElem->state = BATCH_STATE_RDY;
@@ -1069,6 +1112,7 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate)
 	DEFiRet;
 
 	assert(pBatch != NULL);
+dbgprintf("XXXX: running processBatchMain\n");
 
 	pbShutdownImmdtSave = pBatch->pbShutdownImmediate;
 	pBatch->pbShutdownImmediate = pbShutdownImmediate;
@@ -1098,6 +1142,7 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate)
 		iRet = localRet;
 
 finalize_it:
+dbgprintf("XXXX: exiting processBatchMain\n");
 	pBatch->pbShutdownImmediate = pbShutdownImmdtSave;
 	RETiRet;
 }
@@ -1163,12 +1208,33 @@ static rsRetVal setActionQueType(void __attribute__((unused)) *pVal, uchar *pszT
 }
 
 
+/* This submits the message to the action queue in case we do NOT need to handle repeat
+ * message processing. That case permits us to gain lots of freedom during processing
+ * and thus speed. This is also utilized to submit messages in complex case once
+ * the complex logic has been applied ;)
+ * rgerhards, 2010-06-08
+ */
+static inline rsRetVal
+doSubmitToActionQ(action_t *pAction, msg_t *pMsg)
+{
+	DEFiRet;
+
+	if(pAction->pQueue->qType == QUEUETYPE_DIRECT)
+		iRet = qqueueEnqObjDirect(pAction->pQueue, (void*) MsgAddRef(pMsg));
+	else
+		iRet = qqueueEnqObj(pAction->pQueue, pMsg->flowCtlType, (void*) MsgAddRef(pMsg));
+
+	RETiRet;
+}
+
+
 /* This function builds up a batch of messages to be (later)
  * submitted to the action queue.
  */
 rsRetVal
-actionWriteToAction(action_t *pAction)
+actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 {
+dbgprintf("XXXX: enter actionWriteToAction idx %d\n", idxBtch);
 	msg_t *pMsgSave;	/* to save current message pointer, necessary to restore
 				   it in case it needs to be updated (e.g. repeated msgs) */
 	DEFiRet;
@@ -1238,7 +1304,7 @@ actionWriteToAction(action_t *pAction)
 		pAction->f_pMsg = pMsg;	/* use the new msg (pointer will be restored below) */
 	}
 
-	DBGPRINTF("Called action, logging to %s\n", module.GetStateName(pAction->pMod));
+	DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod));
 
 	/* now check if we need to drop the message because otherwise the action would be too
 	 * frequently called. -- rgerhards, 2008-04-08
@@ -1256,15 +1322,45 @@ actionWriteToAction(action_t *pAction)
 		FINALIZE;
 	}
 
-	/* we use reception time, not dequeue time - this is considered more appropriate and also faster ;) -- rgerhards, 2008-09-17 */
+	/* we use reception time, not dequeue time - this is considered more appropriate and also faster ;)
+	 * rgerhards, 2008-09-17 */
 	pAction->tLastExec = getActNow(pAction); /* re-init time flags */
 	pAction->f_time = pAction->f_pMsg->ttGenTime;
 
+dbgprintf("XXXX:actionWriteToAction %d: bExecWhenPrevSusp: %d, bPrevWasSuspended %d ptr %p\n", idxBtch, pAction->bExecWhenPrevSusp, pBatch->pElem[idxBtch].bPrevWasSuspended, &(pBatch->pElem[idxBtch].bPrevWasSuspended));
 	/* When we reach this point, we have a valid, non-disabled action.
 	 * So let's enqueue our message for execution. -- rgerhards, 2007-07-24
 	 */
-	iRet = qqueueEnqObj(pAction->pQueue, pAction->f_pMsg->flowCtlType, (void*) MsgAddRef(pAction->f_pMsg));
+	if(pAction->bExecWhenPrevSusp  == 1 && pBatch->pElem[idxBtch].bPrevWasSuspended) {
+		/* in that case, we need to create a special batch which reflects the
+		 * suspended state. Otherwise, that information would be dropped inside
+		 * the queue engine. TODO: in later releases (v6?) create a better
+		 * solution than what we do here. However, for v5 this sounds much too
+		 * intrusive. -- rgerhardsm, 2011-03-16
+		 * (Code is copied over from queue.c and slightly modified)
+		 */
+		batch_t singleBatch;
+		batch_obj_t batchObj;
+		int i;
+		memset(&batchObj, 0, sizeof(batch_obj_t));
+		memset(&singleBatch, 0, sizeof(batch_t));
+		batchObj.state = BATCH_STATE_RDY;
+		batchObj.pUsrp = (obj_t*) pAction->f_pMsg;
+		batchObj.bPrevWasSuspended = 1;
+		batchObj.bFilterOK = 1;
+		singleBatch.nElem = 1; /* there always is only one in direct mode */
+		singleBatch.pElem = &batchObj;
+
+		iRet = qqueueEnqObjDirectBatch(pAction->pQueue, &singleBatch);
+
+		for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) {
+			free(batchObj.staticActStrings[i]);
+		}
+	} else { /* standard case, just submit */
+		iRet = doSubmitToActionQ(pAction, pAction->f_pMsg);
+	}
 
+dbgprintf("XXXX: action iRet %d\n", iRet);
 	if(iRet == RS_RET_OK)
 		pAction->f_prevcount = 0; /* message processed, so we start a new cycle */
 
@@ -1291,10 +1387,13 @@ finalize_it:
  * pthread_cleanup_push() POSIX macro...
  */
 static inline rsRetVal
-doActionCallAction(action_t *pAction, msg_t *pMsg)
+doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 {
+	msg_t *pMsg;
 	DEFiRet;
 
+dbgprintf("XXXX: enter doActionCallAction\n");
+	pMsg = (msg_t*)(pBatch->pElem[idxBtch].pUsrp);
 	pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */
 
 	/* don't output marks to recently written outputs */
@@ -1321,7 +1420,7 @@ doActionCallAction(action_t *pAction, msg_t *pMsg)
 		 * isolated messages), but back off so we'll flush less often in the future.
 		 */
 		if(getActNow(pAction) > REPEATTIME(pAction)) {
-			iRet = actionWriteToAction(pAction);
+			iRet = actionWriteToAction(pAction, pBatch, idxBtch);
 			BACKOFF(pAction);
 		}
 	} else {/* new message, save it */
@@ -1330,7 +1429,7 @@ doActionCallAction(action_t *pAction, msg_t *pMsg)
 		 */
 		if(pAction->f_pMsg != NULL) {
 			if(pAction->f_prevcount > 0)
-				actionWriteToAction(pAction);
+				actionWriteToAction(pAction, pBatch, idxBtch);
 				/* we do not care about iRet above - I think it's right but if we have
 				 * some troubles, you know where to look at ;) -- rgerhards, 2007-08-01
 				 */
@@ -1338,33 +1437,21 @@ doActionCallAction(action_t *pAction, msg_t *pMsg)
 		}
 		pAction->f_pMsg = MsgAddRef(pMsg);
 		/* call the output driver */
-		iRet = actionWriteToAction(pAction);
+		iRet = actionWriteToAction(pAction, pBatch, idxBtch);
 	}
 
 finalize_it:
-	RETiRet;
-}
-
-/* This submits the message to the action queue in case we do NOT need to handle repeat
- * message processing. That case permits us to gain lots of freedom during processing
- * and thus speed.
- * rgerhards, 2010-06-08
- */
-static inline rsRetVal
-doSubmitToActionQ(action_t *pAction, msg_t *pMsg)
-{
-	DEFiRet;
-
-	if(pAction->pQueue->qType == QUEUETYPE_DIRECT)
-		iRet = qqueueEnqObjDirect(pAction->pQueue, (void*) MsgAddRef(pMsg));
-	else
-		iRet = qqueueEnqObj(pAction->pQueue, pMsg->flowCtlType, (void*) MsgAddRef(pMsg));
+	/* we need to update the batch to handle failover processing correctly */
+	if(iRet == RS_RET_OK) {
+		pBatch->pElem[idxBtch].bPrevWasSuspended = 0;
+	} else if(iRet == RS_RET_ACTION_FAILED) {
+		pBatch->pElem[idxBtch].bPrevWasSuspended = 1;
+	}
 
 	RETiRet;
 }
 
 
-
 /* This submits the message to the action queue in case where we need to handle
  * bWriteAllMarkMessage == FALSE only. Note that we use a non-blocking CAS loop
  * for the synchronization. Here, we just modify the filter condition to be false when
@@ -1480,9 +1567,11 @@ helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch)
 
 	DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod));
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
+dbgprintf("XXXX: helper complex %d: bExecWhenPrevSusp: %d, bPrevWasSuspended %d ptr %p\n", i, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended, &(pBatch->pElem[i].bPrevWasSuspended));
 		if(   pBatch->pElem[i].bFilterOK
-                   && pBatch->pElem[i].state != BATCH_STATE_DISC) {
-			doActionCallAction(pAction, (msg_t*)(pBatch->pElem[i].pUsrp));
+                   && pBatch->pElem[i].state != BATCH_STATE_DISC
+		   && ((pAction->bExecWhenPrevSusp  == 0) || pBatch->pElem[i].bPrevWasSuspended) ) {
+			doActionCallAction(pAction, pBatch, i);
 		}
 	}
 
diff --git a/action.h b/action.h
index 0c86ef88..3e5d5266 100644
--- a/action.h
+++ b/action.h
@@ -101,26 +101,9 @@ rsRetVal actionDestruct(action_t *pThis);
 rsRetVal actionDbgPrint(action_t *pThis);
 rsRetVal actionSetGlobalResumeInterval(int iNewVal);
 rsRetVal actionDoAction(action_t *pAction);
-rsRetVal actionWriteToAction(action_t *pAction);
+//rsRetVal actionWriteToAction(action_t *pAction);
 rsRetVal actionCallHUPHdlr(action_t *pAction);
 rsRetVal actionClassInit(void);
 rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended);
 
-#if 1
-#define actionIsSuspended(pThis) ((pThis)->bSuspended == 1)
-#else
-/* The function is a debugging aid */
-inline int actionIsSuspended(action_t *pThis)
-{
-	int i;
-	ASSERT(pThis != NULL);
-	i =  pThis->bSuspended == 1;
-	dbgprintf("in IsSuspend(), returns %d\n", i);
-	return i;
-}
-#endif
-
 #endif /* #ifndef ACTION_H_INCLUDED */
-/*
- * vi:set ai:
- */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 03a4985a..3ca8efca 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -48,6 +48,9 @@ TESTS = $(TESTRUNS) cfg.sh \
 	dircreate_off.sh \
 	discard-rptdmsg.sh \
 	discard.sh \
+	failover-basic.sh \
+	failover-rptd.sh \
+	failover-no-rptd.sh \
 	queue-persist.sh 
 
 if ENABLE_IMPTCP
@@ -217,6 +220,12 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   omod-if-array.sh \
 	   discard.sh \
 	   testsuites/discard.conf \
+	   failover-no-rptd.sh \
+	   testsuites/failover-no-rptd.conf \
+	   failover-rptd.sh \
+	   testsuites/failover-rptd.conf \
+	   failover-basic.sh \
+	   testsuites/failover-basic.conf \
 	   discard-rptdmsg.sh \
 	   testsuites/discard-rptdmsg.conf \
 	   diag.sh \
diff --git a/tests/diag.sh b/tests/diag.sh
index bf12fbda..fc0d9000 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -10,7 +10,7 @@
 #valgrind="valgrind --tool=helgrind --log-fd=1"
 #valgrind="valgrind --tool=exp-ptrcheck --log-fd=1"
 #set -o xtrace
-#export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUG="debug nologfuncflow nostdout"
 #export RSYSLOG_DEBUGLOG="log"
 case $1 in
    'init')	$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason
diff --git a/tests/failover-async.sh b/tests/failover-async.sh
new file mode 100755
index 00000000..f17bc0ff
--- /dev/null
+++ b/tests/failover-async.sh
@@ -0,0 +1,12 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-async.sh\]: async test for failover functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup failover-async.conf
+source $srcdir/diag.sh injectmsg  0 5
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown 
+source $srcdir/diag.sh seq-check  0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/failover-basic.sh b/tests/failover-basic.sh
new file mode 100755
index 00000000..031ea25b
--- /dev/null
+++ b/tests/failover-basic.sh
@@ -0,0 +1,12 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-basic.sh\]: basic test for failover functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup failover-basic.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown 
+source $srcdir/diag.sh seq-check  0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/failover-no-rptd.sh b/tests/failover-no-rptd.sh
new file mode 100755
index 00000000..6abeba44
--- /dev/null
+++ b/tests/failover-no-rptd.sh
@@ -0,0 +1,19 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-no-rptd.sh\]: rptd test for failover functionality - no failover
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup failover-no-rptd.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown 
+# now we need our custom logic to see if the result file is empty
+# (what it should be!)
+cmp rsyslog.out.log /dev/null
+if [ $? -eq 1 ]
+then
+	echo "ERROR, output file not empty"
+	exit 1
+fi
+source $srcdir/diag.sh exit
diff --git a/tests/failover-rptd.sh b/tests/failover-rptd.sh
new file mode 100755
index 00000000..8a313e9e
--- /dev/null
+++ b/tests/failover-rptd.sh
@@ -0,0 +1,12 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-rptd.sh\]: rptd test for failover functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup failover-rptd.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown 
+source $srcdir/diag.sh seq-check  0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/failover-async.conf b/tests/testsuites/failover-async.conf
new file mode 100644
index 00000000..76445de3
--- /dev/null
+++ b/tests/testsuites/failover-async.conf
@@ -0,0 +1,9 @@
+# see the equally-named .sh file for details
+$IncludeConfig diag-common.conf
+
+$template outfmt,"%msg:F,58:2%\n"
+# note: the target server shall not be available!
+
+$ActionQueueType LinkedList
+:msg, contains, "msgnum:" @@127.0.0.1:13514
+& ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/failover-basic.conf b/tests/testsuites/failover-basic.conf
new file mode 100644
index 00000000..a858769c
--- /dev/null
+++ b/tests/testsuites/failover-basic.conf
@@ -0,0 +1,8 @@
+# see the equally-named .sh file for details
+$IncludeConfig diag-common.conf
+
+$template outfmt,"%msg:F,58:2%\n"
+# note: the target server shall not be available!
+:msg, contains, "msgnum:" @@127.0.0.1:13514
+$ActionExecOnlyWhenPreviousIsSuspended on
+& ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/failover-no-rptd.conf b/tests/testsuites/failover-no-rptd.conf
new file mode 100644
index 00000000..a46ce116
--- /dev/null
+++ b/tests/testsuites/failover-no-rptd.conf
@@ -0,0 +1,9 @@
+# see the equally-named .sh file for details
+$IncludeConfig diag-common.conf
+
+$RepeatedMsgReduction on
+
+# second action should never execute
+:msg, contains, "msgnum:" /dev/null
+$ActionExecOnlyWhenPreviousIsSuspended on
+& ./rsyslog.out.log
diff --git a/tests/testsuites/failover-rptd.conf b/tests/testsuites/failover-rptd.conf
new file mode 100644
index 00000000..d3553dbb
--- /dev/null
+++ b/tests/testsuites/failover-rptd.conf
@@ -0,0 +1,10 @@
+# see the equally-named .sh file for details
+$IncludeConfig diag-common.conf
+
+$RepeatedMsgReduction on
+
+$template outfmt,"%msg:F,58:2%\n"
+# note: the target server shall not be available!
+:msg, contains, "msgnum:" @@127.0.0.1:13514
+$ActionExecOnlyWhenPreviousIsSuspended on
+& ./rsyslog.out.log;outfmt
-- 
cgit 


From 9e18ecc9bd14093b78199fd36c315cfce4d26a26 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 15:04:43 +0100
Subject: fixed a regression of last patch and some cleanup

---
 action.c                                | 18 ++++--------------
 action.h                                |  2 +-
 tests/Makefile.am                       |  3 +++
 tests/diag.sh                           |  2 +-
 tests/failover-no-basic.sh              | 19 +++++++++++++++++++
 tests/testsuites/failover-no-basic.conf |  9 +++++++++
 tools/syslogd.c                         |  2 +-
 7 files changed, 38 insertions(+), 17 deletions(-)
 create mode 100755 tests/failover-no-basic.sh
 create mode 100644 tests/testsuites/failover-no-basic.conf

diff --git a/action.c b/action.c
index 4312c377..8726217f 100644
--- a/action.c
+++ b/action.c
@@ -747,7 +747,6 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch)
 	ASSERT(pAction != NULL);
 
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
-dbgprintf("XXXX: releaseBatch %d: bPrevWasSuspended %d\n", i, pBatch->pElem[i].bPrevWasSuspended);
 		pElem = &(pBatch->pElem[i]);
 		if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) {
 			switch(pAction->eParamPassing) {
@@ -935,7 +934,6 @@ tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem)
 	while(iElemProcessed <= *pnElem && i < pBatch->nElem) {
 		if(*(pBatch->pbShutdownImmediate))
 			ABORT_FINALIZE(RS_RET_FORCE_TERM);
-dbgprintf("XXXX:tryDoAction %d: bExecWhenPrevSusp: %d, bPrevWasSuspended %d ptr %p\n", i, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended, &(pBatch->pElem[i].bPrevWasSuspended));
 		if(   pBatch->pElem[i].bFilterOK
 		   && pBatch->pElem[i].state != BATCH_STATE_DISC//) {
 		   && ((pAction->bExecWhenPrevSusp  == 0) || pBatch->pElem[i].bPrevWasSuspended) ) {
@@ -1001,7 +999,6 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem)
 	bDone = 0;
 	do {
 		localRet = tryDoAction(pAction, pBatch, &nElem);
-dbgprintf("XXXX: tryDoAction returns %d\n", localRet);
 		if(localRet == RS_RET_FORCE_TERM) {
 			ABORT_FINALIZE(RS_RET_FORCE_TERM);
 		}
@@ -1021,14 +1018,12 @@ dbgprintf("XXXX: tryDoAction returns %d\n", localRet);
 		} else if(localRet == RS_RET_SUSPENDED) {
 			; /* do nothing, this will retry the full batch */
 		} else if(localRet == RS_RET_ACTION_FAILED) {
-dbgprintf("XXXX: in ACTION_FAILED branch, doneUpTo %d\n", pBatch->iDoneUpTo);
 			/* in this case, everything not yet committed is BAD */
 			for(i = pBatch->iDoneUpTo ; i < wasDoneTo + nElem ; ++i) {
 				if(   pBatch->pElem[i].state != BATCH_STATE_DISC
 				   && pBatch->pElem[i].state != BATCH_STATE_COMM ) {
 					pBatch->pElem[i].state = BATCH_STATE_BAD;
 					pBatch->pElem[i].bPrevWasSuspended = 1;
-dbgprintf("XXXX: setting susps for item %d ptr %p\n", i, &(pBatch->pElem[i].bPrevWasSuspended));
 				}
 			}
 			bDone = 1;
@@ -1071,7 +1066,6 @@ prepareBatch(action_t *pAction, batch_t *pBatch)
 
 	pBatch->iDoneUpTo = 0;
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
-dbgprintf("XXXX: prepareBatch %d: bPrevWasSuspended %d\n", i, pBatch->pElem[i].bPrevWasSuspended);
 		pElem = &(pBatch->pElem[i]);
 		if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) {
 			pElem->state = BATCH_STATE_RDY;
@@ -1112,7 +1106,6 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate)
 	DEFiRet;
 
 	assert(pBatch != NULL);
-dbgprintf("XXXX: running processBatchMain\n");
 
 	pbShutdownImmdtSave = pBatch->pbShutdownImmediate;
 	pBatch->pbShutdownImmediate = pbShutdownImmediate;
@@ -1142,7 +1135,6 @@ dbgprintf("XXXX: running processBatchMain\n");
 		iRet = localRet;
 
 finalize_it:
-dbgprintf("XXXX: exiting processBatchMain\n");
 	pBatch->pbShutdownImmediate = pbShutdownImmdtSave;
 	RETiRet;
 }
@@ -1230,11 +1222,12 @@ doSubmitToActionQ(action_t *pAction, msg_t *pMsg)
 
 /* This function builds up a batch of messages to be (later)
  * submitted to the action queue.
+ * Note: this function is also called from syslogd itself as part of its
+ * flush processing. If so, pBatch will be NULL and idxBtch undefined.
  */
 rsRetVal
 actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 {
-dbgprintf("XXXX: enter actionWriteToAction idx %d\n", idxBtch);
 	msg_t *pMsgSave;	/* to save current message pointer, necessary to restore
 				   it in case it needs to be updated (e.g. repeated msgs) */
 	DEFiRet;
@@ -1327,11 +1320,11 @@ dbgprintf("XXXX: enter actionWriteToAction idx %d\n", idxBtch);
 	pAction->tLastExec = getActNow(pAction); /* re-init time flags */
 	pAction->f_time = pAction->f_pMsg->ttGenTime;
 
-dbgprintf("XXXX:actionWriteToAction %d: bExecWhenPrevSusp: %d, bPrevWasSuspended %d ptr %p\n", idxBtch, pAction->bExecWhenPrevSusp, pBatch->pElem[idxBtch].bPrevWasSuspended, &(pBatch->pElem[idxBtch].bPrevWasSuspended));
 	/* When we reach this point, we have a valid, non-disabled action.
 	 * So let's enqueue our message for execution. -- rgerhards, 2007-07-24
 	 */
-	if(pAction->bExecWhenPrevSusp  == 1 && pBatch->pElem[idxBtch].bPrevWasSuspended) {
+	if(   pBatch != NULL
+	   && (pAction->bExecWhenPrevSusp  == 1 && pBatch->pElem[idxBtch].bPrevWasSuspended)) {
 		/* in that case, we need to create a special batch which reflects the
 		 * suspended state. Otherwise, that information would be dropped inside
 		 * the queue engine. TODO: in later releases (v6?) create a better
@@ -1360,7 +1353,6 @@ dbgprintf("XXXX:actionWriteToAction %d: bExecWhenPrevSusp: %d, bPrevWasSuspended
 		iRet = doSubmitToActionQ(pAction, pAction->f_pMsg);
 	}
 
-dbgprintf("XXXX: action iRet %d\n", iRet);
 	if(iRet == RS_RET_OK)
 		pAction->f_prevcount = 0; /* message processed, so we start a new cycle */
 
@@ -1392,7 +1384,6 @@ doActionCallAction(action_t *pAction, batch_t *pBatch, int idxBtch)
 	msg_t *pMsg;
 	DEFiRet;
 
-dbgprintf("XXXX: enter doActionCallAction\n");
 	pMsg = (msg_t*)(pBatch->pElem[idxBtch].pUsrp);
 	pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */
 
@@ -1567,7 +1558,6 @@ helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch)
 
 	DBGPRINTF("Called action(complex case), logging to %s\n", module.GetStateName(pAction->pMod));
 	for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) {
-dbgprintf("XXXX: helper complex %d: bExecWhenPrevSusp: %d, bPrevWasSuspended %d ptr %p\n", i, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended, &(pBatch->pElem[i].bPrevWasSuspended));
 		if(   pBatch->pElem[i].bFilterOK
                    && pBatch->pElem[i].state != BATCH_STATE_DISC
 		   && ((pAction->bExecWhenPrevSusp  == 0) || pBatch->pElem[i].bPrevWasSuspended) ) {
diff --git a/action.h b/action.h
index 3e5d5266..dc33d034 100644
--- a/action.h
+++ b/action.h
@@ -101,7 +101,7 @@ rsRetVal actionDestruct(action_t *pThis);
 rsRetVal actionDbgPrint(action_t *pThis);
 rsRetVal actionSetGlobalResumeInterval(int iNewVal);
 rsRetVal actionDoAction(action_t *pAction);
-//rsRetVal actionWriteToAction(action_t *pAction);
+rsRetVal actionWriteToAction(action_t *pAction, batch_t *pBatch, int idxBtch);
 rsRetVal actionCallHUPHdlr(action_t *pAction);
 rsRetVal actionClassInit(void);
 rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended);
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 3ca8efca..137e2115 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -51,6 +51,7 @@ TESTS = $(TESTRUNS) cfg.sh \
 	failover-basic.sh \
 	failover-rptd.sh \
 	failover-no-rptd.sh \
+	failover-no-basic.sh \
 	queue-persist.sh 
 
 if ENABLE_IMPTCP
@@ -222,6 +223,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/discard.conf \
 	   failover-no-rptd.sh \
 	   testsuites/failover-no-rptd.conf \
+	   failover-no-basic.sh \
+	   testsuites/failover-no-basic.conf \
 	   failover-rptd.sh \
 	   testsuites/failover-rptd.conf \
 	   failover-basic.sh \
diff --git a/tests/diag.sh b/tests/diag.sh
index fc0d9000..3075aa57 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -31,7 +31,7 @@ case $1 in
 		rm -f work rsyslog.out.log rsyslog2.out.log rsyslog.out.log.save # common work files
 		rm -rf test-spool test-logdir 
 		rm -f rsyslog.out.*.log rsyslog.random.data work-presort rsyslog.pipe
-		rm -f rsyslog.input
+		rm -f rsyslog.input stat-file1
 		echo  -------------------------------------------------------------------------------
 		;;
    'startup')   # start rsyslogd with default params. $2 is the config file name to use
diff --git a/tests/failover-no-basic.sh b/tests/failover-no-basic.sh
new file mode 100755
index 00000000..6177e10d
--- /dev/null
+++ b/tests/failover-no-basic.sh
@@ -0,0 +1,19 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-no-basic.sh\]: basic test for failover functionality - no failover
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup failover-no-basic.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown 
+# now we need our custom logic to see if the result file is empty
+# (what it should be!)
+cmp rsyslog.out.log /dev/null
+if [ $? -eq 1 ]
+then
+	echo "ERROR, output file not empty"
+	exit 1
+fi
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/failover-no-basic.conf b/tests/testsuites/failover-no-basic.conf
new file mode 100644
index 00000000..b40ef7d7
--- /dev/null
+++ b/tests/testsuites/failover-no-basic.conf
@@ -0,0 +1,9 @@
+# see the equally-named .sh file for details
+$IncludeConfig diag-common.conf
+
+$RepeatedMsgReduction off
+
+# second action should never execute
+:msg, contains, "msgnum:" /dev/null
+$ActionExecOnlyWhenPreviousIsSuspended on
+& ./rsyslog.out.log
diff --git a/tools/syslogd.c b/tools/syslogd.c
index dd9729ef..1148108c 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -799,7 +799,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions)
 		DBGPRINTF("flush %s: repeated %d times, %d sec.\n",
 		    module.GetStateName(pAction->pMod), pAction->f_prevcount,
 		    repeatinterval[pAction->f_repeatcount]);
-		actionWriteToAction(pAction);
+		actionWriteToAction(pAction, NULL, 0);
 		BACKOFF(pAction);
 	}
 	UnlockObj(pAction);
-- 
cgit 


From 4a80242ffcfa93348453ff716f7f1abda417a416 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 15:53:45 +0100
Subject: added some more valgrind-based tests

---
 tests/Makefile.am             |  8 ++++++++
 tests/failover-basic-vg.sh    | 13 +++++++++++++
 tests/failover-no-basic-vg.sh | 20 ++++++++++++++++++++
 tests/failover-no-rptd-vg.sh  | 20 ++++++++++++++++++++
 tests/failover-rptd-vg.sh     | 13 +++++++++++++
 5 files changed, 74 insertions(+)
 create mode 100755 tests/failover-basic-vg.sh
 create mode 100755 tests/failover-no-basic-vg.sh
 create mode 100755 tests/failover-no-rptd-vg.sh
 create mode 100755 tests/failover-rptd-vg.sh

diff --git a/tests/Makefile.am b/tests/Makefile.am
index c1818f36..ebbe0cf6 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -65,6 +65,10 @@ if HAVE_VALGRIND
 TESTS +=  \
 	discard-rptdmsg-vg.sh \
 	discard-allmark-vg.sh \
+	failover-basic-vg.sh \
+	failover-rptd-vg.sh \
+	failover-no-basic-vg.sh \
+	failover-no-rptd-vg.sh \
 	tcp-msgreduc-vg.sh
 endif
 
@@ -265,12 +269,16 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   discard.sh \
 	   testsuites/discard.conf \
 	   failover-no-rptd.sh \
+	   failover-no-rptd-vg.sh \
 	   testsuites/failover-no-rptd.conf \
 	   failover-no-basic.sh \
+	   failover-no-basic-vg.sh \
 	   testsuites/failover-no-basic.conf \
 	   failover-rptd.sh \
+	   failover-rptd-vg.sh \
 	   testsuites/failover-rptd.conf \
 	   failover-basic.sh \
+	   failover-basic-vg.sh \
 	   testsuites/failover-basic.conf \
 	   discard-rptdmsg.sh \
 	   discard-rptdmsg-vg.sh \
diff --git a/tests/failover-basic-vg.sh b/tests/failover-basic-vg.sh
new file mode 100755
index 00000000..0eb77caa
--- /dev/null
+++ b/tests/failover-basic-vg.sh
@@ -0,0 +1,13 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-basic.sh\]: basic test for failover functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup-vg failover-basic.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown-vg
+source $srcdir/diag.sh check-exit-vg
+source $srcdir/diag.sh seq-check  0 4999
+source $srcdir/diag.sh exit
diff --git a/tests/failover-no-basic-vg.sh b/tests/failover-no-basic-vg.sh
new file mode 100755
index 00000000..266163d4
--- /dev/null
+++ b/tests/failover-no-basic-vg.sh
@@ -0,0 +1,20 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-no-basic.sh\]: basic test for failover functionality - no failover
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup-vg failover-no-basic.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown-vg
+source $srcdir/diag.sh check-exit-vg
+# now we need our custom logic to see if the result file is empty
+# (what it should be!)
+cmp rsyslog.out.log /dev/null
+if [ $? -eq 1 ]
+then
+	echo "ERROR, output file not empty"
+	exit 1
+fi
+source $srcdir/diag.sh exit
diff --git a/tests/failover-no-rptd-vg.sh b/tests/failover-no-rptd-vg.sh
new file mode 100755
index 00000000..3b77a0a4
--- /dev/null
+++ b/tests/failover-no-rptd-vg.sh
@@ -0,0 +1,20 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-no-rptd.sh\]: rptd test for failover functionality - no failover
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup-vg failover-no-rptd.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown-vg
+source $srcdir/diag.sh check-exit-vg
+# now we need our custom logic to see if the result file is empty
+# (what it should be!)
+cmp rsyslog.out.log /dev/null
+if [ $? -eq 1 ]
+then
+	echo "ERROR, output file not empty"
+	exit 1
+fi
+source $srcdir/diag.sh exit
diff --git a/tests/failover-rptd-vg.sh b/tests/failover-rptd-vg.sh
new file mode 100755
index 00000000..d208003b
--- /dev/null
+++ b/tests/failover-rptd-vg.sh
@@ -0,0 +1,13 @@
+# This file is part of the rsyslog project, released under GPLv3
+echo ===============================================================================
+echo \[failover-rptd.sh\]: rptd test for failover functionality
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup-vg failover-rptd.conf
+source $srcdir/diag.sh injectmsg  0 5000
+echo doing shutdown
+source $srcdir/diag.sh shutdown-when-empty
+echo wait on shutdown
+source $srcdir/diag.sh wait-shutdown-vg
+source $srcdir/diag.sh check-exit-vg
+source $srcdir/diag.sh seq-check  0 4999
+source $srcdir/diag.sh exit
-- 
cgit 


From f057fb2db388cc16475f9c5a7de64aeb1f3bdb43 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 16:32:30 +0100
Subject: added bug info to change log

---
 ChangeLog | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d2d2a75a..4e1824b7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,9 @@ Version 5.9.0  [V5-DEVEL] (rgerhards), 2011-03-??
 - enhanced omhdfs to support batching mode. This permits to increase
   performance, as we now call the HDFS API with much larger message
   sizes and far more infrequently
+- bugfix: failover did not work correctly if repeated msg reduction was on
+  affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236
 ---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-??
 - improved testbench
@@ -138,6 +141,7 @@ Version 5.7.0  [V5-DEVEL] (rgerhards), 2010-09-16
 Version 5.6.5  [V5-STABLE] (rgerhards), 2011-03-??
 - bugfix: failover did not work correctly if repeated msg reduction was on
   affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236
 - bugfix: omlibdbi did not use password from rsyslog.con
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=203
 - bugfix(kind of): tell users that config graph can currently not be
-- 
cgit 


From 32db6291bc310062cffd3266fc297c7b13c571a7 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 16:59:49 +0100
Subject: backporting patch for clean make distcheck

---
 plugins/imdiag/imdiag.c | 3 +++
 plugins/imfile/imfile.c | 8 ++++++++
 runtime/cfsysline.c     | 3 +++
 3 files changed, 14 insertions(+)

diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c
index 0a69ee43..404cebc7 100644
--- a/plugins/imdiag/imdiag.c
+++ b/plugins/imdiag/imdiag.c
@@ -433,6 +433,9 @@ CODESTARTmodExit
 		net.DestructPermittedPeers(&pPermPeersRoot);
 	}
 
+	/* free some globals to keep valgrind happy */
+	free(pszInputName);
+
 	/* release objects we used */
 	objRelease(net, LM_NET_FILENAME);
 	objRelease(netstrm, LM_NETSTRMS_FILENAME);
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index acb58dad..cac3a55d 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -331,6 +331,11 @@ ENDrunInput
  */
 BEGINwillRun
 CODESTARTwillRun
+	/* free config variables we do no longer needed */
+	free(pszFileName);
+	free(pszFileTag);
+	free(pszStateFile);
+
 	if(iFilPtr == 0) {
 		errmsg.LogError(0, RS_RET_NO_RUN, "No files configured to be monitored");
 		ABORT_FINALIZE(RS_RET_NO_RUN);
@@ -398,6 +403,9 @@ CODESTARTafterRun
 			persistStrmState(&files[i]);
 			strm.Destruct(&(files[i].pStrm));
 		}
+		free(files[i].pszFileName);
+		free(files[i].pszTag);
+		free(files[i].pszStateFile);
 	}
 
 	if(pInputName != NULL)
diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c
index 037e9f84..646ab39b 100644
--- a/runtime/cfsysline.c
+++ b/runtime/cfsysline.c
@@ -41,6 +41,7 @@
 #include "obj.h"
 #include "errmsg.h"
 #include "srUtils.h"
+#include "unicode-helper.h"
 
 
 /* static data */
@@ -511,6 +512,8 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void
 	CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0));
 	pStrB = NULL;
 
+	DBGPRINTF("doGetWord: get newval '%s' (len %d), hdlr %p\n",
+		  pNewVal, (int) ustrlen(pNewVal), pSetHdlr);
 	/* we got the word, now set it */
 	if(pSetHdlr == NULL) {
 		/* we should set value directly to var */
-- 
cgit 


From 709ce7647ca3a44c7c6d47e0445e70a9254a2fdb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 17:17:34 +0100
Subject: preparing for 5.7.9

---
 ChangeLog       | 5 ++++-
 configure.ac    | 2 +-
 doc/manual.html | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 595a4585..13f6608d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,13 @@
 ---------------------------------------------------------------------------
-Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-??
+Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
   among others, life tests for ommysql (against a test database) have
   been added, valgrind-based testing enhanced, ...
 - enhance: fallback *at runtime* to epoll_create if epoll_create1 is not
   available. Thanks to Michael Biebl for analysis and patch!
+- bugfix: failover did not work correctly if repeated msg reduction was on
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236
+  affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on
 - bugfix: minor memory leak in omlibdbi (< 1k per instance and run)
 - bugfix: (regression) omhdfs did no longer compile
 - bugfix: omlibdbi did not use password from rsyslog.conf
diff --git a/configure.ac b/configure.ac
index 58b203ff..4f368bcc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[5.7.8],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[5.7.9],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
diff --git a/doc/manual.html b/doc/manual.html
index 945a5a62..807ac0da 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 5.7.8 (beta branch) of rsyslog.</b>
+<p><b>This documentation is for version 5.7.9 (beta branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
-- 
cgit 


From 14f3857a0a122aa212b23a59175bd4e504176372 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 17 Mar 2011 12:29:43 +0100
Subject: some minor enhancements to doc set -- point to more examples

---
 doc/rsyslog_conf.html | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html
index 6990c6bd..703e7a6e 100644
--- a/doc/rsyslog_conf.html
+++ b/doc/rsyslog_conf.html
@@ -2,13 +2,16 @@
 <html><head><title>rsyslog.conf file</title></head>
 <body>
 <h1>rsyslog.conf configuration file</h1>
-<p><b>This document is currently being enhanced. Please
-pardon its current appearance.</b></p>
 <p><b>Rsyslogd is configured via the rsyslog.conf file</b>,
 typically found in /etc. By default, rsyslogd reads the file
 /etc/rsyslog.conf. This may be changed by a command line option.</p>
 <p><a href="http://wiki.rsyslog.com/index.php/Configuration_Samples">
-Configuration file examples can be found in the rsyslog wiki</a>.</p>
+Configuration file examples can be found in the rsyslog wiki</a>. Also
+keep the
+<a href="http://www.rsyslog.com/config-snippets/">rsyslog config snippets</a>
+on your mind. These are ready-to-use
+real building blocks for rsyslog configuration.
+</p>
 <p>There is also one sample file provided together with the
 documentation set. If you do not like to read, be sure to have at least
 a quick look at
@@ -74,7 +77,7 @@ such features is available in rsyslogd, only.</p>
 [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
 <a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
-Copyright &copy; 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+Copyright &copy; 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
 version 3 or higher.</font></p>
 </body>
-- 
cgit 


From 68bde6252d22cc4e88edc2b51523726bb858c818 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 17 Mar 2011 13:19:38 +0100
Subject: prepared skeleton for new strgen module

---
 ChangeLog                                 |   2 +
 Makefile.am                               |   4 +
 configure.ac                              |  18 ++++
 plugins/sm_cust_bindcdr/Makefile.am       |   6 ++
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 138 ++++++++++++++++++++++++++++++
 5 files changed, 168 insertions(+)
 create mode 100644 plugins/sm_cust_bindcdr/Makefile.am
 create mode 100644 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c

diff --git a/ChangeLog b/ChangeLog
index 13f6608d..3503ed69 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,6 @@
 ---------------------------------------------------------------------------
+Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
   among others, life tests for ommysql (against a test database) have
diff --git a/Makefile.am b/Makefile.am
index 53c04922..2d73013b 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -112,6 +112,10 @@ if ENABLE_CUST1
 SUBDIRS += plugins/cust1
 endif
 
+if ENABLE_SMCUSTBINDCDR
+SUBDIRS += plugins/sm_cust_bindcdr
+endif
+
 if ENABLE_IMTEMPLATE
 SUBDIRS += plugins/imtemplate
 endif
diff --git a/configure.ac b/configure.ac
index 4f368bcc..832a5b20 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1081,6 +1081,20 @@ AC_ARG_ENABLE(cust1,
 AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes)
 
 
+# A custom strgen that also serves as a sample of how to do
+# SQL-generating strgen's
+AC_ARG_ENABLE(smcustbindcdr,
+        [AS_HELP_STRING([--enable-smcustbindcdr],[Compiles smcustbindcdr module @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_smcustbindcdr="yes" ;;
+          no) enable_smcustbindcdr="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-smcustbindcdr) ;;
+         esac],
+        [enable_smcustbindcdr=no]
+)
+AM_CONDITIONAL(ENABLE_SMCUSTBINDCDR, test x$enable_smcustbindcdr = xyes)
+
+
 # settings for the template input module; copy and modify this code
 # if you intend to add your own module. Be sure to replace imtemplate
 # by the actual name of your module.
@@ -1178,6 +1192,7 @@ AC_CONFIG_FILES([Makefile \
 		plugins/omsnmp/Makefile \
 		plugins/omoracle/Makefile \
 		plugins/omudpspoof/Makefile \
+		plugins/sm_cust_bindcdr/Makefile \
 		plugins/cust1/Makefile \
 		java/Makefile \
 		tests/Makefile])
@@ -1223,6 +1238,9 @@ echo "    pmcisconames module will be compiled:     $enable_pmcisconames"
 echo "    pmaixforwardedfrom module w.be compiled:  $enable_pmaixforwardedfrom"
 echo "    pmsnare module will be compiled:          $enable_pmsnare"
 echo
+echo "---{ strgen modules }---"
+echo "    sm_cust_bindcdr module will be compiled:  $enable_sm_cust_bindcdr"
+echo
 echo "---{ database support }---"
 echo "    MySql support enabled:                    $enable_mysql"
 echo "    libdbi support enabled:                   $enable_libdbi"
diff --git a/plugins/sm_cust_bindcdr/Makefile.am b/plugins/sm_cust_bindcdr/Makefile.am
new file mode 100644
index 00000000..1f71d499
--- /dev/null
+++ b/plugins/sm_cust_bindcdr/Makefile.am
@@ -0,0 +1,6 @@
+pkglib_LTLIBRARIES = sm_cust_bindcdr.la
+
+sm_cust_bindcdr_la_SOURCES = sm_cust_bindcdr.c
+sm_cust_bindcdr_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS)
+sm_cust_bindcdr_la_LDFLAGS = -module -avoid-version
+sm_cust_bindcdr_la_LIBADD = 
diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
new file mode 100644
index 00000000..cf3cdeef
--- /dev/null
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -0,0 +1,138 @@
+/* sm_cust_bindcdr.c
+ * This is a custom developed plugin to process bind information into
+ * a specific SQL statement. While the actual processing may be too specific
+ * to be of general use, this module serves as a template on how this type
+ * of processing can be done.
+ *
+ * Format generated:
+ * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+ * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However,
+ * we have copied over the code from there, it is too simple to go through all the hassle
+ * of having a single code base.
+ *
+ * NOTE: read comments in module-template.h to understand how this file
+ *       works!
+ *
+ * File begun on 2011-03-17 by RGerhards
+ *
+ * Copyright 2011 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include "conf.h"
+#include "syslogd-types.h"
+#include "template.h"
+#include "msg.h"
+#include "module-template.h"
+#include "unicode-helper.h"
+
+MODULE_TYPE_STRGEN
+MODULE_TYPE_NOKEEP
+STRGEN_NAME("Custom_BindCDR")
+
+/* internal structures
+ */
+DEF_SMOD_STATIC_DATA
+
+
+/* config data */
+
+
+/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings
+ * needed (including their length) and then calculating the actual space required. So when we 
+ * finally copy, we know exactly what we need. So we do at most one alloc.
+ */
+BEGINstrgen
+	register int iBuf;
+	uchar *pTimeStamp;
+	size_t lenTimeStamp;
+	uchar *pHOSTNAME;
+	size_t lenHOSTNAME;
+	uchar *pTAG;
+	int lenTAG;
+	uchar *pMSG;
+	size_t lenMSG;
+	size_t lenTotal;
+CODESTARTstrgen
+	/* first obtain all strings and their length (if not fixed) */
+	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
+	lenTimeStamp = ustrlen(pTimeStamp);
+	pHOSTNAME = (uchar*) getHOSTNAME(pMsg);
+	lenHOSTNAME = getHOSTNAMELen(pMsg);
+	getTAG(pMsg, &pTAG, &lenTAG);
+	pMSG = getMSG(pMsg);
+	lenMSG = getMSGLen(pMsg);
+
+	/* calculate len, constants for spaces and similar fixed strings */
+	lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2;
+	if(pMSG[0] != ' ')
+		++lenTotal; /* then we need to introduce one additional space */
+
+	/* now make sure buffer is large enough */
+	if(lenTotal  >= *pLenBuf)
+		CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal));
+
+	/* and concatenate the resulting string */
+	memcpy(*ppBuf, pTimeStamp, lenTimeStamp);
+	iBuf = lenTimeStamp;
+	*(*ppBuf + iBuf++) = ' ';
+
+	memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME);
+	iBuf += lenHOSTNAME;
+	*(*ppBuf + iBuf++) = ' ';
+
+	memcpy(*ppBuf + iBuf, pTAG, lenTAG);
+	iBuf += lenTAG;
+
+	if(pMSG[0] != ' ')
+		*(*ppBuf + iBuf++) = ' ';
+	memcpy(*ppBuf + iBuf, pMSG, lenMSG);
+	iBuf += lenMSG;
+
+	/* trailer */
+	*(*ppBuf + iBuf++) = '\n';
+	*(*ppBuf + iBuf) = '\0';
+
+finalize_it:
+ENDstrgen
+
+
+BEGINmodExit
+CODESTARTmodExit
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_SMOD_QUERIES
+ENDqueryEtryPt
+
+
+BEGINmodInit(sm_cust_bindcdr)
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+
+	dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION);
+ENDmodInit
-- 
cgit 


From 459f0d094fcb48fd58204ad3527f931715839bcb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 17 Mar 2011 14:59:18 +0100
Subject: setup test environment and test code in order to look at...

bug http://bugzilla.adiscon.com/show_bug.cgi?id=195
That bug currently prevents strgen's to be used with databases
---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 45 ++++++++++---------------------
 1 file changed, 14 insertions(+), 31 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index cf3cdeef..5ea7e653 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -63,56 +63,39 @@ DEF_SMOD_STATIC_DATA
  * needed (including their length) and then calculating the actual space required. So when we 
  * finally copy, we know exactly what we need. So we do at most one alloc.
  */
+//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
+#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('"
 BEGINstrgen
 	register int iBuf;
 	uchar *pTimeStamp;
 	size_t lenTimeStamp;
-	uchar *pHOSTNAME;
-	size_t lenHOSTNAME;
-	uchar *pTAG;
-	int lenTAG;
-	uchar *pMSG;
-	size_t lenMSG;
 	size_t lenTotal;
 CODESTARTstrgen
 	/* first obtain all strings and their length (if not fixed) */
 	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
 	lenTimeStamp = ustrlen(pTimeStamp);
-	pHOSTNAME = (uchar*) getHOSTNAME(pMsg);
-	lenHOSTNAME = getHOSTNAMELen(pMsg);
-	getTAG(pMsg, &pTAG, &lenTAG);
-	pMSG = getMSG(pMsg);
-	lenMSG = getMSGLen(pMsg);
 
 	/* calculate len, constants for spaces and similar fixed strings */
-	lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2;
-	if(pMSG[0] != ' ')
-		++lenTotal; /* then we need to introduce one additional space */
+	lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2;
 
 	/* now make sure buffer is large enough */
 	if(lenTotal  >= *pLenBuf)
 		CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal));
 
 	/* and concatenate the resulting string */
-	memcpy(*ppBuf, pTimeStamp, lenTimeStamp);
-	iBuf = lenTimeStamp;
-	*(*ppBuf + iBuf++) = ' ';
+	memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1);
+	iBuf = sizeof(SQL_STMT) - 1;
 
-	memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME);
-	iBuf += lenHOSTNAME;
-	*(*ppBuf + iBuf++) = ' ';
+	// SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); 
 
-	memcpy(*ppBuf + iBuf, pTAG, lenTAG);
-	iBuf += lenTAG;
+	memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp);
+	iBuf += lenTimeStamp;
+	memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1);
+	iBuf += sizeof("', '") - 1;
 
-	if(pMSG[0] != ' ')
-		*(*ppBuf + iBuf++) = ' ';
-	memcpy(*ppBuf + iBuf, pMSG, lenMSG);
-	iBuf += lenMSG;
-
-	/* trailer */
-	*(*ppBuf + iBuf++) = '\n';
-	*(*ppBuf + iBuf) = '\0';
+	/* end of SQL statement/trailer (NUL is contained in string!) */
+	memcpy(*ppBuf + iBuf, "');", sizeof("');"));
+	iBuf += sizeof("');");
 
 finalize_it:
 ENDstrgen
@@ -129,7 +112,7 @@ CODEqueryEtryPt_STD_SMOD_QUERIES
 ENDqueryEtryPt
 
 
-BEGINmodInit(sm_cust_bindcdr)
+BEGINmodInit()
 CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
-- 
cgit 


From e2f13df22c7599fa1fc7d6b4c20081aac78b355a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 21 Mar 2011 09:53:55 +0100
Subject: potential fix to issue that strgen's do not support SQL option

needs testig and verification (wrong system for doing that ;))
---
 template.c | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/template.c b/template.c
index 5aaf1bcb..cc339653 100644
--- a/template.c
+++ b/template.c
@@ -856,19 +856,37 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine)
 {
 	uchar *pSrc, *pDst;
 	uchar szMod[2048];
+	unsigned lenMod;
 	strgen_t *pStrgen;
 	DEFiRet;
 
 	pSrc = *ppRestOfConfLine;
 	pDst = szMod;
-	while(*pSrc && !isspace(*pSrc) && pDst < &(szMod[sizeof(szMod) - 1])) {
-		*pDst++ = *pSrc++;
+	lenMod = 0;
+	while(*pSrc && !isspace(*pSrc) && lenMod < sizeof(szMod) - 1) {
+		szMod[lenMod] = *pSrc++;
+		lenMod++;
+		
 	}
-	*pDst = '\0';
+	szMod[lenMod] = '\0';
 	*ppRestOfConfLine = pSrc;
 	CHKiRet(strgen.FindStrgen(&pStrgen, szMod));
 	pTpl->pStrgen = pStrgen->pModule->mod.sm.strgen;
-	dbgprintf("template bound to strgen '%s'\n", szMod);
+	DBGPRINTF("template bound to strgen '%s'\n", szMod);
+	/* check if the name potentially contains some well-known options
+	 * Note: we have opted to let the name contain all options. This sounds 
+	 * useful, because the strgen MUST actually implement a specific set
+	 * of options. Doing this via the name looks to the enduser as if the
+	 * regular syntax were used, and it make sure the strgen postively
+	 * acknowledged implementing the option. -- rgerhards, 2011-03-21
+	 */
+	if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 4, ",stdsql")) {
+		pTpl->optFormatForSQL = 2;
+		DBGPRINTF("strgen suports the stdsql option\n");
+	} else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 7, ",sql")) {
+		pTpl->optFormatForSQL = 1;
+		DBGPRINTF("strgen suports the sql option\n");
+	}
 
 finalize_it:
 	RETiRet;
@@ -936,8 +954,7 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine)
 		/* we simply make the template defunct in this case by setting
 		 * its name to a zero-string. We do not free it, as this would
 		 * require additional code and causes only a very small memory
-		 * consumption. Memory is freed, however, in normal operation
-		 * and most importantly by HUPing syslogd.
+		 * consumption.
 		 */
 		*pTpl->pszName = '\0';
 		return NULL;
-- 
cgit 


From 41d79b8fead00f85614547606b0c1117f3efa04e Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 21 Mar 2011 10:34:43 +0100
Subject: bugfix: strgen could not be used together with database outputs

because the sql/stdsql option could not be specified. This has been
solved by permitting the strgen to include the opton inside its name.
closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195
---
 ChangeLog                                 | 6 ++++++
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 2 +-
 template.c                                | 4 ++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3503ed69..70d7d322 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 ---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+- new sample plugin for a strgen to generate sql statement consumable
+  by a database plugin
+- bugfix: strgen could not be used together with database outputs
+  because the sql/stdsql option could not be specified. This has been
+  solved by permitting the strgen to include the opton inside its name.
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195
 ---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index 5ea7e653..d1634252 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -49,7 +49,7 @@
 
 MODULE_TYPE_STRGEN
 MODULE_TYPE_NOKEEP
-STRGEN_NAME("Custom_BindCDR")
+STRGEN_NAME("Custom_BindCDR,sql")
 
 /* internal structures
  */
diff --git a/template.c b/template.c
index cc339653..3f6942c2 100644
--- a/template.c
+++ b/template.c
@@ -880,10 +880,10 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine)
 	 * regular syntax were used, and it make sure the strgen postively
 	 * acknowledged implementing the option. -- rgerhards, 2011-03-21
 	 */
-	if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 4, ",stdsql")) {
+	if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) {
 		pTpl->optFormatForSQL = 2;
 		DBGPRINTF("strgen suports the stdsql option\n");
-	} else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 7, ",sql")) {
+	} else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) {
 		pTpl->optFormatForSQL = 1;
 		DBGPRINTF("strgen suports the sql option\n");
 	}
-- 
cgit 


From 5808af91e7e5e552a5cde4603687685978934626 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 12:17:20 +0100
Subject: added internal processing of BIND records

still some questions outstanding, so actual SQL statement is
not usable at the moment
---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 121 ++++++++++++++++++++++++++++--
 1 file changed, 115 insertions(+), 6 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index d1634252..be01db77 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -37,6 +37,7 @@
 #include "config.h"
 #include "rsyslog.h"
 #include <stdlib.h>
+#include <ctype.h>
 #include <string.h>
 #include <assert.h>
 #include <errno.h>
@@ -64,19 +65,112 @@ DEF_SMOD_STATIC_DATA
  * finally copy, we know exactly what we need. So we do at most one alloc.
  */
 //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
-#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('"
+//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('"
+#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('"
+#define ADD_SQL_DELIM \
+	memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \
+	iBuf += sizeof("', '") - 1;
+#define SQL_STMT_END "');\n"
 BEGINstrgen
 	register int iBuf;
+	uchar *psz;
 	uchar *pTimeStamp;
+	uchar szClient[64];
+	unsigned lenClient;
+	uchar szView[64];
+	unsigned lenView;
+	uchar szQuery[64];
+	unsigned lenQuery;
+	uchar szIP[64];
+	unsigned lenIP;
 	size_t lenTimeStamp;
 	size_t lenTotal;
 CODESTARTstrgen
+	/* first create an empty statement. This is to be replaced if
+	 * we have better data to fill in.
+	 */
+	/* now make sure buffer is large enough */
+	if(*pLenBuf < 2)
+		CHKiRet(ExtendBuf(ppBuf, pLenBuf, 2));
+	memcpy(*ppBuf, ";", sizeof(";"));
+
 	/* first obtain all strings and their length (if not fixed) */
 	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
 	lenTimeStamp = ustrlen(pTimeStamp);
+	
+	/* "client" */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "client ");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: client part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("client ") - 1; /* skip "label" */
+		for(  lenClient = 0
+		    ; *psz && *psz != '#' && lenClient < sizeof(szClient) - 1
+		    ; ++lenClient) {
+			szClient[lenClient] = *psz++;
+		}
+		szClient[lenClient] = '\0';
+	}
+
+	/* "view" */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "view ");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: view part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("view ") - 1; /* skip "label" */
+		for(  lenView = 0
+		    ; *psz && *psz != ':' && lenView < sizeof(szView) - 1
+		    ; ++lenView) {
+			szView[lenView] = *psz++;
+		}
+		szView[lenView] = '\0';
+	}
+
+	/* "query" - we must extract just the number, and in reverse! */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "query: ");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: query part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("query: ") - 1; /* skip "label" */
+		/* first find end-of-string to process */
+		while(*psz && (isdigit(*psz) || *psz == '.')) {
+dbgprintf("XXXX: step 1: %c\n", *psz);
+			psz++;
+		}
+		/* now shuffle data */
+		for(  lenQuery = 0
+		    ; *psz && *psz != ' ' && lenQuery < sizeof(szQuery) - 1
+		    ; --psz) {
+			if(isdigit(*psz))
+				szQuery[lenQuery++] = *psz;
+		}
+		szQuery[lenQuery] = '\0';
+	}
+
+	/* "ip" */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + (");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: ip part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("IN TXT + (") - 1; /* skip "label" */
+		for(  lenIP = 0
+		    ; *psz && *psz != ')' && lenIP < sizeof(szIP) - 1
+		    ; ++lenIP) {
+			szIP[lenIP] = *psz++;
+		}
+		szIP[lenIP] = '\0';
+	}
+
+
+	/* --- strings extracted ---- */
 
 	/* calculate len, constants for spaces and similar fixed strings */
-	lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2;
+	lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5
+		   + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
 
 	/* now make sure buffer is large enough */
 	if(lenTotal  >= *pLenBuf)
@@ -90,12 +184,27 @@ CODESTARTstrgen
 
 	memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp);
 	iBuf += lenTimeStamp;
-	memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1);
-	iBuf += sizeof("', '") - 1;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szClient, lenClient);
+	iBuf += lenClient;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szView, lenView);
+	iBuf += lenView;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szQuery, lenQuery);
+	iBuf += lenQuery;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szIP, lenIP);
+	iBuf += lenIP;
+	ADD_SQL_DELIM
 
 	/* end of SQL statement/trailer (NUL is contained in string!) */
-	memcpy(*ppBuf + iBuf, "');", sizeof("');"));
-	iBuf += sizeof("');");
+	memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END));
+	iBuf += sizeof(SQL_STMT_END);
 
 finalize_it:
 ENDstrgen
-- 
cgit 


From 0d80f4e4612fb209cc9dda5faf7b71cd261c0c20 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 14:22:38 +0100
Subject: sm_bind_cdr: added capability to configure "allowed IPs"

---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 83 +++++++++++++++++++++++++++++--
 1 file changed, 80 insertions(+), 3 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index be01db77..005a4ba9 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -43,10 +43,12 @@
 #include <errno.h>
 #include "conf.h"
 #include "syslogd-types.h"
+#include "cfsysline.h"
 #include "template.h"
 #include "msg.h"
 #include "module-template.h"
 #include "unicode-helper.h"
+#include "errmsg.h"
 
 MODULE_TYPE_STRGEN
 MODULE_TYPE_NOKEEP
@@ -55,10 +57,65 @@ STRGEN_NAME("Custom_BindCDR,sql")
 /* internal structures
  */
 DEF_SMOD_STATIC_DATA
+DEFobjCurrIf(errmsg)
+
+/* list of "allowed" IPs */
+typedef struct allowedip_s {
+	uchar *pszIP;
+	struct allowedip_s *next;
+} allowedip_t;
+
+static allowedip_t *root;
 
 
 /* config data */
 
+/* check if the provided IP is (already) in the allowed list
+ */
+static int
+isAllowed(uchar *pszIP)
+{
+	allowedip_t *pallow;
+	int ret = 0;
+
+	for(pallow = root ; pallow != NULL ; pallow = pallow->next) {
+		DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP);
+		if(!ustrcmp(pallow->pszIP, pszIP)) {
+			ret = 1;
+			goto finalize_it;
+		}
+	}
+finalize_it: return ret;
+}
+
+/* This function is called to add an additional allowed IP. It adds
+ * the IP to the linked list of them. An error is emitted if the IP
+ * already exists.
+ */
+static rsRetVal addAllowedIP(void __attribute__((unused)) *pVal, uchar *pNewVal)
+{
+	allowedip_t *pNew;
+	DEFiRet;
+
+	if(isAllowed(pNewVal)) {
+		errmsg.LogError(0, NO_ERRCODE, "error: allowed IP '%s' already configured "
+				"duplicate ignored", pNewVal);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	CHKmalloc(pNew = malloc(sizeof(allowedip_t)));
+	pNew->pszIP = pNewVal;
+	pNew->next = root;
+	root = pNew;
+	DBGPRINTF("sm_cust_bindcdr: allowed IP '%s' added.\n", pNewVal);
+
+finalize_it:
+	if(iRet != RS_RET_OK) {
+		free(pNewVal);
+	}
+
+	RETiRet;
+}
 
 /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings
  * needed (including their length) and then calculating the actual space required. So when we 
@@ -72,9 +129,10 @@ DEF_SMOD_STATIC_DATA
 	iBuf += sizeof("', '") - 1;
 #define SQL_STMT_END "');\n"
 BEGINstrgen
-	register int iBuf;
+	int iBuf;
 	uchar *psz;
 	uchar *pTimeStamp;
+	size_t lenTimeStamp;
 	uchar szClient[64];
 	unsigned lenClient;
 	uchar szView[64];
@@ -83,7 +141,6 @@ BEGINstrgen
 	unsigned lenQuery;
 	uchar szIP[64];
 	unsigned lenIP;
-	size_t lenTimeStamp;
 	size_t lenTotal;
 CODESTARTstrgen
 	/* first create an empty statement. This is to be replaced if
@@ -137,7 +194,6 @@ CODESTARTstrgen
 		psz += sizeof("query: ") - 1; /* skip "label" */
 		/* first find end-of-string to process */
 		while(*psz && (isdigit(*psz) || *psz == '.')) {
-dbgprintf("XXXX: step 1: %c\n", *psz);
 			psz++;
 		}
 		/* now shuffle data */
@@ -168,6 +224,14 @@ dbgprintf("XXXX: step 1: %c\n", *psz);
 
 	/* --- strings extracted ---- */
 
+	/* now check if the IP is "allowed", in which case we should not
+	 * insert into the database.
+	 */
+	if(isAllowed(szIP)) {
+		DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n");
+		FINALIZE;
+	}
+
 	/* calculate len, constants for spaces and similar fixed strings */
 	lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5
 		   + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
@@ -211,7 +275,16 @@ ENDstrgen
 
 
 BEGINmodExit
+	allowedip_t *pallow, *pdel;
 CODESTARTmodExit
+	for(pallow = root ; pallow != NULL ; ) {
+		pdel = pallow;
+		pallow = pallow->next;
+		free(pdel->pszIP);
+		free(pdel);
+	}
+
+	objRelease(errmsg, CORE_COMPONENT);
 ENDmodExit
 
 
@@ -225,6 +298,10 @@ BEGINmodInit()
 CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 
+	root = NULL;
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"sgcustombindcdrallowedip", 0, eCmdHdlrGetWord,
+		addAllowedIP, NULL, STD_LOADABLE_MODULE_ID));
 	dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION);
 ENDmodInit
-- 
cgit 


From 27ab3768e2eaefaed7054144939f0730b3fac84f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 15:02:15 +0100
Subject: bugfix: RFC5424 parser confused by empty structured data

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
---
 ChangeLog         | 2 ++
 tools/pmrfc5424.c | 3 +--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 13f6608d..de55c799 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5 @@
+- bugfix: RFC5424 parser confused by empty structured data
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
 ---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c
index 2bd18042..b06f1347 100644
--- a/tools/pmrfc5424.c
+++ b/tools/pmrfc5424.c
@@ -142,7 +142,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr
 	 * structured data. There may also be \] inside the structured data, which
 	 * do NOT terminate an element.
 	 */
-	if(lenStr == 0 || *p2parse != '[')
+	if(lenStr == 0 || (*p2parse != '[' && *p2parse != '-'))
 		return 1; /* this is NOT structured data! */
 
 	if(*p2parse == '-') { /* empty structured data? */
@@ -211,7 +211,6 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr
  *
  * rger, 2005-11-24
  */
-//static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 BEGINparse
 	uchar *p2parse;
 	uchar *pBuf = NULL;
-- 
cgit 


From e69efbb6d7907339ef81a2062f858eb082233f61 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 16:55:11 +0100
Subject: enhance: added $BOM system property to ease writing byte order masks

---
 ChangeLog                  |  1 +
 doc/property_replacer.html |  6 ++++++
 runtime/msg.c              | 10 ++++++++++
 runtime/rsyslog.h          |  1 +
 4 files changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index de55c799..846d2850 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,4 @@
+- enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
 ---------------------------------------------------------------------------
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 4d242a34..cd357f67 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -156,6 +156,12 @@ than messages generated somewhere.
 </td>
 </tr>
 <tr>
+<td><b>$bom</b></td>
+<td>The UTF-8 encoded Unicode byte-order mask (BOM). This may be useful
+in templates for RFC5424 support, when the character set is know to be
+Unicode.</td>
+</tr>
+<tr>
 <td><b>$now</b></td>
 <td>The current date stamp in the format YYYY-MM-DD</td>
 </tr>
diff --git a/runtime/msg.c b/runtime/msg.c
index b0261faa..e8e60963 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -444,6 +444,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID)
 		*pPropID = PROP_SYS_MINUTE;
 	} else if(!strcmp((char*) pName, "$myhostname")) {
 		*pPropID = PROP_SYS_MYHOSTNAME;
+	} else if(!strcmp((char*) pName, "$bom")) {
+		*pPropID = PROP_SYS_BOM;
 	} else {
 		*pPropID = PROP_INVALID;
 		iRet = RS_RET_VAR_NOT_FOUND;
@@ -525,6 +527,8 @@ uchar *propIDToName(propid_t propID)
 			return UCHAR_CONSTANT("$MINUTE");
 		case PROP_SYS_MYHOSTNAME:
 			return UCHAR_CONSTANT("$MYHOSTNAME");
+		case PROP_SYS_BOM:
+			return UCHAR_CONSTANT("$BOM");
 		default:
 			return UCHAR_CONSTANT("*invalid property id*");
 	}
@@ -2427,6 +2431,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 		case PROP_SYS_MYHOSTNAME:
 			pRes = glbl.GetLocalHostName();
 			break;
+		case PROP_SYS_BOM:
+			if(*pbMustBeFreed == 1)
+				free(pRes);
+			pRes = (uchar*) "\xEF\xBB\xBF";
+			*pbMustBeFreed = 0;
+			break;
 		default:
 			/* there is no point in continuing, we may even otherwise render the
 			 * error message unreadable. rgerhards, 2007-07-10
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 78e61bcb..d63dbe4f 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -137,6 +137,7 @@ typedef uintTiny	propid_t;
 #define PROP_SYS_QHOUR			156
 #define PROP_SYS_MINUTE			157
 #define PROP_SYS_MYHOSTNAME		158
+#define PROP_SYS_BOM			159
 
 
 /* The error codes below are orginally "borrowed" from
-- 
cgit 


From eefc9b6eb69c130015955cfb15dc45e36e431d5a Mon Sep 17 00:00:00 2001
From: Corey Smith <corsmith@gmail.com>
Date: Thu, 24 Mar 2011 17:25:03 +0100
Subject: bugfix: PRI was invalid on Solaris for message from local log socket

Signed-off-by: root <root@localhost.localdomain>
---
 ChangeLog        | 1 +
 runtime/parser.c | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 846d2850..dc6b9872 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,4 @@
+- bugfix: PRI was invalid on Solaris for message from local log socket
 - enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
diff --git a/runtime/parser.c b/runtime/parser.c
index d3644976..b385c54b 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -453,10 +453,10 @@ ParsePRI(msg_t *pMsg)
 			if(pri & ~(LOG_FACMASK|LOG_PRIMASK))
 				pri = DEFUPRI;
 		}
+		pMsg->iFacility = LOG_FAC(pri);
+		pMsg->iSeverity = LOG_PRI(pri);
+		MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg);
 	}
-	pMsg->iFacility = LOG_FAC(pri);
-	pMsg->iSeverity = LOG_PRI(pri);
-	MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg);
 	RETiRet;
 }
 
-- 
cgit 


From 0fcbbfbe3b40e2739531cfa3a308c6fc8681ee7f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 28 Mar 2011 16:09:07 +0200
Subject: sm_cust_bindcdr: custom date parsing added

---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 100 +++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 10 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index 005a4ba9..fa16acb5 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -120,6 +120,8 @@ finalize_it:
 /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings
  * needed (including their length) and then calculating the actual space required. So when we 
  * finally copy, we know exactly what we need. So we do at most one alloc.
+ * An actual message sample for what we intend to parse is (one line):
+  <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96)
  */
 //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
 //#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('"
@@ -131,8 +133,12 @@ finalize_it:
 BEGINstrgen
 	int iBuf;
 	uchar *psz;
-	uchar *pTimeStamp;
-	size_t lenTimeStamp;
+	uchar szDate[64];
+	unsigned lenDate;
+	uchar szTime[64];
+	unsigned lenTime;
+	uchar szMSec[64];
+	unsigned lenMSec;
 	uchar szClient[64];
 	unsigned lenClient;
 	uchar szView[64];
@@ -152,9 +158,76 @@ CODESTARTstrgen
 	memcpy(*ppBuf, ";", sizeof(";"));
 
 	/* first obtain all strings and their length (if not fixed) */
-	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
-	lenTimeStamp = ustrlen(pTimeStamp);
-	
+	/* Note that there are two date fields present, one in the header
+	 * and one more in the actual message. We use the one from the message
+	 * and parse that our. We check validity based on some fixe fields. In-
+	 * depth verification is probably not worth the effort (CPU time), because
+	 * we do various other checks on the message format below).
+	 */
+	psz = getMSG(pMsg);
+	if(psz[0] == ' ' && psz[3] == '-' && psz[7] == '-') {
+		memcpy(szDate, psz+8,  4);
+		szDate[4] = '-';
+		if(!strncmp((char*)psz+4, "Jan", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '1';
+		} else if(!strncmp((char*)psz+4, "Feb", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '2';
+		} else if(!strncmp((char*)psz+4, "Mar", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '3';
+		} else if(!strncmp((char*)psz+4, "Apr", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '4';
+		} else if(!strncmp((char*)psz+4, "May", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '5';
+		} else if(!strncmp((char*)psz+4, "Jun", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '6';
+		} else if(!strncmp((char*)psz+4, "Jul", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '7';
+		} else if(!strncmp((char*)psz+4, "Aug", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '8';
+		} else if(!strncmp((char*)psz+4, "Sep", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '9';
+		} else if(!strncmp((char*)psz+4, "Oct", 3)) {
+			szDate[5] = '1';
+			szDate[6] = '0';
+		} else if(!strncmp((char*)psz+4, "Nov", 3)) {
+			szDate[5] = '1';
+			szDate[6] = '1';
+		} else if(!strncmp((char*)psz+4, "Dec", 3)) {
+			szDate[5] = '1';
+			szDate[6] = '2';
+		}
+		szDate[7] = '-';
+		szDate[8] = psz[1];
+		szDate[9] = psz[2];
+		szDate[10] = '\0';
+		lenDate = 10;
+	} else {
+		dbgprintf("Custom_BindCDR: date part in msg missing\n");
+		FINALIZE;
+	}
+
+	/* now time (pull both regular time and ms) */
+	if(psz[12] == ' ' && psz[15] == ':' && psz[18] == ':' && psz[21] == '.' && psz[25] == ' ') {
+		memcpy(szTime, (char*)psz+13, 8);
+		szTime[9] = '\0';
+		lenTime = 8;
+		memcpy(szMSec, (char*)psz+22, 3);
+		szMSec[4] = '\0';
+		lenMSec = 3;
+	} else {
+		dbgprintf("Custom_BindCDR: date part in msg missing\n");
+		FINALIZE;
+	}
+
 	/* "client" */
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "client ");
 	if(psz == NULL) {
@@ -233,8 +306,8 @@ CODESTARTstrgen
 	}
 
 	/* calculate len, constants for spaces and similar fixed strings */
-	lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5
-		   + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
+	lenTotal = lenDate + lenTime + lenMSec + lenClient + lenView + lenQuery
+		   + lenIP + 7 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
 
 	/* now make sure buffer is large enough */
 	if(lenTotal  >= *pLenBuf)
@@ -246,8 +319,16 @@ CODESTARTstrgen
 
 	// SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); 
 
-	memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp);
-	iBuf += lenTimeStamp;
+	memcpy(*ppBuf + iBuf, szDate, lenDate);
+	iBuf += lenDate;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szTime, lenTime);
+	iBuf += lenTime;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szMSec, lenMSec);
+	iBuf += lenMSec;
 	ADD_SQL_DELIM
 
 	memcpy(*ppBuf + iBuf, szClient, lenClient);
@@ -264,7 +345,6 @@ CODESTARTstrgen
 
 	memcpy(*ppBuf + iBuf, szIP, lenIP);
 	iBuf += lenIP;
-	ADD_SQL_DELIM
 
 	/* end of SQL statement/trailer (NUL is contained in string!) */
 	memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END));
-- 
cgit 


From 6f9773f1faa79c87c4c35c679455151d4e6d9e00 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 28 Mar 2011 17:40:15 +0200
Subject: bugfix: rsyslog did not build with --disable-regexp configure option

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
---
 ChangeLog  |  2 ++
 template.c | 12 +++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index dc6b9872..cbc91b87 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5 @@
+- bugfix: rsyslog did not build with --disable-regexp configure option
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
 - bugfix: PRI was invalid on Solaris for message from local log socket
 - enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
diff --git a/template.c b/template.c
index cc339653..1e1a6c3f 100644
--- a/template.c
+++ b/template.c
@@ -42,10 +42,13 @@
 /* static data */
 DEFobjCurrIf(obj)
 DEFobjCurrIf(errmsg)
-DEFobjCurrIf(regexp)
 DEFobjCurrIf(strgen)
 
+#ifdef FEATURE_REGEXP
+DEFobjCurrIf(regexp)
 static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */
+#endif
+
 static struct template *tplRoot = NULL;	/* the root of the template list */
 static struct template *tplLast = NULL;	/* points to the last element of the template list */
 static struct template *tplLastStatic = NULL; /* last static element of the template list */
@@ -549,10 +552,9 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 	cstr_t *pStrB;
 	struct templateEntry *pTpe;
 	int iNum;	/* to compute numbers */
-	rsRetVal iRetLocal;
-
 #ifdef FEATURE_REGEXP
 	/* APR: variables for regex */
+	rsRetVal iRetLocal;
 	int longitud;
 	unsigned char *regex_char;
 	unsigned char *regex_end;
@@ -1084,11 +1086,13 @@ void tplDeleteAll(void)
 				break;
 			case FIELD:
 				/* check if we have a regexp and, if so, delete it */
+#ifdef FEATURE_REGEXP
 				if(pTpeDel->data.field.has_regex != 0) {
 					if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) {
 						regexp.regfree(&(pTpeDel->data.field.re));
 					}
 				}
+#endif
 				break;
 			}
 			/*dbgprintf("\n");*/
@@ -1137,12 +1141,14 @@ void tplDeleteNew(void)
 				free(pTpeDel->data.constant.pConstant);
 				break;
 			case FIELD:
+#ifdef FEATURE_REGEXP
 				/* check if we have a regexp and, if so, delete it */
 				if(pTpeDel->data.field.has_regex != 0) {
 					if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) {
 						regexp.regfree(&(pTpeDel->data.field.re));
 					}
 				}
+#endif
 				break;
 			}
 			/*dbgprintf("\n");*/
-- 
cgit 


From d2cce9852bdd9ba9f93ac94d6c8b0178d5b82bb0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 11:48:45 +0200
Subject: bugfix: error return from strgen caused abort, now causes action to
 be ignored

...(just like a failed filter)
---
 ChangeLog | 2 ++
 action.c  | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 70d7d322..7d868d64 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+- bugfix: error return from strgen caused abort, now causes action to be
+  ignored (just like a failed filter)
 - new sample plugin for a strgen to generate sql statement consumable
   by a database plugin
 - bugfix: strgen could not be used together with database outputs
diff --git a/action.c b/action.c
index ea4358fb..c5bd03cb 100644
--- a/action.c
+++ b/action.c
@@ -1071,7 +1071,8 @@ prepareBatch(action_t *pAction, batch_t *pBatch)
 		pElem = &(pBatch->pElem[i]);
 		if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) {
 			pElem->state = BATCH_STATE_RDY;
-			prepareDoActionParams(pAction, pElem);
+			if(prepareDoActionParams(pAction, pElem) != RS_RET_OK)
+				pElem->bFilterOK = FALSE;
 		}
 	}
 	RETiRet;
-- 
cgit 


From 030fd145a3d8759367b73cc66e03023ff131c951 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 11:50:32 +0200
Subject: did mapping of data items to database table columns

to facilitate testing
---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index fa16acb5..3fe96ac4 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -79,7 +79,6 @@ isAllowed(uchar *pszIP)
 	int ret = 0;
 
 	for(pallow = root ; pallow != NULL ; pallow = pallow->next) {
-		DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP);
 		if(!ustrcmp(pallow->pszIP, pszIP)) {
 			ret = 1;
 			goto finalize_it;
@@ -123,9 +122,7 @@ finalize_it:
  * An actual message sample for what we intend to parse is (one line):
   <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96)
  */
-//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
-//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('"
-#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('"
+#define SQL_STMT "INSERT INTO CDR(`Date`,`Time`, timeMS, client, view, query, ip) VALUES ('"
 #define ADD_SQL_DELIM \
 	memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \
 	iBuf += sizeof("', '") - 1;
@@ -212,7 +209,7 @@ CODESTARTstrgen
 		lenDate = 10;
 	} else {
 		dbgprintf("Custom_BindCDR: date part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	/* now time (pull both regular time and ms) */
@@ -225,14 +222,14 @@ CODESTARTstrgen
 		lenMSec = 3;
 	} else {
 		dbgprintf("Custom_BindCDR: date part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	/* "client" */
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "client ");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: client part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("client ") - 1; /* skip "label" */
 		for(  lenClient = 0
@@ -247,7 +244,7 @@ CODESTARTstrgen
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "view ");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: view part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("view ") - 1; /* skip "label" */
 		for(  lenView = 0
@@ -262,10 +259,10 @@ CODESTARTstrgen
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "query: ");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: query part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("query: ") - 1; /* skip "label" */
-		/* first find end-of-string to process */
+		/* first find end-of-strihttp://www.rsyslog.com/doc/omruleset.htmlng to process */
 		while(*psz && (isdigit(*psz) || *psz == '.')) {
 			psz++;
 		}
@@ -283,7 +280,7 @@ CODESTARTstrgen
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + (");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: ip part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("IN TXT + (") - 1; /* skip "label" */
 		for(  lenIP = 0
@@ -302,7 +299,7 @@ CODESTARTstrgen
 	 */
 	if(isAllowed(szIP)) {
 		DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	/* calculate len, constants for spaces and similar fixed strings */
@@ -317,8 +314,6 @@ CODESTARTstrgen
 	memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1);
 	iBuf = sizeof(SQL_STMT) - 1;
 
-	// SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); 
-
 	memcpy(*ppBuf + iBuf, szDate, lenDate);
 	iBuf += lenDate;
 	ADD_SQL_DELIM
-- 
cgit 


From 12f126a811dd08553ac9387392c3a506eba3fe55 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 11:59:22 +0200
Subject: manually merging ChangeLog

---
 ChangeLog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d59871fe..fdd45aa1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,11 @@
+---------------------------------------------------------------------------
+Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
 - bugfix: rsyslog did not build with --disable-regexp configure option
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
 - bugfix: PRI was invalid on Solaris for message from local log socket
 - enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
----------------------------------------------------------------------------
-Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
 - bugfix: error return from strgen caused abort, now causes action to be
   ignored (just like a failed filter)
 - new sample plugin for a strgen to generate sql statement consumable
-- 
cgit 


From 9003632c7df5e6020b06b7d19e1c226ea46640de Mon Sep 17 00:00:00 2001
From: Christian Kastner <debian@kvr.at>
Date: Tue, 29 Mar 2011 12:00:56 +0200
Subject: Force ANSI SQL treatment of strings for Postgres

Signed-off-by: Rainer Gerhards <rgerhards@adiscon.com>
---
 plugins/ompgsql/ompgsql.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c
index ab8e4d2c..ea4b4b75 100644
--- a/plugins/ompgsql/ompgsql.c
+++ b/plugins/ompgsql/ompgsql.c
@@ -149,8 +149,13 @@ static rsRetVal initPgSQL(instanceData *pData, int bSilent)
 
 	dbgprintf("host=%s dbname=%s uid=%s\n",pData->f_dbsrv,pData->f_dbname,pData->f_dbuid);
 
+	/* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may
+	 * get all sorts of side effects (e.g.: backslash escapes) and warnings
+	 */
+	const char *PgConnectionOptions = "-c standard_conforming_strings=on";
+
 	/* Connect to database */
-	if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, NULL, NULL,
+	if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, PgConnectionOptions, NULL,
 				pData->f_dbname, pData->f_dbuid, pData->f_dbpwd)) == NULL) {
 		reportDBError(pData, bSilent);
 		closePgSQL(pData); /* ignore any error we may get */
-- 
cgit 


From c091e8491c0fa100dce951c09f8da77ee833059a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 12:02:52 +0200
Subject: doc: adding bugfix to ChangeLog

---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index fdd45aa1..90bc0f2d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+- bugfix: ompgsql did not work properly with ANSI SQL strings
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229
 - bugfix: rsyslog did not build with --disable-regexp configure option
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
 - bugfix: PRI was invalid on Solaris for message from local log socket
-- 
cgit 


From 0fb11a5a11e3dcb389d6139e24b0d8aa72164dd6 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 14:22:28 +0200
Subject: preparing for 5.7.10

---
 ChangeLog       | 2 +-
 configure.ac    | 2 +-
 doc/manual.html | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 90bc0f2d..e4c25889 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,5 @@
 ---------------------------------------------------------------------------
-Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-29
 - bugfix: ompgsql did not work properly with ANSI SQL strings
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229
 - bugfix: rsyslog did not build with --disable-regexp configure option
diff --git a/configure.ac b/configure.ac
index 832a5b20..2e563490 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[5.7.9],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[5.7.10],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
diff --git a/doc/manual.html b/doc/manual.html
index 807ac0da..f28d18d2 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 5.7.9 (beta branch) of rsyslog.</b>
+<p><b>This documentation is for version 5.7.10 (beta branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
-- 
cgit 


From 5bfa8d2a254e0ead0608b954bdab6dd2a7d9f3e0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 14:31:14 +0200
Subject: testbench samples incorrect, corrected

this was due to the old RFC5424 parser being considered correct,
what it was not. As such, the samples generated from it were
incorrect as well. Now with the corrected parsers, the tests failed.
---
 tests/testsuites/rfc5424-1.parse1 | 2 +-
 tests/testsuites/rfc5424-2.parse1 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1
index 23836c9f..8a66d7ba 100644
--- a/tests/testsuites/rfc5424-1.parse1
+++ b/tests/testsuites/rfc5424-1.parse1
@@ -1,3 +1,3 @@
 #Example from RFC5424, section 6.5 / sample 1
 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8
-34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8
+34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,BOM'su root' failed for lonvick on /dev/pts/8
diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1
index a86fbc35..f5449e68 100644
--- a/tests/testsuites/rfc5424-2.parse1
+++ b/tests/testsuites/rfc5424-2.parse1
@@ -1,4 +1,4 @@
 <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts.
-165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts.
+165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],%% It's time to make the do-nuts.
 #Example from RFC5424, section 6.5 / sample 2
 #Only the first two lines are important, you may place anything behind them!
-- 
cgit 


From b59f8fd65d4f464ce3673439725db1606ec31329 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 30 Mar 2011 11:19:22 +0200
Subject: fixed problem in testbench & added new test (not yet integrated)

The test is a setup scenario for this bug tracker:
http://bugzilla.adiscon.com/show_bug.cgi?id=241
---
 plugins/imdiag/imdiag.c                        |  5 +-
 tests/diag.sh                                  | 14 +++-
 tests/diagtalker.c                             |  6 +-
 tests/rcvr_fail_restore.sh                     | 96 ++++++++++++++++++++++++++
 tests/testsuites/diag-common.conf              |  5 +-
 tests/testsuites/rcvr_fail_restore_rcvr.conf   |  8 +++
 tests/testsuites/rcvr_fail_restore_sender.conf | 15 ++++
 7 files changed, 143 insertions(+), 6 deletions(-)
 create mode 100755 tests/rcvr_fail_restore.sh
 create mode 100644 tests/testsuites/rcvr_fail_restore_rcvr.conf
 create mode 100644 tests/testsuites/rcvr_fail_restore_sender.conf

diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c
index 404cebc7..ed6ef509 100644
--- a/plugins/imdiag/imdiag.c
+++ b/plugins/imdiag/imdiag.c
@@ -205,7 +205,7 @@ doInjectMsg(int iNum)
 	DEFiRet;
 
 	snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar),
-		 "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum);
+		 "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum);
 
 	datetime.getCurrTime(&stTime, &ttGenTime);
 	/* we now create our own message object and submit it to the queue */
@@ -247,6 +247,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess)
 	}
 
 	CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs));
+	DBGPRINTF("imdiag: %d messages injected\n", nMsgs);
 
 finalize_it:
 	RETiRet;
@@ -279,6 +280,7 @@ waitMainQEmpty(tcps_sess_t *pSess)
 	}
 
 	CHKiRet(sendResponse(pSess, "mainqueue empty\n"));
+	DBGPRINTF("imdiag: mainqueue empty\n");
 
 finalize_it:
 	RETiRet;
@@ -314,6 +316,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg)
 	if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) {
 		CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize));
 		CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize));
+		DBGPRINTF("imdiag: %d messages in main queue\n", iMsgQueueSize);
 	} else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) {
 		CHKiRet(waitMainQEmpty(pSess));
 	} else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) {
diff --git a/tests/diag.sh b/tests/diag.sh
index 8fbe42ed..d1242fb1 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -84,15 +84,27 @@ case $1 in
 			exit 1
 		fi
 		;;
+   'get-mainqueuesize') # show the current main queue size
+		if [ "$2" == "2" ]
+		then
+			echo getmainmsgqueuesize | ./diagtalker -p13501
+		else
+			echo getmainmsgqueuesize | ./diagtalker
+		fi
+		;;
    'wait-queueempty') # wait for main message queue to be empty. $2 is the instance.
 		if [ "$2" == "2" ]
 		then
-			echo WaitMainQueueEmpty | ./diagtalker
+			echo WaitMainQueueEmpty | ./diagtalker -p13501
 		else
 			echo WaitMainQueueEmpty | ./diagtalker
 		fi
 		;;
    'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance.
+		if [ "$2" == "2" ]
+		then
+		   echo Shutting down instance 2
+		fi
    		$srcdir/diag.sh wait-queueempty $2
 		kill `cat rsyslog$2.pid`
 		# note: we do not wait for the actual termination!
diff --git a/tests/diagtalker.c b/tests/diagtalker.c
index 6a721e47..0d728bda 100644
--- a/tests/diagtalker.c
+++ b/tests/diagtalker.c
@@ -1,7 +1,7 @@
 /* A yet very simple tool to talk to imdiag (this replaces the
  * previous Java implementation in order to get fewer dependencies).
  *
- * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2010,2011 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -126,7 +126,7 @@ doProcessing()
 		len = strlen(line);
 		sendCmd(fd, line, len);
 		waitRsp(fd, line, sizeof(line));
-		printf("imdiag: %s", line);
+		printf("imdiag[%d]: %s", targetPort, line);
 	}
 }
 
@@ -139,7 +139,7 @@ int main(int argc, char *argv[])
 	int ret = 0;
 	int opt;
 
-	while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) {
+	while((opt = getopt(argc, argv, "t:p:")) != -1) {
 		switch (opt) {
 		case 't':	targetIP = optarg;
 				break;
diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh
new file mode 100755
index 00000000..e2f0a0d8
--- /dev/null
+++ b/tests/rcvr_fail_restore.sh
@@ -0,0 +1,96 @@
+# Copyright (C) 2011 by Rainer Gerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo \[rcvr_fail_restore.sh\]: test failed receiver restore case
+source $srcdir/diag.sh init
+#
+# STEP1: start both instances and send 1000 messages.
+# Note: receiver is instance 2, sender instance 1.
+#
+# start up the instances. Note that the envrionment settings can be changed to
+# set instance-specific debugging parameters!
+export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log2"
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+export RSYSLOG_DEBUGLOG="log"
+#valgrind="valgrind"
+source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf
+# re-set params so that new instances do not thrash it...
+unset RSYSLOG_DEBUG
+unset RSYSLOG_DEBUGLOG
+
+# now inject the messages into instance 2. It will connect to instance 1,
+# and that instance will record the data.
+source $srcdir/diag.sh injectmsg  1 1000
+source $srcdir/diag.sh wait-queueempty
+./msleep 3000 # make sure some retries happen (retry interval is set to 1 second)
+#
+# Step 2: shutdown receiver, then send some more data, which then
+# needs to go into the queue.
+#
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+source $srcdir/diag.sh injectmsg  1001 10000
+sleep 1 # we need to wait, otherwise we may be so fast that the receiver
+# comes up before we have finally suspended the action
+source $srcdir/diag.sh get-mainqueuesize
+ls -l test-spool
+
+#
+# Step 3: restart receiver, wait that the sender drains its queue
+#
+#export RSYSLOG_DEBUGLOG="log2"
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+echo waiting for sender to drain queue [may need a short while]
+source $srcdir/diag.sh wait-queueempty
+ls -l test-spool
+OLDFILESIZE=$(stat -c%s test-spool/mainq.00000001)
+echo file size to expect is $OLDFILESIZE
+
+
+#
+# Step 4: send new data. Queue files are not permitted to grow now
+# (but one file continous to exist).
+#
+source $srcdir/diag.sh injectmsg  11001 10
+source $srcdir/diag.sh wait-queueempty
+
+# at this point, the queue file shall not have grown. Note
+# that we MUST NOT shut down the instance right now, because it
+# would clean up the queue files! So we need to do our checks
+# first (here!).
+ls -l test-spool
+NEWFILESIZE=$(stat -c%s test-spool/mainq.00000001)
+if [ $NEWFILESIZE != $OLDFILESIZE ]
+then
+   echo file sizes do not match, expected $OLDFILESIZE, actual $NEWFILESIZE
+   echo this means that data has been written to the queue file where it
+   echo no longer should be written.
+   # abort will happen below, because we must ensure proper system shutdown
+   # HOWEVER, during actual testing it may be useful to do an exit here (so
+   # that e.g. the debug log is pointed right at the correct spot).
+   # exit 1
+fi
+
+
+#
+# Queue file size checks done. Now it is time to terminate the system
+# and see if everything could be received (the usual check, done here
+# for completeness, more or less as a bonus).
+#
+source $srcdir/diag.sh shutdown-when-empty
+source $srcdir/diag.sh wait-shutdown
+
+# now it is time to stop the receiver as well
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+# now abort test if we need to (due to filesize predicate)
+if [ $NEWFILESIZE != $OLDFILESIZE ]
+then
+   exit 1
+fi
+# do the final check
+source $srcdir/diag.sh seq-check 1 11010
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf
index 9e9e28fe..d76e2d15 100644
--- a/tests/testsuites/diag-common.conf
+++ b/tests/testsuites/diag-common.conf
@@ -13,4 +13,7 @@ $IMDiagServerRun 13500
 $template startupfile,"rsyslogd.started" # trick to use relative path names!
 :syslogtag, contains, "rsyslogd"  ?startupfile
 
-$ErrorMessagesToStderr off
+# I have disabled the directive below, so that we see errors in testcase
+# creation. I am not sure why it was present in the first place, so for
+# now I just leave it commented out -- rgerhards, 2011-03-30
+#$ErrorMessagesToStderr off
diff --git a/tests/testsuites/rcvr_fail_restore_rcvr.conf b/tests/testsuites/rcvr_fail_restore_rcvr.conf
new file mode 100644
index 00000000..38ebad29
--- /dev/null
+++ b/tests/testsuites/rcvr_fail_restore_rcvr.conf
@@ -0,0 +1,8 @@
+$IncludeConfig diag-common2.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# then SENDER sends to this port (not tcpflood!)
+$InputTCPServerRun 13515
+
+$template outfmt,"%msg:F,58:2%\n"
+:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/rcvr_fail_restore_sender.conf b/tests/testsuites/rcvr_fail_restore_sender.conf
new file mode 100644
index 00000000..6b11aa4a
--- /dev/null
+++ b/tests/testsuites/rcvr_fail_restore_sender.conf
@@ -0,0 +1,15 @@
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# this listener is for message generation by the test framework!
+$InputTCPServerRun 13514
+
+$WorkDirectory test-spool
+$MainMsgQueueMaxFileSize 1g
+$MainMsgQueueFileName mainq
+
+# we use the shortest resume interval a) to let the test not run too long 
+# and b) make sure some retries happen before the reconnect
+$ActionResumeInterval 1
+$ActionResumeRetryCount -1
+*.*	@@127.0.0.1:13515
-- 
cgit 


From 00e1f24187ee814801e6969629b82a7ae030beaf Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 30 Mar 2011 12:13:14 +0200
Subject: stop adding data to DA queue when low watermark has been reached

potentially closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241
But needs more verification.
---
 runtime/queue.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/queue.c b/runtime/queue.c
index ef6e843b..50ae307c 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1774,9 +1774,13 @@ qqueueChkStopWrkrDA(qqueue_t *pThis)
 {
 	DEFiRet;
 
+//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis));
 	if(pThis->bEnqOnly) {
 		iRet = RS_RET_TERMINATE_WHEN_IDLE;
 	}
+	if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) {
+		iRet = RS_RET_TERMINATE_NOW;
+	}
 
 	RETiRet;
 }
-- 
cgit 


From eccfbf71f9dbc235309c2ad40035d41620f5e5df Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 31 Mar 2011 15:14:22 +0200
Subject: improved testbench, updated ChangeLog to reflect recent bugfix

After trying out some more things with the new code (after last
bugfix), I now think that the bugfix is OK to use and without
regressions.
---
 ChangeLog                  |  4 ++++
 tests/Makefile.am          |  6 ++++--
 tests/rcvr_fail_restore.sh | 42 ++++++++++++++++++++++++++++++++++--------
 3 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e4c25889..1eba3db9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,8 @@
 ---------------------------------------------------------------------------
+Version 5.8.0  [V5-stable] (rgerhards), 2011-04-??
+- bugfix: DA queue was never shutdown once it was started
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241
+---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-29
 - bugfix: ompgsql did not work properly with ANSI SQL strings
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ebbe0cf6..930aa304 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -56,8 +56,7 @@ TESTS +=  \
 	failover-rptd.sh \
 	failover-no-rptd.sh \
 	failover-no-basic.sh \
-	queue-persist.sh 
-	queue-persist.sh \
+	rcvr_fail_restore.sh \
 	linkedlistqueue.sh
 endif
 
@@ -289,6 +288,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   diag.sh \
 	   testsuites/diag-common.conf \
 	   testsuites/diag-common2.conf \
+           rcvr_fail_restore.sh \
+	   testsuites/rcvr_fail_restore_rcvr.conf \
+	   testsuites/rcvr_fail_restore_sender.conf \
            daqueue-persist.sh \
 	   daqueue-persist-drvr.sh \
 	   queue-persist.sh \
diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh
index e2f0a0d8..79486f10 100755
--- a/tests/rcvr_fail_restore.sh
+++ b/tests/rcvr_fail_restore.sh
@@ -9,31 +9,32 @@ source $srcdir/diag.sh init
 #
 # start up the instances. Note that the envrionment settings can be changed to
 # set instance-specific debugging parameters!
-export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUG="debug nostdout"
 #export RSYSLOG_DEBUGLOG="log2"
 source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUGLOG="log"
 #valgrind="valgrind"
 source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf
 # re-set params so that new instances do not thrash it...
-unset RSYSLOG_DEBUG
-unset RSYSLOG_DEBUGLOG
+#unset RSYSLOG_DEBUG
+#unset RSYSLOG_DEBUGLOG
 
 # now inject the messages into instance 2. It will connect to instance 1,
 # and that instance will record the data.
 source $srcdir/diag.sh injectmsg  1 1000
 source $srcdir/diag.sh wait-queueempty
-./msleep 3000 # make sure some retries happen (retry interval is set to 1 second)
+./msleep 1000 # let things settle down a bit
+
 #
 # Step 2: shutdown receiver, then send some more data, which then
 # needs to go into the queue.
 #
+
 source $srcdir/diag.sh shutdown-when-empty 2
 source $srcdir/diag.sh wait-shutdown 2
 
 source $srcdir/diag.sh injectmsg  1001 10000
-sleep 1 # we need to wait, otherwise we may be so fast that the receiver
-# comes up before we have finally suspended the action
+./msleep 3000 # make sure some retries happen (retry interval is set to 3 second)
 source $srcdir/diag.sh get-mainqueuesize
 ls -l test-spool
 
@@ -73,6 +74,31 @@ then
    # exit 1
 fi
 
+#
+# We now do an extra test (so this is two in one ;)) to see if the DA
+# queue can be reactivated after its initial shutdown. In essence, we 
+# redo steps 2 and 3.
+#
+# Step 5: stop receiver again, then send some more data, which then
+# needs to go into the queue.
+#
+echo "*** done primary test *** now checking if DA can be restarted"
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+source $srcdir/diag.sh injectmsg  11011 10000
+sleep 1 # we need to wait, otherwise we may be so fast that the receiver
+# comes up before we have finally suspended the action
+source $srcdir/diag.sh get-mainqueuesize
+ls -l test-spool
+
+#
+# Step 6: restart receiver, wait that the sender drains its queue
+#
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+echo waiting for sender to drain queue [may need a short while]
+source $srcdir/diag.sh wait-queueempty
+ls -l test-spool
 
 #
 # Queue file size checks done. Now it is time to terminate the system
@@ -92,5 +118,5 @@ then
    exit 1
 fi
 # do the final check
-source $srcdir/diag.sh seq-check 1 11010
+source $srcdir/diag.sh seq-check 1 21010
 source $srcdir/diag.sh exit
-- 
cgit