Merge branch 'v5-stable' into v5-beta

Conflicts:
	ChangeLog
	configure.ac
	doc/manual.html

5 files changed, 110 insertions, 9 deletions

diff --git a/ChangeLog b/ChangeLog
index ac1e8c10..542cc61c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,16 @@
 ---------------------------------------------------------------------------
 Version 5.9.7  [V5-BETA], 2012-04-??
 - added capability to specify substrings for field extraction mode
+- bugfix: ommysql did not properly init/exit the mysql runtime library
+  this could lead to segfaults. Triggering condition: multiple action
+  instances using ommysql.  Thanks to Tomas Heinrich for reporting this
+  problem and providing an initial patch (which my solution is based on,
+  I need to add more code to clean the mess up).
+- bugfix: rsyslog did not terminate when delayable inputs were blocked
+  due to unvailable sources. Fixes:
+  http://bugzilla.adiscon.com/show_bug.cgi?id=299
+  Thanks to Marcin M for bringing up this problem and Andre Lorbach
+  for helping to reproduce and fix it.
 ---------------------------------------------------------------------------
 Version 5.9.6  [V5-BETA], 2012-04-12
 - added configuration directives to customize queue light delay marks
@@ -134,7 +144,19 @@ Version 5.9.0  [V5-DEVEL] (rgerhards), 2011-06-08
   affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236
 ---------------------------------------------------------------------------
-Version 5.8.11  [V5-stable] 2012-04-??
+Version 5.8.11  [V5-stable] 2012-05-03
+- bugfix: ommysql did not properly init/exit the mysql runtime library
+  this could lead to segfaults. Triggering condition: multiple action
+  instances using ommysql.  Thanks to Tomas Heinrich for reporting this
+  problem and providing an initial patch (which my solution is based on,
+  I need to add more code to clean the mess up).
+- bugfix: rsyslog did not terminate when delayable inputs were blocked
+  due to unvailable sources. Fixes:
+  http://bugzilla.adiscon.com/show_bug.cgi?id=299
+  Thanks to Marcin M for bringing up this problem and Andre Lorbach
+  for helping to reproduce and fix it.
+- bugfix: active input in "light delay state" could block rsyslog
+  termination, at least for prolonged period of time
 - bugfix: imptcp input name could not be set
   config directive was accepted, but had no effect
 - bugfix: assigned ruleset was lost when using disk queues
@@ -144,6 +166,10 @@ Version 5.8.11  [V5-stable] 2012-04-??
 - bugfix: hostname was not requeried on HUP
   Thanks to Per Jessen for reporting this bug and Marius Tomaschewski for
   his help in testing the fix.
+- bugfix: inside queue.c, some thread cancel states were not correctly
+  reset. While this is a bug, we assume it did have no practical effect
+  because the reset as it was done was set to the state the code actually
+  had at this point. But better fix this...
 ---------------------------------------------------------------------------
 Version 5.8.10  [V5-stable] 2012-04-05
 - bugfix: segfault on startup if $actionqueuefilename was missing for disk
diff --git a/configure.ac b/configure.ac
index 6e140b31..60cc7674 100644
--- a/configure.ac
+++ b/configure.ac
@@ -517,8 +517,24 @@ if test "x$enable_mysql" = "xyes"; then
     [AC_MSG_FAILURE([MySQL library is missing])],
     [`mysql_config --libs`]
   )
+  AC_MSG_CHECKING(if we have mysql_library_init)
+  save_CFLAGS="$CFLAGS"
+  CFLAGS="$CFLAGS $MYSQL_CFLAGS"
+  save_LIBS="$LIBS"
+  LIBS="$LIBS $MYSQL_LIBS"
+  AC_TRY_LINK(
+    [#include <mysql.h>
+     #include <stdio.h>],
+    [mysql_library_init(0, NULL, NULL)],
+    [have_mysql_library_init=yes],
+    [have_mysql_library_init=no])
+  CFLAGS="$save_CFLAGS"
+  LIBS="$save_LIBS"
 fi
 AM_CONDITIONAL(ENABLE_MYSQL, test x$enable_mysql = xyes)
+if test "$have_mysql_library_init" = "yes"; then
+  AC_DEFINE([HAVE_MYSQL_LIBRARY_INIT], [1], [mysql_library_init available])
+fi
 AC_SUBST(MYSQL_CFLAGS)
 AC_SUBST(MYSQL_LIBS)
 
diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c
index 58e347bf..7ff89f5a 100644
--- a/plugins/ommysql/ommysql.c
+++ b/plugins/ommysql/ommysql.c
@@ -90,7 +90,6 @@ static void closeMySQL(instanceData *pData)
 	ASSERT(pData != NULL);
 
 	if(pData->f_hmysql != NULL) {	/* just to be on the safe side... */
-		mysql_server_end();
 		mysql_close(pData->f_hmysql);	
 		pData->f_hmysql = NULL;
 	}
@@ -319,6 +318,11 @@ ENDparseSelectorAct
 
 BEGINmodExit
 CODESTARTmodExit
+#	ifdef HAVE_MYSQL_LIBRARY_INIT
+	mysql_library_end();
+#	else
+	mysql_server_end();
+#	endif
 ENDmodExit
 
 
@@ -346,6 +350,20 @@ CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+
+	/* we need to init the MySQL library. If that fails, we cannot run */
+	if(
+#	ifdef HAVE_MYSQL_LIBRARY_INIT
+	   mysql_library_init(0, NULL, NULL)
+#	else
+	   mysql_server_init(0, NULL, NULL)
+#	endif
+	                                   ) {
+		errmsg.LogError(0, NO_ERRCODE, "ommysql: mysql_server_init() failed, plugin "
+		                "can not run");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
 	/* register our config handlers */
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionommysqlserverport", 0, eCmdHdlrInt, NULL, &iSrvPort, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID));
diff --git a/runtime/queue.c b/runtime/queue.c
index 00e957e6..7893a1db 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1770,7 +1770,7 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti)
 	}
 
 	/* but now cancellation is no longer permitted */
-	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+	pthread_setcancelstate(iCancelStateSave, NULL);
 
 finalize_it:
 	DBGPRINTF("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet,
@@ -1823,7 +1823,7 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti)
 	}
 
 	/* but now cancellation is no longer permitted */
-	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+	pthread_setcancelstate(iCancelStateSave, NULL);
 
 	/* now we are done, but need to re-aquire the mutex */
 	d_pthread_mutex_lock(pThis->mut);
@@ -1902,7 +1902,6 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
 	int wrk;
 	uchar *qName;
 	size_t lenBuf;
-	int iQueueSizeSave;
 
 	ASSERT(pThis != NULL);
 
@@ -2307,8 +2306,13 @@ static inline rsRetVal
 doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
 {
 	DEFiRet;
+	int err;
 	struct timespec t;
 
+	if(glbl.GetGlobalInputTermState()) {
+		ABORT_FINALIZE(RS_RET_FORCE_TERM);
+	}
+
 	STATSCOUNTER_INC(pThis->ctrEnqueued, pThis->mutCtrEnqueued);
 	/* first check if we need to discard this message (which will cause CHKiRet() to exit)
 	 */
@@ -2335,15 +2339,48 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
 	 * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14
 	 */
 	if(flowCtlType == eFLOWCTL_FULL_DELAY) {
+		DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message "
+		           "- blocking.\n");
 		while(pThis->iQueueSize >= pThis->iFullDlyMrk) {
-			DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n");
-			pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */
+			/* We have a problem during shutdown if we block eternally. In that
+			 * case, the the input thread cannot be terminated. So we wake up
+			 * from time to time to check for termination.
+			 * TODO/v6(at earliest): check if we could signal the condition during
+			 * shutdown. However, this requires new queue registries and thus is
+			 * far to much change for a stable version (and I am still not sure it
+			 * is worth the effort, given how seldom this situation occurs and how
+			 * few resources the wakeups need). -- rgerhards, 2012-05-03
+			 * In any case, this was the old code (if we do the TODO):
+			 * pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut);
+			 */
+			timeoutComp(&t, 1000);
+			err = pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t);
+			if(err != 0 && err != ETIMEDOUT) {
+				/* Something is really wrong now. Report to debug log and abort the
+				 * wait. That keeps us running, even though we may lose messages.
+				 */
+				DBGOPRINT((obj_t*) pThis, "potential program bug: pthread_cond_timedwait()"
+				          "/fulldelay returned %d\n", err);
+				break;
+				
+			}
+			DBGPRINTF("wti worker in full delay timed out, checking termination...\n");
+			if(glbl.GetGlobalInputTermState()) {
+				ABORT_FINALIZE(RS_RET_FORCE_TERM);
+			}
 		}
 	} else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) {
 		if(pThis->iQueueSize >= pThis->iLightDlyMrk) {
-			DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n");
+			DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light "
+			          "delayable message - blocking a bit.\n");
 			timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */
-			pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */
+			err = pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t);
+			if(err != 0 && err != ETIMEDOUT) {
+				/* Something is really wrong now. Report to debug log */
+				DBGOPRINT((obj_t*) pThis, "potential program bug: pthread_cond_timedwait()"
+				          "/lightdelay returned %d\n", err);
+				
+			}
 		}
 	}
 
@@ -2356,6 +2393,9 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
 	      || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0
 	      	  && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) {
 		DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n");
+		if(glbl.GetGlobalInputTermState()) {
+			ABORT_FINALIZE(RS_RET_FORCE_TERM);
+		}
 		timeoutComp(&t, pThis->toEnq);
 		STATSCOUNTER_INC(pThis->ctrFull, pThis->mutCtrFull);
 // TODO : handle enqOnly => discard!
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 7cb4d367..41e23a10 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -1498,6 +1498,7 @@ runInputModules(void)
 	pMod = module.GetNxtType(NULL, eMOD_IN);
 	while(pMod != NULL) {
 		if(pMod->mod.im.bCanRun) {
+			DBGPRINTF("trying to start input module '%s'\n", pMod->pszName);
 			/* activate here */
 			bNeedsCancel = (pMod->isCompatibleWithFeature(sFEATURENonCancelInputTermination) == RS_RET_OK) ?
 				       0 : 1;