From ebc931db47bf1e28502a043dc2984acf5fd532b6 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 17 Sep 2008 15:46:19 +0200
Subject: doc bugfix: queue doc had wrong parameter name

...for setting controlling worker thread shutdown period
---
 ChangeLog       | 2 ++
 doc/queues.html | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 1995ca10..c11d6aa8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -11,6 +11,8 @@ Version 3.21.5  [DEVEL] (rgerhards), 2008-09-??
   each input module (else it is blank).
 - added system property "$myhostname", which contains the name of the
   local host as it knows itself.
+- doc bugfix: queue doc had wrong parameter name for setting controlling
+  worker thread shutdown period
 ---------------------------------------------------------------------------
 Version 3.21.4  [DEVEL] (rgerhards), 2008-09-04
 - removed compile time fixed message size limit (was 2K), limit can now
diff --git a/doc/queues.html b/doc/queues.html
index a2074d36..b8fcb481 100644
--- a/doc/queues.html
+++ b/doc/queues.html
@@ -219,7 +219,7 @@ parall. Thus, the upper limit ca be set via "<i>$&lt;object&gt;QueueWorkerThread
 If it, for example, is set to four, no more than four workers will ever be 
 started, no matter how many elements are enqueued. </p>
 <p>Worker threads that have been started are kept running until an inactivity 
-timeout happens. The timeout can be set via "<i>$&lt;object&gt;QueueWorkerTimeoutShutdown</i>" 
+timeout happens. The timeout can be set via "<i>$&lt;object&gt;QueueWorkerTimeoutThreadShutdown</i>" 
 and is specified in milliseconds. If you do not like to keep the workers 
 running, simply set it to 0, which means immediate timeout and thus immediate 
 shutdown. But consider that creating threads involves some overhead, and this is 
@@ -357,4 +357,4 @@ parameters, because not all are applicable. For example, in current output
 module design, actions do not support multi-threading. Consequently, the number 
 of worker threads is fixed to one for action queues and can not be changed.</p>
 
-</body></html>
\ No newline at end of file
+</body></html>
-- 
cgit 


From 7b63cd6feda5087c43652bdcf8e694b544295d5b Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 18 Sep 2008 12:44:00 +0200
Subject: minor things, mostly improved debug info

---
 doc/queues.html   | 3 ++-
 runtime/debug.c   | 6 ++++--
 runtime/msg.c     | 4 ++--
 runtime/srutils.c | 4 ++++
 runtime/wtp.c     | 2 +-
 threads.c         | 2 +-
 tools/syslogd.c   | 8 +++-----
 7 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/doc/queues.html b/doc/queues.html
index b8fcb481..7461121b 100644
--- a/doc/queues.html
+++ b/doc/queues.html
@@ -223,7 +223,8 @@ timeout happens. The timeout can be set via "<i>$&lt;object&gt;QueueWorkerTimeou
 and is specified in milliseconds. If you do not like to keep the workers 
 running, simply set it to 0, which means immediate timeout and thus immediate 
 shutdown. But consider that creating threads involves some overhead, and this is 
-why we keep them running.</p>
+why we keep them running. If you would like to never shutdown any worker
+threads, specify -1 for this parameter.</p>
 <h2>Discarding Messages</h2>
 <p>If the queue reaches the so called "discard watermark" (a number of queued 
 elements), less important messages can automatically be discarded. This is in an 
diff --git a/runtime/debug.c b/runtime/debug.c
index 1450d029..958fb596 100644
--- a/runtime/debug.c
+++ b/runtime/debug.c
@@ -1,3 +1,4 @@
+#include <sys/syscall.h>
 /* debug.c
  *
  * This file proides debug and run time error analysis support. Some of the
@@ -879,7 +880,7 @@ dbgprintf(char *fmt, ...)
 	}
 
 	/* do not cache the thread name, as the caller might have changed it
-	 * TODO: optimized, invalidate cache when new name is set
+	 * TODO: optimize, invalidate cache when new name is set
 	 */
 	dbgGetThrdName(pszThrdName, sizeof(pszThrdName), ptLastThrdID, 0);
 
@@ -889,7 +890,8 @@ dbgprintf(char *fmt, ...)
 			if(stddbg != NULL) fprintf(stddbg, "%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec);
 			if(altdbg != NULL) fprintf(altdbg, "%4.4ld.%9.9ld:", (long) (t.tv_sec % 10000), t.tv_nsec);
 		}
-		if(stddbg != NULL) fprintf(stddbg, "%s: ", pszThrdName);
+		if(stddbg != NULL) fprintf(stddbg, "{%ld}%s: ", (long) syscall(SYS_gettid), pszThrdName);
+		/*if(stddbg != NULL) fprintf(stddbg, "%s: ", pszThrdName);*/
 		if(altdbg != NULL) fprintf(altdbg, "%s: ", pszThrdName);
 	}
 	bWasNL = (*(fmt + strlen(fmt) - 1) == '\n') ? 1 : 0;
diff --git a/runtime/msg.c b/runtime/msg.c
index f4eb9414..164c3517 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -140,8 +140,8 @@ void (*funcMsgPrepareEnqueue)(msg_t *pMsg);
 #define MsgLock(pMsg) 	funcLock(pMsg)
 #define MsgUnlock(pMsg) funcUnlock(pMsg)
 #else
-#define MsgLock(pMsg) 	{dbgprintf("line %d\n - ", __LINE__); funcLock(pMsg);; }
-#define MsgUnlock(pMsg) {dbgprintf("line %d - ", __LINE__); funcUnlock(pMsg); }
+#define MsgLock(pMsg) 	{dbgprintf("MsgLock line %d\n - ", __LINE__); funcLock(pMsg);; }
+#define MsgUnlock(pMsg) {dbgprintf("MsgUnlock line %d - ", __LINE__); funcUnlock(pMsg); }
 #endif
 
 /* the next function is a dummy to be used by the looking functions
diff --git a/runtime/srutils.c b/runtime/srutils.c
index 97cc3252..1280e40d 100644
--- a/runtime/srutils.c
+++ b/runtime/srutils.c
@@ -371,6 +371,7 @@ int getNumberDigits(long lNum)
 rsRetVal
 timeoutComp(struct timespec *pt, long iTimeout)
 {
+	BEGINfunc
 	assert(pt != NULL);
 	/* compute timeout */
 	clock_gettime(CLOCK_REALTIME, pt);
@@ -379,6 +380,7 @@ timeoutComp(struct timespec *pt, long iTimeout)
 		pt->tv_nsec -= 1000000000;
 	}
 	pt->tv_sec += iTimeout / 1000;
+	ENDfunc
 	return RS_RET_OK; /* so far, this is static... */
 }
 
@@ -393,6 +395,7 @@ timeoutVal(struct timespec *pt)
 {
 	struct timespec t;
 	long iTimeout;
+	BEGINfunc
 
 	assert(pt != NULL);
 	/* compute timeout */
@@ -403,6 +406,7 @@ timeoutVal(struct timespec *pt)
 	if(iTimeout < 0)
 		iTimeout = 0;
 
+	ENDfunc
 	return iTimeout;
 }
 
diff --git a/runtime/wtp.c b/runtime/wtp.c
index 8b041ea2..ee9fc765 100644
--- a/runtime/wtp.c
+++ b/runtime/wtp.c
@@ -453,7 +453,7 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex)
 
 	ISOBJ_TYPE_assert(pThis, wtp);
 
-	wtpProcessThrdChanges(pThis);
+	wtpProcessThrdChanges(pThis);	// TODO: Performance: this causes a lot of FUTEX calls
 
 	BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex);
 
diff --git a/threads.c b/threads.c
index 61ea8f29..13222694 100644
--- a/threads.c
+++ b/threads.c
@@ -127,7 +127,7 @@ static void* thrdStarter(void *arg)
 	assert(pThis != NULL);
 	assert(pThis->pUsrThrdMain != NULL);
 
-	/* block all signalsi */
+	/* block all signals */
 	sigset_t sigSet;
 	sigfillset(&sigSet);
 	pthread_sigmask(SIG_BLOCK, &sigSet, NULL);
diff --git a/tools/syslogd.c b/tools/syslogd.c
index b6e1d826..90b38748 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -1928,7 +1928,7 @@ die(int sig)
 
 	/* close the inputs */
 	dbgprintf("Terminating input threads...\n");
-	thrdTerminateAll(); /* TODO: inputs only, please */
+	thrdTerminateAll();
 
 	/* and THEN send the termination log message (see long comment above) */
 	if (sig) {
@@ -2168,8 +2168,8 @@ static void dbgPrintInitInfo(void)
 			cCCEscapeChar);
 
 	dbgprintf("Main queue size %d messages.\n", iMainMsgQueueSize);
-	dbgprintf("Main queue worker threads: %d, Perists every %d updates.\n",
-		  iMainMsgQueueNumWorkers, iMainMsgQPersistUpdCnt);
+	dbgprintf("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n",
+		  iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt);
 	dbgprintf("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n",
 		   iMainMsgQtoQShutdown, iMainMsgQtoActShutdown, iMainMsgQtoEnq);
 	dbgprintf("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n",
@@ -2179,11 +2179,9 @@ static void dbgPrintInitInfo(void)
 	/* TODO: add
 	iActionRetryCount = 0;
 	iActionRetryInterval = 30000;
-	static int iMainMsgQtoWrkShutdown = 60000;
 	static int iMainMsgQtoWrkMinMsgs = 100;	
 	static int iMainMsgQbSaveOnShutdown = 1;
 	iMainMsgQueMaxDiskSpace = 0;
-	setQPROP(queueSettoWrkShutdown, "$MainMsgQueueTimeoutWorkerThreadShutdown", 5000);
 	setQPROP(queueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100);
 	setQPROP(queueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1);
 	 */
-- 
cgit 


From b005a1dcf8c74bf19ef7fd6890e764a480a38d3d Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 18 Sep 2008 17:16:17 +0200
Subject: added a small diag tool to generate local log messages quickly

... I think it may be useful to extend this later, thus I move it
into git.
---
 tools/Makefile.am |  3 ++-
 tools/msggen.c    | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 tools/msggen.c

diff --git a/tools/Makefile.am b/tools/Makefile.am
index a265af9c..b66b8e0c 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -26,8 +26,9 @@ rsyslogd_LDADD = $(zlib_libs) $(pthreads_libs) $(rsrt_libs)
 rsyslogd_LDFLAGS = -export-dynamic
 
 if ENABLE_DIAGTOOLS
-sbin_PROGRAMS += rsyslog_diag_hostname
+sbin_PROGRAMS += rsyslog_diag_hostname msggen
 rsyslog_diag_hostname_SOURCES = gethostn.c
+msggen_SOURCES = msggen.c
 endif
 
 EXTRA_DIST = $(man_MANS)
diff --git a/tools/msggen.c b/tools/msggen.c
new file mode 100644
index 00000000..7990a3c8
--- /dev/null
+++ b/tools/msggen.c
@@ -0,0 +1,38 @@
+/* msggen - a small diagnostic utility that does very quick
+ * syslog() calls.
+ *
+ * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+
+#include <stdio.h>
+#include <syslog.h>
+
+int main(int argc, char *argv[])
+{
+	int i;
+
+	openlog("msggen", 0 , LOG_LOCAL0);
+
+	for(i = 0 ; i < 10 ; ++i)
+		syslog(LOG_NOTICE, "This is message number %d", i);
+
+	closelog();
+	return 0;
+}
-- 
cgit 


From 5ce9b5ad831aae00a7d28d77ccd1ec974f71835f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 18 Sep 2008 17:17:47 +0200
Subject: TESTING COMMIT: commiting not fully correct code

... for the purpose of conducting a few external tests. This is a
first approach at removing the time() call in the output, but it shows
there are some subleties we need to address.
---
 action.c | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/action.c b/action.c
index 5c5bdbe9..c3cd7db6 100644
--- a/action.c
+++ b/action.c
@@ -625,14 +625,43 @@ actionWriteToAction(action_t *pAction)
 		dbgprintf("action not yet ready again to be executed, onceInterval %d, tCurr %d, tNext %d\n",
 			  (int) pAction->iSecsExecOnceInterval, (int) getActNow(pAction),
 			  (int) (pAction->iSecsExecOnceInterval + pAction->tLastExec));
+		/* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */
+		pAction->tLastExec = getActNow(pAction); /* re-init time flags */
 		FINALIZE;
 	}
 
-	pAction->f_time = pAction->tLastExec = getActNow(pAction); /* re-init time flags */
+
+
+	/* TODO: move this to msg object or some other object. This is just for quick testing!
+	 * ALSO, THIS DOES NOT YET WORK PROPERLY!
+	 * The reason is that we do not know the DST status, which is major pain. I need to
+	 * think about obtaining this information (or the actual Unix timestamp) when I
+	 * create the reception timestamp, but that also means I need to preserve that information
+	 * while in the on-disk queue. Also need to think about a few other implications.
+	 * rgerhards, 2008-09-17
+	 */
+	{
+		struct tm tTm;
+		tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second;
+		tTm.tm_min = pAction->f_pMsg->tRcvdAt.minute;
+		tTm.tm_hour = pAction->f_pMsg->tRcvdAt.hour;
+		tTm.tm_mday = pAction->f_pMsg->tRcvdAt.day;
+		tTm.tm_mon = pAction->f_pMsg->tRcvdAt.month - 1;
+		tTm.tm_year = pAction->f_pMsg->tRcvdAt.year - 1900;
+		/********************************************************************************/
+		tTm.tm_isdst = 1;      /* TODO THIS IS JUST VALID FOR THE NEXT FEW DAYS ;) TODO */
+		/********************************************************************************/
+		pAction->f_time = mktime(&tTm);
+dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL));
+	}
+
+	//pAction->f_time = getActNow(pAction); /* re-init time flags */
 	/* Note: tLastExec could be set in the if block above, but f_time causes us a hard time
 	 * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16
 	 */
 
+
+
 	/* When we reach this point, we have a valid, non-disabled action.
 	 * So let's enqueue our message for execution. -- rgerhards, 2007-07-24
 	 */
-- 
cgit 


From 1b68464ddfac8abdca203f3918da98e7d6c54ad4 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 2 Oct 2008 11:21:45 +0200
Subject: some code cleanup

removed code not actually needed in imupd
---
 plugins/imudp/imudp.c | 98 +++++++++++++++++++++++++--------------------------
 1 file changed, 48 insertions(+), 50 deletions(-)

diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index 5792a999..d4c9d759 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -127,6 +127,8 @@ finalize_it:
 
 
 /* This function is called to gather input.
+ * Note that udpLstnSocks must be non-NULL because otherwise we would not have
+ * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02
  */
 BEGINrunInput
 	int maxfds;
@@ -154,17 +156,14 @@ CODESTARTrunInput
 	        maxfds = 0;
 	        FD_ZERO (&readfds);
 
-		/* Add the UDP listen sockets to the list of read descriptors.
-		 */
-		if(udpLstnSocks != NULL) {
-                        for (i = 0; i < *udpLstnSocks; i++) {
-                                if (udpLstnSocks[i+1] != -1) {
-					if(Debug)
-						net.debugListenInfo(udpLstnSocks[i+1], "UDP");
-                                        FD_SET(udpLstnSocks[i+1], &readfds);
-					if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1];
-				}
-                        }
+		/* Add the UDP listen sockets to the list of read descriptors. */
+		for (i = 0; i < *udpLstnSocks; i++) {
+			if (udpLstnSocks[i+1] != -1) {
+				if(Debug)
+					net.debugListenInfo(udpLstnSocks[i+1], "UDP");
+				FD_SET(udpLstnSocks[i+1], &readfds);
+				if(udpLstnSocks[i+1]>maxfds) maxfds=udpLstnSocks[i+1];
+			}
 		}
 		if(Debug) {
 			dbgprintf("--------imUDP calling select, active file descriptors (max %d): ", maxfds);
@@ -177,46 +176,45 @@ CODESTARTrunInput
 		/* wait for io to become ready */
 		nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL);
 
-		if(udpLstnSocks != NULL) {
-		       for (i = 0; nfds && i < *udpLstnSocks; i++) {
-			       if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
-				       socklen = sizeof(frominet);
-				       l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0,
-						    (struct sockaddr *)&frominet, &socklen);
-				       if (l > 0) {
-					       if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
-						       dbgprintf("Message from inetd socket: #%d, host: %s\n",
-							       udpLstnSocks[i+1], fromHost);
-						       /* Here we check if a host is permitted to send us
-							* syslog messages. If it isn't, we do not further
-							* process the message but log a warning (if we are
-							* configured to do this).
-							* rgerhards, 2005-09-26
-							*/
-						       if(net.isAllowedSender(net.pAllowedSenders_UDP,
-							  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
-							       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
-							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp");
-						       } else {
-							       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
-							       if(glbl.GetOption_DisallowWarning) {
-								       errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded",
-										  (char*)fromHost);
-							       }	
-						       }
-					       }
-				       } else if (l < 0 && errno != EINTR && errno != EAGAIN) {
-						char errStr[1024];
-						rs_strerror_r(errno, errStr, sizeof(errStr));
-						dbgprintf("INET socket error: %d = %s.\n", errno, errStr);
-						       errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet");
-						       /* should be harmless */
-						       sleep(1);
+	       for (i = 0; nfds && i < *udpLstnSocks; i++) {
+		       if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
+			       socklen = sizeof(frominet);
+			       l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0,
+					    (struct sockaddr *)&frominet, &socklen);
+			       if (l > 0) {
+				       if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
+					       dbgprintf("Message from inetd socket: #%d, host: %s\n",
+						       udpLstnSocks[i+1], fromHost);
+					       /* Here we check if a host is permitted to send us
+						* syslog messages. If it isn't, we do not further
+						* process the message but log a warning (if we are
+						* configured to do this).
+						* rgerhards, 2005-09-26
+						*/
+					       if(net.isAllowedSender(net.pAllowedSenders_UDP,
+						  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
+						       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
+						       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp");
+					       } else {
+						       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
+						       if(glbl.GetOption_DisallowWarning) {
+							       errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded",
+									  (char*)fromHost);
+						       }	
 					       }
-					--nfds; /* indicate we have processed one */
-				}
-		       }
-		}
+				       }
+			       } else if (l < 0 && errno != EINTR && errno != EAGAIN) {
+					char errStr[1024];
+					rs_strerror_r(errno, errStr, sizeof(errStr));
+					dbgprintf("INET socket error: %d = %s.\n", errno, errStr);
+					       errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet");
+					       /* should be harmless */
+					       sleep(1);
+				       }
+				--nfds; /* indicate we have processed one */
+			}
+	       }
+	       /* end of a run, back to loop for next recv() */
 	}
 
 	return iRet;
-- 
cgit 


From 6290cc9056dacad3ff80945408c4caad240002b0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 2 Oct 2008 11:44:50 +0200
Subject: performance-optimized imudp

---
 configure.ac          |  4 +--
 plugins/imudp/imudp.c | 76 +++++++++++++++++++++++++++++----------------------
 2 files changed, 46 insertions(+), 34 deletions(-)

diff --git a/configure.ac b/configure.ac
index fea7c063..6b6a699d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -53,7 +53,7 @@ AC_SUBST(dl_libs)
 AC_HEADER_RESOLV
 AC_HEADER_STDC
 AC_HEADER_SYS_WAIT
-AC_CHECK_HEADERS([arpa/inet.h libgen.h fcntl.h locale.h netdb.h netinet/in.h paths.h stddef.h stdlib.h string.h sys/file.h sys/ioctl.h sys/param.h sys/socket.h sys/time.h sys/stat.h syslog.h unistd.h utmp.h])
+AC_CHECK_HEADERS([arpa/inet.h libgen.h fcntl.h locale.h netdb.h netinet/in.h paths.h stddef.h stdlib.h string.h sys/file.h sys/ioctl.h sys/param.h sys/socket.h sys/time.h sys/stat.h syslog.h unistd.h utmp.h sys/epoll.h])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
@@ -88,7 +88,7 @@ AC_TYPE_SIGNAL
 AC_FUNC_STAT
 AC_FUNC_STRERROR_R
 AC_FUNC_VPRINTF
-AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r])
+AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait])
 
 # Check for MAXHOSTNAMELEN
 AC_MSG_CHECKING(for MAXHOSTNAMELEN)
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index d4c9d759..f4830a47 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -179,39 +179,51 @@ CODESTARTrunInput
 	       for (i = 0; nfds && i < *udpLstnSocks; i++) {
 		       if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
 			       socklen = sizeof(frominet);
-			       l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0,
-					    (struct sockaddr *)&frominet, &socklen);
-			       if (l > 0) {
-				       if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
-					       dbgprintf("Message from inetd socket: #%d, host: %s\n",
-						       udpLstnSocks[i+1], fromHost);
-					       /* Here we check if a host is permitted to send us
-						* syslog messages. If it isn't, we do not further
-						* process the message but log a warning (if we are
-						* configured to do this).
-						* rgerhards, 2005-09-26
-						*/
-					       if(net.isAllowedSender(net.pAllowedSenders_UDP,
-						  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
-						       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
-						       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp");
-					       } else {
-						       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
-						       if(glbl.GetOption_DisallowWarning) {
-							       errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded",
-									  (char*)fromHost);
-						       }	
+			        do {
+					/* we now try to read from the file descriptor until there
+					 * is no more data. This is done in the hope to get better performance
+					 * out of the system. However, this also means that a descriptor
+					 * monopolizes processing while it contains data. This can lead to
+					 * data loss in other descriptors. However, if the system is incapable of
+					 * handling the workload, we will loss data in any case. So it doesn't really
+					 * matter where the actual loss occurs - it is always random, because we depend
+					 * on scheduling order. -- rgerhards, 2008-10-02
+					 */
+					l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0,
+						    (struct sockaddr *)&frominet, &socklen);
+					if(l > 0) {
+					       if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
+						       dbgprintf("Message from inetd socket: #%d, host: %s\n",
+							       udpLstnSocks[i+1], fromHost);
+						       /* Here we check if a host is permitted to send us
+							* syslog messages. If it isn't, we do not further
+							* process the message but log a warning (if we are
+							* configured to do this).
+							* rgerhards, 2005-09-26
+							*/
+						       if(net.isAllowedSender(net.pAllowedSenders_UDP,
+							  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
+							       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
+							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp");
+						       } else {
+							       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
+							       if(glbl.GetOption_DisallowWarning) {
+								       errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded",
+										  (char*)fromHost);
+							       }	
+						       }
 					       }
-				       }
-			       } else if (l < 0 && errno != EINTR && errno != EAGAIN) {
-					char errStr[1024];
-					rs_strerror_r(errno, errStr, sizeof(errStr));
-					dbgprintf("INET socket error: %d = %s.\n", errno, errStr);
-					       errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet");
-					       /* should be harmless */
-					       sleep(1);
-				       }
-				--nfds; /* indicate we have processed one */
+					} else if(l < 0 && errno != EINTR && errno != EAGAIN) {
+						char errStr[1024];
+						rs_strerror_r(errno, errStr, sizeof(errStr));
+						dbgprintf("INET socket error: %d = %s.\n", errno, errStr);
+						       errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet");
+						       /* should be harmless */
+						       sleep(1);
+					}
+				} while(l > 0); /* Warning: do ... while()! */
+
+				--nfds; /* indicate we have processed one descriptor */
 			}
 	       }
 	       /* end of a run, back to loop for next recv() */
-- 
cgit 


From 1e53745dc9f11a90c613d177b2caba563c2b83be Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 2 Oct 2008 12:20:12 +0200
Subject: very minor: performance optimization hint added

---
 action.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/action.c b/action.c
index c3cd7db6..a25eca23 100644
--- a/action.c
+++ b/action.c
@@ -652,6 +652,7 @@ actionWriteToAction(action_t *pAction)
 		tTm.tm_isdst = 1;      /* TODO THIS IS JUST VALID FOR THE NEXT FEW DAYS ;) TODO */
 		/********************************************************************************/
 		pAction->f_time = mktime(&tTm);
+/* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */
 dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL));
 	}
 
-- 
cgit 


From 65f74a712b111f946faaddd0583a6c7b22bd062f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 6 Oct 2008 14:25:23 +0200
Subject: added $UDPServerTimeRequery option

...which enables to work with
less acurate timestamps in favor of performance. This enables querying
of the time only every n-th time if imudp is running in the tight
receive loop (aka receiving messsages at a high rate)
---
 ChangeLog                   |  4 +++
 dirty.h                     |  2 +-
 doc/rsyslog_conf.html       |  8 +++++
 plugins/imrelp/imrelp.c     |  2 +-
 plugins/imudp/imudp.c       | 20 +++++++++++--
 plugins/imuxsock/imuxsock.c |  2 +-
 runtime/msg.c               | 72 ++++++++++++++++++++++++++++++++++++---------
 runtime/msg.h               |  4 +--
 tcps_sess.c                 |  8 ++---
 tools/syslogd.c             | 39 ++++++++++++++++++------
 10 files changed, 126 insertions(+), 35 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ed6f8723..346ff39c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 ---------------------------------------------------------------------------
 Version 3.21.6  [DEVEL] (rgerhards), 2008-10-??
+- added $UDPServerTimeRequery option which enables to work with
+  less acurate timestamps in favor of performance. This enables querying
+  of the time only every n-th time if imudp is running in the tight
+  receive loop (aka receiving messsages at a high rate)
 - doc bugfix: queue doc had wrong parameter name for setting controlling
   worker thread shutdown period
 - consolidated time calls during msg object creation, improves performance
diff --git a/dirty.h b/dirty.h
index 796df18e..58c4ea39 100644
--- a/dirty.h
+++ b/dirty.h
@@ -40,7 +40,7 @@
 
 rsRetVal submitMsg(msg_t *pMsg);
 rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags);
-rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName);
+rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime);
 
 /* TODO: the following 2 need to go in conf obj interface... */
 rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName);
diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html
index 7cd40cb7..2b773476 100644
--- a/doc/rsyslog_conf.html
+++ b/doc/rsyslog_conf.html
@@ -243,6 +243,14 @@ address (or name) the UDP listens should bind to</li>
 <li>$UDPServerRun &lt;port&gt; (imudp) -- former
 -r&lt;port&gt; option, default 514, start UDP server on this
 port, "*" means all addresses</li>
+<li>$UDPServerTimeRequery &lt;nbr-of-times&gt; (imudp) -- this is a performance 
+optimization. Getting the system time is very costly. With this setting, imudp can
+be instructed to obtain the precise time only once every n-times. This logic is
+only activated if messages come in at a very fast rate, so doing less frequent
+time calls should usually be acceptable. The default value is two, because we have
+seen that even without optimization the kernel often returns twice the identical time.
+You can set this value as high as you like, but do so at your own risk. The higher
+the value, the less precise the timestamp.
 <li><a href="rsconf1_umask.html">$UMASK</a></li>
 </ul>
 <p><b>Where &lt;size_nbr&gt; is specified above,</b>
diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c
index b01dd98b..f3771237 100644
--- a/plugins/imrelp/imrelp.c
+++ b/plugins/imrelp/imrelp.c
@@ -84,7 +84,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s
 {
 	DEFiRet;
 	parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME,
-			      NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp");
+			      NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL);
 
 	RETiRet;
 }
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index f4830a47..1865d777 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -40,6 +40,7 @@
 #include "srUtils.h"
 #include "errmsg.h"
 #include "glbl.h"
+#include "datetime.h"
 
 MODULE_TYPE_INPUT
 
@@ -50,6 +51,7 @@ DEF_IMOD_STATIC_DATA
 DEFobjCurrIf(errmsg)
 DEFobjCurrIf(glbl)
 DEFobjCurrIf(net)
+DEFobjCurrIf(datetime)
 
 static int iMaxLine;			/* maximum UDP message size supported */
 static int *udpLstnSocks = NULL;	/* Internet datagram sockets, first element is nbr of elements
@@ -59,6 +61,8 @@ static uchar *pRcvBuf = NULL;		/* receive buffer (for a single packet). We use a
 					 * it so that we can check available memory in willRun() and request
 					 * termination if we can not get it. -- rgerhards, 2007-12-27
 					 */
+#define TIME_REQUERY_DFLT 2
+static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */
 
 /* config settings */
 
@@ -141,6 +145,8 @@ BEGINrunInput
 	uchar fromHostIP[NI_MAXHOST];
 	uchar fromHostFQDN[NI_MAXHOST];
 	ssize_t l;
+	struct syslogTime stTime;
+	int iNbrTimeUsed;
 CODESTARTrunInput
 	/* this is an endless loop - it is terminated when the thread is
 	 * signalled to do so. This, however, is handled by the framework,
@@ -179,6 +185,7 @@ CODESTARTrunInput
 	       for (i = 0; nfds && i < *udpLstnSocks; i++) {
 		       if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
 			       socklen = sizeof(frominet);
+				iNbrTimeUsed = 0;
 			        do {
 					/* we now try to read from the file descriptor until there
 					 * is no more data. This is done in the hope to get better performance
@@ -203,8 +210,11 @@ CODESTARTrunInput
 							*/
 						       if(net.isAllowedSender(net.pAllowedSenders_UDP,
 							  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
+								if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) {
+									datetime.getCurrTime(&stTime);
+								}
 							       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
-							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp");
+							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime);
 						       } else {
 							       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
 							       if(glbl.GetOption_DisallowWarning) {
@@ -274,6 +284,7 @@ CODESTARTmodExit
 	/* release what we no longer need */
 	objRelease(errmsg, CORE_COMPONENT);
 	objRelease(glbl, CORE_COMPONENT);
+	objRelease(datetime, CORE_COMPONENT);
 	objRelease(net, LM_NET_FILENAME);
 ENDmodExit
 
@@ -293,6 +304,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 		net.closeUDPListenSockets(udpLstnSocks);
 		udpLstnSocks = NULL;
 	}
+	iTimeRequery = TIME_REQUERY_DFLT;/* the default is to query only every second time */
 	return RS_RET_OK;
 }
 
@@ -303,6 +315,7 @@ CODESTARTmodInit
 CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 	CHKiRet(objUse(glbl, CORE_COMPONENT));
+	CHKiRet(objUse(datetime, CORE_COMPONENT));
 	CHKiRet(objUse(net, LM_NET_FILENAME));
 
 	/* register config file handlers */
@@ -310,9 +323,10 @@ CODEmodInit_QueryRegCFSLineHdlr
 		addListner, NULL, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord,
 		NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt,
+		NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler,
 		resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID));
 ENDmodInit
-/*
- * vi:set ai:
+/* vim:set ai:
  */
diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c
index 55b8b2df..77d347d9 100644
--- a/plugins/imuxsock/imuxsock.c
+++ b/plugins/imuxsock/imuxsock.c
@@ -221,7 +221,7 @@ static rsRetVal readSocket(int fd, int iSock)
 	if (iRcvd > 0) {
 		parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock],
 				      (uchar*)"127.0.0.1", pRcv,
-			 	      iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock");
+			 	      iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL);
 	} else if (iRcvd < 0 && errno != EINTR) {
 		char errStr[1024];
 		rs_strerror_r(errno, errStr, sizeof(errStr));
diff --git a/runtime/msg.c b/runtime/msg.c
index 69296710..df8c1572 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -239,12 +239,21 @@ rsRetVal MsgEnableThreadSafety(void)
 /* end locking functions */
 
 
-/* "Constructor" for a msg "object". Returns a pointer to
+/* This is common code for all Constructors. It is defined in an
+ * inline'able function so that we can save a function call in the
+ * actual constructors (otherwise, the msgConstruct would need
+ * to call msgConstructWithTime(), which would require a
+ * function call). Now, both can use this inline function. This
+ * enables us to be optimal, but still have the code just once.
  * the new object or NULL if no such object could be allocated.
  * An object constructed via this function should only be destroyed
- * via "msgDestruct()".
+ * via "msgDestruct()". This constructor does not query system time
+ * itself but rather uses a user-supplied value. This enables the caller
+ * to do some tricks to save processing time (done, for example, in the
+ * udp input).
+ * rgerhards, 2008-10-06
  */
-rsRetVal msgConstruct(msg_t **ppThis)
+static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 {
 	DEFiRet;
 	msg_t *pM;
@@ -257,21 +266,58 @@ rsRetVal msgConstruct(msg_t **ppThis)
 	pM->iRefCount = 1;
 	pM->iSeverity = -1;
 	pM->iFacility = -1;
+	objConstructSetObjInfo(pM);
+
+	/* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/
+
+	*ppThis = pM;
+
+finalize_it:
+	RETiRet;
+}
+
+
+/* "Constructor" for a msg "object". Returns a pointer to
+ * the new object or NULL if no such object could be allocated.
+ * An object constructed via this function should only be destroyed
+ * via "msgDestruct()". This constructor does not query system time
+ * itself but rather uses a user-supplied value. This enables the caller
+ * to do some tricks to save processing time (done, for example, in the
+ * udp input).
+ * rgerhards, 2008-10-06
+ */
+rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime)
+{
+	DEFiRet;
+
+	CHKiRet(msgBaseConstruct(ppThis));
+	memcpy(&(*ppThis)->tRcvdAt, stTime, sizeof(struct syslogTime));
+	memcpy(&(*ppThis)->tTIMESTAMP, stTime, sizeof(struct syslogTime));
 
+finalize_it:
+	RETiRet;
+}
+
+
+/* "Constructor" for a msg "object". Returns a pointer to
+ * the new object or NULL if no such object could be allocated.
+ * An object constructed via this function should only be destroyed
+ * via "msgDestruct()". This constructor, for historical reasons,
+ * also sets the two timestamps to the current time.
+ */
+rsRetVal msgConstruct(msg_t **ppThis)
+{
+	DEFiRet;
+
+	CHKiRet(msgBaseConstruct(ppThis));
 	/* we initialize both timestamps to contain the current time, so that they
 	 * are consistent. Also, this saves us from doing any further time calls just
 	 * to obtain a timestamp. The memcpy() should not really make a difference,
 	 * especially as I think there is no codepath currently where it would not be
 	 * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02
 	 */
-	datetime.getCurrTime(&(pM->tRcvdAt));
-	memcpy(&pM->tTIMESTAMP, &pM->tRcvdAt, sizeof(struct syslogTime));
-	
-	objConstructSetObjInfo(pM);
-
-	/* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/
-
-	*ppThis = pM;
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime));
 
 finalize_it:
 	RETiRet;
@@ -2492,7 +2538,5 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE)
 	funcDeleteMutex = MsgLockingDummy;
 	funcMsgPrepareEnqueue = MsgLockingDummy;
 ENDObjClassInit(msg)
-
-/*
- * vi:set ai:
+/* vim:set ai:
  */
diff --git a/runtime/msg.h b/runtime/msg.h
index 21cb2c64..d3c0718b 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -119,6 +119,7 @@ struct msg {
 PROTOTYPEObjClassInit(msg);
 char* getProgramName(msg_t*);
 rsRetVal msgConstruct(msg_t **ppThis);
+rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime);
 rsRetVal msgDestruct(msg_t **ppM);
 msg_t* MsgDup(msg_t* pOld);
 msg_t *MsgAddRef(msg_t *pM);
@@ -180,6 +181,5 @@ extern void (*funcMsgPrepareEnqueue)(msg_t *pMsg);
 #define MsgPrepareEnqueue(pMsg) funcMsgPrepareEnqueue(pMsg)
 
 #endif /* #ifndef MSG_H_INCLUDED */
-/*
- * vi:set ai:
+/* vim:set ai:
  */
diff --git a/tcps_sess.c b/tcps_sess.c
index f5420fc0..7ce9f2f8 100644
--- a/tcps_sess.c
+++ b/tcps_sess.c
@@ -230,7 +230,7 @@ PrepareClose(tcps_sess_t *pThis)
 		 */
 		dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n");
 		parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME,
-				      NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */
+				      NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
 		pThis->bAtStrtOfFram = 1;
 	}
 
@@ -313,7 +313,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 			/* emergency, we now need to flush, no matter if we are at end of message or not... */
 			dbgprintf("error: message received is larger than max msg size, we split it\n");
 			parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */
+					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
 			pThis->iMsg = 0;
 			/* we might think if it is better to ignore the rest of the
 			 * message than to treat it as a new one. Maybe this is a good
@@ -324,7 +324,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 
 		if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */
 			parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */
+					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
 			pThis->iMsg = 0;
 			pThis->inputState = eAtStrtFram;
 		} else {
@@ -343,7 +343,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 			if(pThis->iOctetsRemain < 1) {
 				/* we have end of frame! */
 				parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */
+					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
 				pThis->iMsg = 0;
 				pThis->inputState = eAtStrtFram;
 			}
diff --git a/tools/syslogd.c b/tools/syslogd.c
index b576bb6d..a6e17d8f 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -581,9 +581,15 @@ void untty(void)
  * Interface change: added new parameter "InputName", permits the input to provide 
  * a string that identifies it. May be NULL, but must be a valid char* pointer if
  * non-NULL.
+ *
+ * rgerhards, 2008-10-06:
+ * Interface change: added new parameter "stTime", which enables the caller to provide
+ * a timestamp that is to be used as timegenerated instead of the current system time.
+ * This is meant to facilitate performance optimization. Some inputs support such modes.
+ * If stTime is NULL, the current system time is used.
  */
-rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType,
-	uchar *pszInputName)
+static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType,
+	uchar *pszInputName, struct syslogTime *stTime)
 {
 	DEFiRet;
 	register uchar *p;
@@ -591,7 +597,11 @@ rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int
 	msg_t *pMsg;
 
 	/* Now it is time to create the message object (rgerhards) */
-	CHKiRet(msgConstruct(&pMsg));
+	if(stTime == NULL) {
+		CHKiRet(msgConstruct(&pMsg));
+	} else {
+		CHKiRet(msgConstructWithTime(&pMsg, stTime));
+	}
 	if(pszInputName != NULL)
 		MsgSetInputName(pMsg, (char*) pszInputName);
 	MsgSetFlowControlType(pMsg, flowCtlType);
@@ -684,10 +694,16 @@ finalize_it:
  * Interface change: added new parameter "InputName", permits the input to provide 
  * a string that identifies it. May be NULL, but must be a valid char* pointer if
  * non-NULL.
+ *
+ * rgerhards, 2008-10-06:
+ * Interface change: added new parameter "stTime", which enables the caller to provide
+ * a timestamp that is to be used as timegenerated instead of the current system time.
+ * This is meant to facilitate performance optimization. Some inputs support such modes.
+ * If stTime is NULL, the current system time is used.
  */
 rsRetVal
 parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType,
-	uchar *pszInputName)
+	uchar *pszInputName, struct syslogTime *stTime)
 {
 	DEFiRet;
 	register int iMsg;
@@ -714,9 +730,6 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 	 * TODO: optimize buffer handling */
 	iMaxLine = glbl.GetMaxLine();
 	CHKmalloc(tmpline = malloc(sizeof(uchar) * (iMaxLine + 1)));
-#	ifdef USE_NETZIP
-	CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1)));
-#	endif
 
 	/* we first check if we have a NUL character at the very end of the
 	 * message. This seems to be a frequent problem with a number of senders.
@@ -762,6 +775,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 		 */
 		int ret;
 		iLenDefBuf = iMaxLine;
+		CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1)));
 		ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1);
 		dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n",
 		        ret, (long) iLenDefBuf, len-1);
@@ -800,7 +814,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 			 */
 			if(iMsg == iMaxLine) {
 				*(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */
-				printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName);
+				printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime);
 			} else {
 				/* This case in theory never can happen. If it happens, we have
 				 * a logic error. I am checking for it, because if I would not,
@@ -852,7 +866,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 	*(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */
 
 	/* typically, we should end up here! */
-	printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName);
+	printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime);
 
 finalize_it:
 	if(tmpline != NULL)
@@ -1344,6 +1358,13 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 	} else {
 		/* we can not parse, so we get the system we
 		 * received the data from.
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
 		 */
 		MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 	}
-- 
cgit 


From 7b3c05da9f126063384c80e9dd6fd5b2ae610074 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 7 Oct 2008 10:28:44 +0200
Subject: simple (yet efficient) name caching added to imudp

---
 plugins/imudp/imudp.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index 1865d777..d13314e8 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -133,6 +133,15 @@ finalize_it:
 /* This function is called to gather input.
  * Note that udpLstnSocks must be non-NULL because otherwise we would not have
  * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02
+ * rgerhards, 2008-10-07: I have implemented a very simple, yet in most cases probably
+ * highly efficient "name caching". Before querying a name, I now check if the name to be
+ * queried is the same as the one queried in the last message processed. If that is the
+ * case, we can simple re-use the previous value. This algorithm works quite well with
+ * few sender, especially if they emit messages in bursts. The more sender and the
+ * more intermixed messages arrive, the less this algorithm works, but the overhead
+ * is so minimal (a simple memory compare and move) that this does not hurt. Even
+ * with a real name lookup cache, this optimization here is useful as it is quicker
+ * than even a cache lookup).
  */
 BEGINrunInput
 	int maxfds;
@@ -140,6 +149,7 @@ BEGINrunInput
 	int i;
 	fd_set readfds;
 	struct sockaddr_storage frominet;
+	struct sockaddr_storage frominetPrev;
 	socklen_t socklen;
 	uchar fromHost[NI_MAXHOST];
 	uchar fromHostIP[NI_MAXHOST];
@@ -148,6 +158,10 @@ BEGINrunInput
 	struct syslogTime stTime;
 	int iNbrTimeUsed;
 CODESTARTrunInput
+	/* start "name caching" algo by making sure the previous system indicator
+	 * is invalidated.
+	 */
+	memset(&frominetPrev, 0, sizeof(frominetPrev));
 	/* this is an endless loop - it is terminated when the thread is
 	 * signalled to do so. This, however, is handled by the framework,
 	 * right into the sleep below.
@@ -184,7 +198,7 @@ CODESTARTrunInput
 
 	       for (i = 0; nfds && i < *udpLstnSocks; i++) {
 		       if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
-			       socklen = sizeof(frominet);
+				socklen = sizeof(frominet);
 				iNbrTimeUsed = 0;
 			        do {
 					/* we now try to read from the file descriptor until there
@@ -199,7 +213,9 @@ CODESTARTrunInput
 					l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0,
 						    (struct sockaddr *)&frominet, &socklen);
 					if(l > 0) {
-					       if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
+					       if(memcmp(&frominet, &frominetPrev, socklen) == 0 ||
+						  net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
+						       memcpy(&frominetPrev, &frominet, socklen);
 						       dbgprintf("Message from inetd socket: #%d, host: %s\n",
 							       udpLstnSocks[i+1], fromHost);
 						       /* Here we check if a host is permitted to send us
-- 
cgit 


From cdecd7e524a5114ccff4f2909b32738bdb33c6ea Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 7 Oct 2008 11:46:46 +0200
Subject: slightly improved lock contention situation by moving out of

the critical section what could so with acceptable consequences
---
 runtime/queue.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/runtime/queue.c b/runtime/queue.c
index 76c2f10f..f5f770b3 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -2101,6 +2101,15 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr)
 
 	ISOBJ_TYPE_assert(pThis, queue);
 
+	/* first check if we need to discard this message (which will cause CHKiRet() to exit)
+	 * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize
+	 * and bRunsDA parameters may not reflect the correct settings here, but they are
+	 * "good enough" in the sense that they can be used to drive the decision. Valgrind's
+	 * threading tools may point this access to be an error, but this is done
+	 * intentional. I do not see this causes problems to us.
+	 */
+	CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr));
+
 	/* Please note that this function is not cancel-safe and consequently
 	 * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE
 	 * during its execution. If that is not done, race conditions occur if the
@@ -2112,9 +2121,6 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr)
 		d_pthread_mutex_lock(pThis->mut);
 	}
 
-	/* first check if we need to discard this message (which will cause CHKiRet() to exit) */
-	CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr));
-
 	/* then check if we need to add an assistance disk queue */
 	if(pThis->bIsDA)
 		CHKiRet(queueChkStrtDA(pThis));
-- 
cgit 


From 8528344ef58b5d2907bba8809f63d0bca2ce8d38 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 7 Oct 2008 14:26:41 +0200
Subject: "output" timestamp now taken from mesg's time generated

This enhances performance and, as some have pointed out, is
probably also more consistent with what users expect how
the various output-timestamp related function should work. This
commit needs some more testing.
---
 ChangeLog                   |  9 +++++++++
 action.c                    |  6 +++++-
 dirty.h                     |  2 +-
 plugins/imrelp/imrelp.c     |  2 +-
 plugins/imudp/imudp.c       |  6 ++++--
 plugins/imuxsock/imuxsock.c |  2 +-
 runtime/datetime.c          | 12 ++++++++++--
 runtime/datetime.h          |  2 +-
 runtime/msg.c               | 15 +++++++++------
 runtime/msg.h               |  9 ++++++++-
 runtime/sysvar.c            |  2 +-
 tcps_sess.c                 |  8 ++++----
 tools/syslogd.c             | 17 +++++------------
 13 files changed, 59 insertions(+), 33 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 346ff39c..0f3d4591 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,14 @@
 ---------------------------------------------------------------------------
 Version 3.21.6  [DEVEL] (rgerhards), 2008-10-??
+
+********************************* WARNING *********************************
+This version has a slightly different on-disk format for message entries.
+As a consequence, old queue files being read by this version may have
+an invalid output timestamp, which could result to some malfunction inside
+the output driver. It is recommended to drain queues with the previous
+version before switching to this one.
+********************************* WARNING *********************************
+
 - added $UDPServerTimeRequery option which enables to work with
   less acurate timestamps in favor of performance. This enables querying
   of the time only every n-th time if imudp is running in the tight
diff --git a/action.c b/action.c
index a25eca23..cc62f028 100644
--- a/action.c
+++ b/action.c
@@ -602,7 +602,7 @@ actionWriteToAction(action_t *pAction)
 		 * ... RAWMSG is a problem ... Please note that digital
 		 * signatures inside the message are also invalidated.
 		 */
-		datetime.getCurrTime(&(pMsg->tRcvdAt));
+		datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime));
 		memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
 		MsgSetMSG(pMsg, (char*)szRepMsg);
 		MsgSetRawMsg(pMsg, (char*)szRepMsg);
@@ -640,6 +640,7 @@ actionWriteToAction(action_t *pAction)
 	 * while in the on-disk queue. Also need to think about a few other implications.
 	 * rgerhards, 2008-09-17
 	 */
+#if 0
 	{
 		struct tm tTm;
 		tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second;
@@ -655,7 +656,10 @@ actionWriteToAction(action_t *pAction)
 /* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */
 dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL));
 	}
+#endif
 
+		pAction->f_time = pAction->f_pMsg->ttGenTime;
+dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL));
 	//pAction->f_time = getActNow(pAction); /* re-init time flags */
 	/* Note: tLastExec could be set in the if block above, but f_time causes us a hard time
 	 * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16
diff --git a/dirty.h b/dirty.h
index 58c4ea39..8e7ffd84 100644
--- a/dirty.h
+++ b/dirty.h
@@ -40,7 +40,7 @@
 
 rsRetVal submitMsg(msg_t *pMsg);
 rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags);
-rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime);
+rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime);
 
 /* TODO: the following 2 need to go in conf obj interface... */
 rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName);
diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c
index f3771237..4515acd7 100644
--- a/plugins/imrelp/imrelp.c
+++ b/plugins/imrelp/imrelp.c
@@ -84,7 +84,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s
 {
 	DEFiRet;
 	parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME,
-			      NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL);
+			      NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0);
 
 	RETiRet;
 }
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index d13314e8..d58a0d8e 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -155,6 +155,7 @@ BEGINrunInput
 	uchar fromHostIP[NI_MAXHOST];
 	uchar fromHostFQDN[NI_MAXHOST];
 	ssize_t l;
+	time_t ttGenTime;
 	struct syslogTime stTime;
 	int iNbrTimeUsed;
 CODESTARTrunInput
@@ -227,10 +228,11 @@ CODESTARTrunInput
 						       if(net.isAllowedSender(net.pAllowedSenders_UDP,
 							  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
 								if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) {
-									datetime.getCurrTime(&stTime);
+									datetime.getCurrTime(&stTime, &ttGenTime);
 								}
 							       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
-							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime);
+							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp",
+							       &stTime, ttGenTime);
 						       } else {
 							       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
 							       if(glbl.GetOption_DisallowWarning) {
diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c
index 77d347d9..efa0365d 100644
--- a/plugins/imuxsock/imuxsock.c
+++ b/plugins/imuxsock/imuxsock.c
@@ -221,7 +221,7 @@ static rsRetVal readSocket(int fd, int iSock)
 	if (iRcvd > 0) {
 		parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock],
 				      (uchar*)"127.0.0.1", pRcv,
-			 	      iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL);
+			 	      iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0);
 	} else if (iRcvd < 0 && errno != EINTR) {
 		char errStr[1024];
 		rs_strerror_r(errno, errStr, sizeof(errStr));
diff --git a/runtime/datetime.c b/runtime/datetime.c
index 20ca6191..aa1956d7 100644
--- a/runtime/datetime.c
+++ b/runtime/datetime.c
@@ -62,9 +62,14 @@ DEFobjCurrIf(errmsg)
  * most portable and removes the need for additional structures
  * (but I have to admit it is somewhat "bulky";)).
  *
- * Obviously, all caller-provided pointers must not be NULL...
+ * Obviously, *t must not be NULL...
+ *
+ * rgerhards, 2008-10-07: added ttSeconds to provide a way to 
+ * obtain the second-resolution UNIX timestamp. This is needed
+ * in some situations to minimize time() calls (namely when doing
+ * output processing). This can be left NULL if not needed.
  */
-static void getCurrTime(struct syslogTime *t)
+static void getCurrTime(struct syslogTime *t, time_t *ttSeconds)
 {
 	struct timeval tp;
 	struct tm *tm;
@@ -83,6 +88,9 @@ static void getCurrTime(struct syslogTime *t)
 #	else
 		gettimeofday(&tp, NULL);
 #	endif
+	if(ttSeconds != NULL)
+		*ttSeconds = tp.tv_sec;
+
 	tm = localtime_r((time_t*) &(tp.tv_sec), &tmBuf);
 
 	t->year = tm->tm_year + 1900;
diff --git a/runtime/datetime.h b/runtime/datetime.h
index 2210af02..0739588d 100644
--- a/runtime/datetime.h
+++ b/runtime/datetime.h
@@ -35,7 +35,7 @@ typedef struct datetime_s {
 
 /* interfaces */
 BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */
-	void (*getCurrTime)(struct syslogTime *t);
+	void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds);
 	rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, char** ppszTS);
 	rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, char** pszTS);
 	int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst);
diff --git a/runtime/msg.c b/runtime/msg.c
index df8c1572..fd838591 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -286,11 +286,12 @@ finalize_it:
  * udp input).
  * rgerhards, 2008-10-06
  */
-rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime)
+rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime)
 {
 	DEFiRet;
 
 	CHKiRet(msgBaseConstruct(ppThis));
+	(*ppThis)->ttGenTime = ttGenTime;
 	memcpy(&(*ppThis)->tRcvdAt, stTime, sizeof(struct syslogTime));
 	memcpy(&(*ppThis)->tTIMESTAMP, stTime, sizeof(struct syslogTime));
 
@@ -316,7 +317,7 @@ rsRetVal msgConstruct(msg_t **ppThis)
 	 * especially as I think there is no codepath currently where it would not be
 	 * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02
 	 */
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
+	datetime.getCurrTime(&((*ppThis)->tRcvdAt), &((*ppThis)->ttGenTime));
 	memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime));
 
 finalize_it:
@@ -442,7 +443,7 @@ msg_t* MsgDup(msg_t* pOld)
 	assert(pOld != NULL);
 
 	BEGINfunc
-	if(msgConstruct(&pNew) != RS_RET_OK) {
+	if(msgConstructWithTime(&pNew, &pOld->tTIMESTAMP, pOld->ttGenTime) != RS_RET_OK) {
 		return NULL;
 	}
 
@@ -453,8 +454,7 @@ msg_t* MsgDup(msg_t* pOld)
 	pNew->bParseHOSTNAME = pOld->bParseHOSTNAME;
 	pNew->msgFlags = pOld->msgFlags;
 	pNew->iProtocolVersion = pOld->iProtocolVersion;
-	memcpy(&pNew->tRcvdAt, &pOld->tRcvdAt, sizeof(struct syslogTime));
-	memcpy(&pNew->tTIMESTAMP, &pOld->tTIMESTAMP, sizeof(struct syslogTime));
+	pNew->ttGenTime = pOld->ttGenTime;
 	tmpCOPYSZ(Severity);
 	tmpCOPYSZ(SeverityStr);
 	tmpCOPYSZ(Facility);
@@ -508,6 +508,7 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm)
 	objSerializeSCALAR(pStrm, iSeverity, SHORT);
 	objSerializeSCALAR(pStrm, iFacility, SHORT);
 	objSerializeSCALAR(pStrm, msgFlags, INT);
+	objSerializeSCALAR(pStrm, ttGenTime, INT);
 	objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME);
 	objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME);
 
@@ -1669,7 +1670,7 @@ static uchar *getNOW(eNOWType eNow)
 		return NULL;
 	}
 
-	datetime.getCurrTime(&t);
+	datetime.getCurrTime(&t, NULL);
 	switch(eNow) {
 	case NOW_NOW:
 		snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day);
@@ -2477,6 +2478,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp)
 		MsgSetPROCID(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr));
 	} else if(isProp("pCSMSGID")) {
 		MsgSetMSGID(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr));
+ 	} else if(isProp("ttGenTime")) {
+		pThis->ttGenTime = pProp->val.num;
 	} else if(isProp("tRcvdAt")) {
 		memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime));
 	} else if(isProp("tTIMESTAMP")) {
diff --git a/runtime/msg.h b/runtime/msg.h
index d3c0718b..d2fc2f30 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -99,6 +99,13 @@ struct msg {
 	cstr_t *pCSAPPNAME;	/* APP-NAME */
 	cstr_t *pCSPROCID;	/* PROCID */
 	cstr_t *pCSMSGID;	/* MSGID */
+	time_t ttGenTime;	/* time msg object was generated, same as tRcvdAt, but a Unix timestamp.
+				   While this field looks redundant, it is required because a Unix timestamp
+				   is used at later processing stages (namely in the output arena). Thanks to
+				   the subleties of how time is defined, there is no reliable way to reconstruct
+				   the Unix timestamp from the syslogTime fields (in practice, we may be close
+				   enough to reliable, but I prefer to leave the subtle things to the OS, where
+				   it obviously is solved in way or another...). */
 	struct syslogTime tRcvdAt;/* time the message entered this program */
 	char *pszRcvdAt3164;	/* time as RFC3164 formatted string (always 15 charcters) */
 	char *pszRcvdAt3339;	/* time as RFC3164 formatted string (32 charcters at most) */
@@ -119,7 +126,7 @@ struct msg {
 PROTOTYPEObjClassInit(msg);
 char* getProgramName(msg_t*);
 rsRetVal msgConstruct(msg_t **ppThis);
-rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime);
+rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime);
 rsRetVal msgDestruct(msg_t **ppM);
 msg_t* MsgDup(msg_t* pOld);
 msg_t *MsgAddRef(msg_t *pM);
diff --git a/runtime/sysvar.c b/runtime/sysvar.c
index 5eec8f67..c102d1f5 100644
--- a/runtime/sysvar.c
+++ b/runtime/sysvar.c
@@ -84,7 +84,7 @@ getNOW(eNOWType eNow, cstr_t **ppStr)
 	uchar szBuf[16];
 	struct syslogTime t;
 
-	datetime.getCurrTime(&t);
+	datetime.getCurrTime(&t, NULL);
 	switch(eNow) {
 	case NOW_NOW:
 		snprintf((char*) szBuf, sizeof(szBuf)/sizeof(uchar), "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day);
diff --git a/tcps_sess.c b/tcps_sess.c
index 7ce9f2f8..13644f45 100644
--- a/tcps_sess.c
+++ b/tcps_sess.c
@@ -230,7 +230,7 @@ PrepareClose(tcps_sess_t *pThis)
 		 */
 		dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n");
 		parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME,
-				      NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
+				      NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 		pThis->bAtStrtOfFram = 1;
 	}
 
@@ -313,7 +313,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 			/* emergency, we now need to flush, no matter if we are at end of message or not... */
 			dbgprintf("error: message received is larger than max msg size, we split it\n");
 			parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
+					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 			pThis->iMsg = 0;
 			/* we might think if it is better to ignore the rest of the
 			 * message than to treat it as a new one. Maybe this is a good
@@ -324,7 +324,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 
 		if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */
 			parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
+					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 			pThis->iMsg = 0;
 			pThis->inputState = eAtStrtFram;
 		} else {
@@ -343,7 +343,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 			if(pThis->iOctetsRemain < 1) {
 				/* we have end of frame! */
 				parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */
+					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 				pThis->iMsg = 0;
 				pThis->inputState = eAtStrtFram;
 			}
diff --git a/tools/syslogd.c b/tools/syslogd.c
index a6e17d8f..e794e2d1 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -589,7 +589,7 @@ void untty(void)
  * If stTime is NULL, the current system time is used.
  */
 static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType,
-	uchar *pszInputName, struct syslogTime *stTime)
+	uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime)
 {
 	DEFiRet;
 	register uchar *p;
@@ -600,7 +600,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b
 	if(stTime == NULL) {
 		CHKiRet(msgConstruct(&pMsg));
 	} else {
-		CHKiRet(msgConstructWithTime(&pMsg, stTime));
+		CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime));
 	}
 	if(pszInputName != NULL)
 		MsgSetInputName(pMsg, (char*) pszInputName);
@@ -703,7 +703,7 @@ finalize_it:
  */
 rsRetVal
 parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType,
-	uchar *pszInputName, struct syslogTime *stTime)
+	uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime)
 {
 	DEFiRet;
 	register int iMsg;
@@ -814,7 +814,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 			 */
 			if(iMsg == iMaxLine) {
 				*(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */
-				printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime);
+				printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime);
 			} else {
 				/* This case in theory never can happen. If it happens, we have
 				 * a logic error. I am checking for it, because if I would not,
@@ -866,7 +866,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 	*(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */
 
 	/* typically, we should end up here! */
-	printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime);
+	printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime);
 
 finalize_it:
 	if(tmpline != NULL)
@@ -1358,13 +1358,6 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 	} else {
 		/* we can not parse, so we get the system we
 		 * received the data from.
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
-	datetime.getCurrTime(&((*ppThis)->tRcvdAt));
 		 */
 		MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 	}
-- 
cgit 


From 0fa23994669417fff4c4c057ce0c9d1e96f6d56c Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 7 Oct 2008 15:10:03 +0200
Subject: cleanup of output timestamp generation

---
 action.c      | 38 ++------------------------------------
 runtime/msg.c |  4 ++++
 2 files changed, 6 insertions(+), 36 deletions(-)

diff --git a/action.c b/action.c
index cc62f028..ec8732bc 100644
--- a/action.c
+++ b/action.c
@@ -630,42 +630,8 @@ actionWriteToAction(action_t *pAction)
 		FINALIZE;
 	}
 
-
-
-	/* TODO: move this to msg object or some other object. This is just for quick testing!
-	 * ALSO, THIS DOES NOT YET WORK PROPERLY!
-	 * The reason is that we do not know the DST status, which is major pain. I need to
-	 * think about obtaining this information (or the actual Unix timestamp) when I
-	 * create the reception timestamp, but that also means I need to preserve that information
-	 * while in the on-disk queue. Also need to think about a few other implications.
-	 * rgerhards, 2008-09-17
-	 */
-#if 0
-	{
-		struct tm tTm;
-		tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second;
-		tTm.tm_min = pAction->f_pMsg->tRcvdAt.minute;
-		tTm.tm_hour = pAction->f_pMsg->tRcvdAt.hour;
-		tTm.tm_mday = pAction->f_pMsg->tRcvdAt.day;
-		tTm.tm_mon = pAction->f_pMsg->tRcvdAt.month - 1;
-		tTm.tm_year = pAction->f_pMsg->tRcvdAt.year - 1900;
-		/********************************************************************************/
-		tTm.tm_isdst = 1;      /* TODO THIS IS JUST VALID FOR THE NEXT FEW DAYS ;) TODO */
-		/********************************************************************************/
-		pAction->f_time = mktime(&tTm);
-/* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */
-dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL));
-	}
-#endif
-
-		pAction->f_time = pAction->f_pMsg->ttGenTime;
-dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL));
-	//pAction->f_time = getActNow(pAction); /* re-init time flags */
-	/* Note: tLastExec could be set in the if block above, but f_time causes us a hard time
-	 * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16
-	 */
-
-
+	/* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */
+	pAction->f_time = pAction->f_pMsg->ttGenTime;
 
 	/* When we reach this point, we have a valid, non-disabled action.
 	 * So let's enqueue our message for execution. -- rgerhards, 2007-07-24
diff --git a/runtime/msg.c b/runtime/msg.c
index fd838591..c030fa45 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -778,6 +778,7 @@ int getPRIi(msg_t *pM)
 
 char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt)
 {
+	BEGINfunc
 	if(pM == NULL)
 		return "";
 
@@ -849,11 +850,13 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt)
 		MsgUnlock(pM);
 		return(pM->pszTIMESTAMP_SecFrac);
 	}
+	ENDfunc
 	return "INVALID eFmt OPTION!";
 }
 
 char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
 {
+	BEGINfunc
 	if(pM == NULL)
 		return "";
 
@@ -925,6 +928,7 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt)
 		MsgUnlock(pM);
 		return(pM->pszRcvdAt_SecFrac);
 	}
+	ENDfunc
 	return "INVALID eFmt OPTION!";
 }
 
-- 
cgit 


From 82b583c4f99dd9beb30360f222c4d2a1152f75e1 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 8 Oct 2008 14:56:02 +0200
Subject: restructured imudp receive loop

cleaned up previous code and redid it in a way that makes
it much easier to extend it

also added a new macro DBGPRINTF which is a performance-optimzed
version of dbgprintf
---
 plugins/imudp/imudp.c | 145 +++++++++++++++++++++++++++++---------------------
 runtime/debug.h       |   1 +
 2 files changed, 86 insertions(+), 60 deletions(-)

diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index d58a0d8e..7f9afb68 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -130,6 +130,85 @@ finalize_it:
 }
 
 
+/* This function is a helper to runInput. I have extracted it
+ * from the main loop just so that we do not have that large amount of code
+ * in a single place. This function takes a socket and pulls messages from
+ * it until the socket does not have any more waiting.
+ * rgerhards, 2008-01-08
+ * We try to read from the file descriptor until there
+ * is no more data. This is done in the hope to get better performance
+ * out of the system. However, this also means that a descriptor
+ * monopolizes processing while it contains data. This can lead to
+ * data loss in other descriptors. However, if the system is incapable of
+ * handling the workload, we will loss data in any case. So it doesn't really
+ * matter where the actual loss occurs - it is always random, because we depend
+ * on scheduling order. -- rgerhards, 2008-10-02
+ */
+static inline rsRetVal
+processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted,
+	      uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP)
+{
+	DEFiRet;
+	int iNbrTimeUsed;
+	time_t ttGenTime;
+	struct syslogTime stTime;
+	socklen_t socklen;
+	ssize_t l;
+	struct sockaddr_storage frominet;
+	char errStr[1024];
+
+	iNbrTimeUsed = 0;
+	while(1) { /* loop is terminated if we have a bad receive, done below in the body */
+		socklen = sizeof(struct sockaddr_storage);
+		l = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen);
+		if(l < 0) {
+			if(errno != EINTR && errno != EAGAIN) {
+				rs_strerror_r(errno, errStr, sizeof(errStr));
+				DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr);
+				errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet");
+			}
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+
+		/* if we reach this point, we had a good receive and can process the packet received */
+		/* check if we have a different sender than before, if so, we need to query some new values */
+		if(memcmp(&frominet, frominetPrev, socklen) != 0) {
+			CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP));
+			memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */
+			/* Here we check if a host is permitted to send us
+			* syslog messages. If it isn't, we do not further
+			* process the message but log a warning (if we are
+			* configured to do this).
+			* rgerhards, 2005-09-26
+			*/
+			*pbIsPermitted = net.isAllowedSender(net.pAllowedSenders_UDP,
+			  			            (struct sockaddr *)&frominet, (char*)fromHostFQDN);
+	
+			if(!*pbIsPermitted) {
+				DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN);
+				if(glbl.GetOption_DisallowWarning) {
+				       errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded",
+						  (char*)fromHost);
+				}
+			}	
+		}
+
+		DBGPRINTF("Message from inetd socket: #%d, host: %s, isPermitted: %d\n", fd, fromHost, *pbIsPermitted);
+		if(*pbIsPermitted)  {
+			if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) {
+				datetime.getCurrTime(&stTime, &ttGenTime);
+			}
+			parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
+				MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime, ttGenTime);
+		}
+	}
+
+
+finalize_it:
+	RETiRet;
+}
+
+
 /* This function is called to gather input.
  * Note that udpLstnSocks must be non-NULL because otherwise we would not have
  * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02
@@ -148,20 +227,16 @@ BEGINrunInput
 	int nfds;
 	int i;
 	fd_set readfds;
-	struct sockaddr_storage frominet;
 	struct sockaddr_storage frominetPrev;
-	socklen_t socklen;
+	int bIsPermitted;
 	uchar fromHost[NI_MAXHOST];
 	uchar fromHostIP[NI_MAXHOST];
 	uchar fromHostFQDN[NI_MAXHOST];
-	ssize_t l;
-	time_t ttGenTime;
-	struct syslogTime stTime;
-	int iNbrTimeUsed;
 CODESTARTrunInput
 	/* start "name caching" algo by making sure the previous system indicator
 	 * is invalidated.
 	 */
+	bIsPermitted = 0;
 	memset(&frominetPrev, 0, sizeof(frominetPrev));
 	/* this is an endless loop - it is terminated when the thread is
 	 * signalled to do so. This, however, is handled by the framework,
@@ -198,61 +273,11 @@ CODESTARTrunInput
 		nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL);
 
 	       for (i = 0; nfds && i < *udpLstnSocks; i++) {
-		       if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
-				socklen = sizeof(frominet);
-				iNbrTimeUsed = 0;
-			        do {
-					/* we now try to read from the file descriptor until there
-					 * is no more data. This is done in the hope to get better performance
-					 * out of the system. However, this also means that a descriptor
-					 * monopolizes processing while it contains data. This can lead to
-					 * data loss in other descriptors. However, if the system is incapable of
-					 * handling the workload, we will loss data in any case. So it doesn't really
-					 * matter where the actual loss occurs - it is always random, because we depend
-					 * on scheduling order. -- rgerhards, 2008-10-02
-					 */
-					l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0,
-						    (struct sockaddr *)&frominet, &socklen);
-					if(l > 0) {
-					       if(memcmp(&frominet, &frominetPrev, socklen) == 0 ||
-						  net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) {
-						       memcpy(&frominetPrev, &frominet, socklen);
-						       dbgprintf("Message from inetd socket: #%d, host: %s\n",
-							       udpLstnSocks[i+1], fromHost);
-						       /* Here we check if a host is permitted to send us
-							* syslog messages. If it isn't, we do not further
-							* process the message but log a warning (if we are
-							* configured to do this).
-							* rgerhards, 2005-09-26
-							*/
-						       if(net.isAllowedSender(net.pAllowedSenders_UDP,
-							  (struct sockaddr *)&frominet, (char*)fromHostFQDN)) {
-								if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) {
-									datetime.getCurrTime(&stTime, &ttGenTime);
-								}
-							       parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
-							       MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp",
-							       &stTime, ttGenTime);
-						       } else {
-							       dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN);
-							       if(glbl.GetOption_DisallowWarning) {
-								       errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded",
-										  (char*)fromHost);
-							       }	
-						       }
-					       }
-					} else if(l < 0 && errno != EINTR && errno != EAGAIN) {
-						char errStr[1024];
-						rs_strerror_r(errno, errStr, sizeof(errStr));
-						dbgprintf("INET socket error: %d = %s.\n", errno, errStr);
-						       errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet");
-						       /* should be harmless */
-						       sleep(1);
-					}
-				} while(l > 0); /* Warning: do ... while()! */
-
-				--nfds; /* indicate we have processed one descriptor */
+			if (FD_ISSET(udpLstnSocks[i+1], &readfds)) {
+		       		processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted,
+					      fromHost, fromHostFQDN, fromHostIP);
 			}
+			--nfds; /* indicate we have processed one descriptor */
 	       }
 	       /* end of a run, back to loop for next recv() */
 	}
diff --git a/runtime/debug.h b/runtime/debug.h
index d9d576b5..7ac29765 100644
--- a/runtime/debug.h
+++ b/runtime/debug.h
@@ -100,6 +100,7 @@ void dbgSetThrdName(uchar *pszName);
 void dbgPrintAllDebugInfo(void);
 
 /* macros */
+#define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); }
 #ifdef RTINST
 #	define BEGINfunc static dbgFuncDB_t *pdbgFuncDB; int dbgCALLStaCK_POP_POINT = dbgEntrFunc(&pdbgFuncDB, __FILE__, __func__, __LINE__);
 #	define ENDfunc dbgExitFunc(pdbgFuncDB, dbgCALLStaCK_POP_POINT, RS_RET_NO_IRET);
-- 
cgit 


From ace4f2f75202aec39449dac11b9eb1deca7428d7 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 8 Oct 2008 18:55:11 +0200
Subject: reordered imudp processing.

Message parsing is now done as part of main message queue worker
processing (was part of the input thread) This should also improve
performance, as potentially more work is done in parallel.
---
 ChangeLog             |   6 +
 dirty.h               |   3 +
 plugins/imudp/imudp.c |  27 ++++-
 runtime/Makefile.am   |   2 +
 runtime/msg.h         |   7 +-
 runtime/parser.c      | 314 ++++++++++++++++++++++++++++++++++++++++++++++++++
 runtime/parser.h      |  30 +++++
 runtime/queue.c       |   2 -
 runtime/rsyslog.h     |   1 +
 runtime/wti.c         |   6 +-
 tools/syslogd.c       |  18 ++-
 11 files changed, 396 insertions(+), 20 deletions(-)
 create mode 100644 runtime/parser.c
 create mode 100644 runtime/parser.h

diff --git a/ChangeLog b/ChangeLog
index 0f3d4591..2f9f34a6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -9,6 +9,12 @@ the output driver. It is recommended to drain queues with the previous
 version before switching to this one.
 ********************************* WARNING *********************************
 
+- reordered imudp processing. Message parsing is now done as part of main
+  message queue worker processing (was part of the input thread)
+  This should also improve performance, as potentially more work is
+  done in parallel.
+- bugfix: compressed syslog messages could be slightly mis-uncompressed
+  if the last byte of the compressed record was a NUL
 - added $UDPServerTimeRequery option which enables to work with
   less acurate timestamps in favor of performance. This enables querying
   of the time only every n-th time if imudp is running in the tight
diff --git a/dirty.h b/dirty.h
index 8e7ffd84..f5d0415e 100644
--- a/dirty.h
+++ b/dirty.h
@@ -61,6 +61,9 @@ extern int  bReduceRepeatMsgs;
 #define	BACKOFF(f)	{ if (++(f)->f_repeatcount > MAXREPEAT) \
 				 (f)->f_repeatcount = MAXREPEAT; \
 			}
+extern int bDropTrailingLF;
+extern uchar cCCEscapeChar;
+extern int  bEscapeCCOnRcv;
 #ifdef USE_NETZIP
 /* config param: minimum message size to try compression. The smaller
  * the message, the less likely is any compression gain. We check for
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index 7f9afb68..aab201a9 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -40,6 +40,8 @@
 #include "srUtils.h"
 #include "errmsg.h"
 #include "glbl.h"
+#include "msg.h"
+#include "parser.h"
 #include "datetime.h"
 
 MODULE_TYPE_INPUT
@@ -153,15 +155,16 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted,
 	time_t ttGenTime;
 	struct syslogTime stTime;
 	socklen_t socklen;
-	ssize_t l;
+	ssize_t lenRcvBuf;
 	struct sockaddr_storage frominet;
+	msg_t *pMsg;
 	char errStr[1024];
 
 	iNbrTimeUsed = 0;
 	while(1) { /* loop is terminated if we have a bad receive, done below in the body */
 		socklen = sizeof(struct sockaddr_storage);
-		l = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen);
-		if(l < 0) {
+		lenRcvBuf = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen);
+		if(lenRcvBuf < 0) {
 			if(errno != EINTR && errno != EAGAIN) {
 				rs_strerror_r(errno, errStr, sizeof(errStr));
 				DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr);
@@ -193,13 +196,25 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted,
 			}	
 		}
 
-		DBGPRINTF("Message from inetd socket: #%d, host: %s, isPermitted: %d\n", fd, fromHost, *pbIsPermitted);
+		DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf);
+
 		if(*pbIsPermitted)  {
 			if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) {
 				datetime.getCurrTime(&stTime, &ttGenTime);
 			}
-			parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l,
-				MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime, ttGenTime);
+			/* we now create our own message object and submit it to the queue */
+			CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime));
+			/* first trim the buffer to what we have actually received */
+			CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf));
+			memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf);
+			pMsg->iLenRawMsg = lenRcvBuf;
+			MsgSetInputName(pMsg, "imudp");
+			MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY);
+			pMsg->bParseHOSTNAME  = MSG_PARSE_HOSTNAME;
+			pMsg->msgFlags  = NOFLAG;
+			MsgSetRcvFrom(pMsg, (char*)fromHost);
+			CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP));
+			CHKiRet(submitMsg(pMsg));
 		}
 	}
 
diff --git a/runtime/Makefile.am b/runtime/Makefile.am
index 81a9d5bd..7c70dd8d 100644
--- a/runtime/Makefile.am
+++ b/runtime/Makefile.am
@@ -16,6 +16,8 @@ librsyslog_la_SOURCES = \
 	glbl.c \
 	conf.c \
 	conf.h \
+	parser.h \
+	parser.c \
 	msg.c \
 	msg.h \
 	linkedlist.c \
diff --git a/runtime/msg.h b/runtime/msg.h
index d2fc2f30..98635f85 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -52,7 +52,10 @@ struct msg {
 	BEGINobjInstance;	/* Data to implement generic object - MUST be the first data element! */
 	pthread_mutexattr_t mutAttr;
 	pthread_mutex_t mut;
-	int	iRefCount;	/* reference counter (0 = unused) */
+	flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because
+				        once data has entered the queue, this property is no longer needed. */
+	short	iRefCount;	/* reference counter (0 = unused) */
+	short	bIsParsed;	/* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */
 	short	bParseHOSTNAME;	/* should the hostname be parsed from the message? */
 	   /* background: the hostname is not present on "regular" messages
 	    * received via UNIX domain sockets from the same machine. However,
@@ -60,8 +63,6 @@ struct msg {
 	    * sockets. All in all, the parser would need parse templates, that would
 	    * resolve all these issues... rgerhards, 2005-10-06
 	    */
-	flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because
-				        once data has entered the queue, this property is no longer needed. */
 	short	iSeverity;	/* the severity 0..7 */
 	uchar *pszSeverity;	/* severity as string... */
 	int iLenSeverity;	/* ... and its length. */
diff --git a/runtime/parser.c b/runtime/parser.c
new file mode 100644
index 00000000..8c4272a0
--- /dev/null
+++ b/runtime/parser.c
@@ -0,0 +1,314 @@
+/* parser.c
+ * This module contains functions for message parsers. It still needs to be
+ * converted into an object (and much extended).
+ *
+ * Module begun 2008-10-09 by Rainer Gerhards (based on previous code from syslogd.c)
+ *
+ * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#include "config.h"
+#include <stdlib.h>
+#include <ctype.h>
+#include <assert.h>
+#ifdef USE_NETZIP
+#include <zlib.h>
+#endif
+
+#include "rsyslog.h"
+#include "dirty.h"
+#include "msg.h"
+#include "obj.h"
+#include "errmsg.h"
+
+/* some defines */
+#define DEFUPRI		(LOG_USER|LOG_NOTICE)
+
+#warning "msg object must be updated with new property for persisting the queue!"
+/* definitions for objects we access */
+DEFobjStaticHelpers
+DEFobjCurrIf(glbl)
+DEFobjCurrIf(errmsg)
+
+/* static data */
+
+
+/* this is a dummy class init
+ */
+rsRetVal parserClassInit(void)
+{
+	DEFiRet;
+
+	/* request objects we use */
+	CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */
+	CHKiRet(objUse(glbl, CORE_COMPONENT));
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+// TODO: free components! see action.c
+finalize_it:
+	RETiRet;
+}
+
+
+/* uncompress a received message if it is compressed.
+ * pMsg->pszRawMsg buffer is updated.
+ * rgerhards, 2008-10-09
+ */
+static inline rsRetVal uncompressMessage(msg_t *pMsg)
+{
+	DEFiRet;
+#	ifdef USE_NETZIP
+	uchar *deflateBuf = NULL;
+	uLongf iLenDefBuf;
+	uchar *pszMsg;
+	size_t lenMsg;
+	
+	assert(pMsg != NULL);
+	pszMsg = pMsg->pszRawMsg;
+	lenMsg = pMsg->iLenRawMsg;
+
+	/* we first need to check if we have a compressed record. If so,
+	 * we must decompress it.
+	 */
+	if(lenMsg > 0 && *pszMsg == 'z') { /* compressed data present? (do NOT change order if conditions!) */
+		/* we have compressed data, so let's deflate it. We support a maximum
+		 * message size of iMaxLine. If it is larger, an error message is logged
+		 * and the message is dropped. We do NOT try to decompress larger messages
+		 * as such might be used for denial of service. It might happen to later
+		 * builds that such functionality be added as an optional, operator-configurable
+		 * feature.
+		 */
+		int ret;
+		iLenDefBuf = glbl.GetMaxLine();
+		CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1)));
+		ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1);
+		DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n",
+		        ret, (long) iLenDefBuf, (int) (lenMsg-1));
+		/* Now check if the uncompression worked. If not, there is not much we can do. In
+		 * that case, we log an error message but ignore the message itself. Storing the
+		 * compressed text is dangerous, as it contains control characters. So we do
+		 * not do this. If someone would like to have a copy, this code here could be
+		 * modified to do a hex-dump of the buffer in question. We do not include
+		 * this functionality right now.
+		 * rgerhards, 2006-12-07
+		 */
+		if(ret != Z_OK) {
+			errmsg.LogError(0, NO_ERRCODE, "Uncompression of a message failed with return code %d "
+			            "- enable debug logging if you need further information. "
+				    "Message ignored.", ret);
+			FINALIZE; /* unconditional exit, nothing left to do... */
+		}
+		free(pMsg->pszRawMsg);
+		pMsg->pszRawMsg = deflateBuf;
+		pMsg->iLenRawMsg = iLenDefBuf;
+		deflateBuf = NULL; /* logically "freed" - caller is now responsible */
+	}
+finalize_it:
+	if(deflateBuf != NULL)
+		free(deflateBuf);
+
+#	else /* ifdef USE_NETZIP */
+
+	/* in this case, we still need to check if the message is compressed. If so, we must
+	 * tell the user we can not accept it.
+	 */
+	if(len > 0 && *msg == 'z') {
+		errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression "
+		         "support enabled. The message will be ignored.");
+		ABORT_FINALIZE(RS_RET_NO_ZIP);
+	}	
+
+#	endif /* ifdef USE_NETZIP */
+
+	RETiRet;
+}
+
+
+/* sanitize a received message
+ * if a message gets to large during sanitization, it is truncated. This is
+ * as specified in the upcoming syslog RFC series.
+ * rgerhards, 2008-10-09
+ * We check if we have a NUL character at the very end of the
+ * message. This seems to be a frequent problem with a number of senders.
+ * So I have now decided to drop these NULs. However, if they are intentional,
+ * that may cause us some problems, e.g. with syslog-sign. On the other hand,
+ * current code always has problems with intentional NULs (as it needs to escape
+ * them to prevent problems with the C string libraries), so that does not
+ * really matter. Just to be on the save side, we'll log destruction of such
+ * NULs in the debug log.
+ * rgerhards, 2007-09-14
+ */
+static inline rsRetVal
+sanitizeMessage(msg_t *pMsg)
+{
+	DEFiRet;
+	uchar *pszMsg;
+	uchar *pDst; /* destination for copy job */
+	size_t lenMsg;
+	size_t iSrc;
+	size_t iDst;
+	size_t iMaxLine;
+
+	assert(pMsg != NULL);
+
+#	ifdef USE_NETZIP
+	CHKiRet(uncompressMessage(pMsg));
+#	endif
+
+	pszMsg = pMsg->pszRawMsg;
+	lenMsg = pMsg->iLenRawMsg;
+
+	/* remove NUL character at end of message (see comment in function header) */
+	if(pszMsg[lenMsg-1] == '\0') {
+		DBGPRINTF("dropped NUL at very end of message\n");
+		lenMsg--;
+	}
+
+	/* then we check if we need to drop trailing LFs, which often make
+	 * their way into syslog messages unintentionally. In order to remain
+	 * compatible to recent IETF developments, we allow the user to
+	 * turn on/off this handling.  rgerhards, 2007-07-23
+	 */
+	if(bDropTrailingLF && pszMsg[lenMsg-1] == '\n') {
+		DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n");
+		lenMsg--;
+	}
+
+	/* now copy over the message and sanitize it */
+	/* TODO: can we get cheaper memory alloc? {alloca()?}*/
+	iMaxLine = glbl.GetMaxLine();
+	CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1)));
+	iSrc = iDst = 0;
+	while(iSrc < lenMsg && iDst < iMaxLine) {
+		if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */
+			/* changed to the sequence (somewhat) proposed in
+			 * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30
+			 */
+			if(iDst + 3 < iMaxLine) { /* do we have space? */
+				pDst[iDst++] =  cCCEscapeChar;
+				pDst[iDst++] = '0';
+				pDst[iDst++] = '0';
+				pDst[iDst++] = '0';
+			} /* if we do not have space, we simply ignore the '\0'... */
+			  /* log an error? Very questionable... rgerhards, 2006-11-30 */
+			  /* decided: we do not log an error, it won't help... rger, 2007-06-21 */
+		} else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) {
+			/* we are configured to escape control characters. Please note
+			 * that this most probably break non-western character sets like
+			 * Japanese, Korean or Chinese. rgerhards, 2007-07-17
+			 * Note: sysklogd logs octal values only for DEL and CCs above 127.
+			 * For others, it logs ^n where n is the control char converted to an
+			 * alphabet character. We like consistency and thus escape it to octal
+			 * in all cases. If someone complains, we may change the mode. At least
+			 * we known now what's going on.
+			 * rgerhards, 2007-07-17
+			 */
+			if(iDst + 3 < iMaxLine) { /* do we have space? */
+				pDst[iDst++] = cCCEscapeChar;
+				pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6);
+				pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3);
+				pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007));
+			} /* again, if we do not have space, we ignore the char - see comment at '\0' */
+		} else {
+			pDst[iDst++] = pszMsg[iSrc];
+		}
+		++iSrc;
+	}
+	pDst[iDst] = '\0'; /* space *is* reserved for this! */
+
+	/* we have a sanitized string. Let's save it now */
+	free(pMsg->pszRawMsg);
+	if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) {
+		/* when we get no new buffer, we use what we already have ;) */
+		pMsg->pszRawMsg = pDst;
+	} else {
+		/* trim buffer */
+		memcpy(pMsg->pszRawMsg, pDst, iDst+1);
+		free(pDst); /* too big! */
+		pMsg->iLenRawMsg = iDst;
+	}
+
+finalize_it:
+	RETiRet;
+}
+
+/* Parse a received message. The object's rawmsg property is taken and
+ * parsed according to the relevant standards. This can later be
+ * extended to support configured parsers.
+ * rgerhards, 2008-10-09
+ */
+rsRetVal parseMsg(msg_t *pMsg)
+{
+	DEFiRet;
+	uchar *msg;
+	int pri;
+
+	CHKiRet(sanitizeMessage(pMsg));
+
+	/* we needed to sanitize first, because we otherwise do not have a C-string we can print... */
+	DBGPRINTF("msg parser: flags %x, from '%s', msg %s\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg);
+
+	/* pull PRI */
+	pri = DEFUPRI;
+	msg = pMsg->pszRawMsg;
+	if(*msg == '<') {
+		pri = 0;
+		while(isdigit((int) *++msg)) {
+			pri = 10 * pri + (*msg - '0');
+		}
+		if(*msg == '>')
+			++msg;
+		if(pri & ~(LOG_FACMASK|LOG_PRIMASK))
+			pri = DEFUPRI;
+	}
+	pMsg->iFacility = LOG_FAC(pri);
+	pMsg->iSeverity = LOG_PRI(pri);
+	MsgSetUxTradMsg(pMsg, (char*) msg);
+
+	if(pMsg->bParseHOSTNAME == 0)
+		MsgSetHOSTNAME(pMsg, (char*) pMsg->pszRcvFrom);
+
+	/* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have
+	 * a traditional syslog message or one formatted according to syslog-protocol.
+	 * We need to apply different parsers depending on that. We use the
+	 * -protocol VERSION field for the detection.
+	 */
+	if(msg[0] == '1' && msg[1] == ' ') {
+		dbgprintf("Message has syslog-protocol format.\n");
+		setProtocolVersion(pMsg, 1);
+		if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { // TODO: parseRFC... should pull flags from pMsg
+			msgDestruct(&pMsg);
+			ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases!
+		}
+	} else { /* we have legacy syslog */
+		dbgprintf("Message has legacy syslog format.\n");
+		setProtocolVersion(pMsg, 0);
+		if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) {
+			msgDestruct(&pMsg);
+			ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases!
+		}
+	}
+
+	/* finalize message object */
+	pMsg->bIsParsed = 1; /* this message is now parsed */
+	MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */
+
+finalize_it:
+	RETiRet;
+}
diff --git a/runtime/parser.h b/runtime/parser.h
new file mode 100644
index 00000000..cec9c083
--- /dev/null
+++ b/runtime/parser.h
@@ -0,0 +1,30 @@
+/* header for parser.c
+ * This is not yet an object, but contains all those code necessary to
+ * parse syslog messages.
+ *
+ * Copyright 2008 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of the rsyslog runtime library.
+ *
+ * The rsyslog runtime library is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * The rsyslog runtime library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the rsyslog runtime library.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution.
+ */
+#ifndef INCLUDED_PARSE_H
+#define INCLUDED_PARSE_H
+
+extern rsRetVal parserClassInit(void);
+extern rsRetVal parseMsg(msg_t*);
+
+#endif /* #ifndef INCLUDED_PARSE_H */
diff --git a/runtime/queue.c b/runtime/queue.c
index f5f770b3..25c0bd5f 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1513,8 +1513,6 @@ queueRateLimiter(queue_t *pThis)
 
 	ISOBJ_TYPE_assert(pThis, queue);
 
-	dbgoprint((obj_t*) pThis, "entering rate limiter\n");
-
 	iDelay = 0;
 	if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */
 		/* time calls are expensive, so only do them when needed */
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 361bfb47..619343bd 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -252,6 +252,7 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_QUEUE_FULL = -2105, /**< queue is full, operation could not be completed */
 	RS_RET_ACCEPT_ERR = -2106, /**< error during accept() system call */
 	RS_RET_INVLD_TIME = -2107, /**< invalid timestamp (e.g. could not be parsed) */
+	RS_RET_NO_ZIP = -2108, /**< ZIP functionality is not present */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
diff --git a/runtime/wti.c b/runtime/wti.c
index 365b25d5..4abca090 100644
--- a/runtime/wti.c
+++ b/runtime/wti.c
@@ -299,7 +299,7 @@ wtiWorkerCancelCleanup(void *arg)
 	pWtp = pThis->pWtp;
 	ISOBJ_TYPE_assert(pWtp, wtp);
 
-	dbgprintf("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis));
+	DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis));
 	
 	/* call user supplied handler (that one e.g. requeues the element) */
 	pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp);
@@ -391,7 +391,7 @@ wtiWorker(wti_t *pThis)
 		/* if we reach this point, we are still protected by the mutex */
 
 		if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) {
-			dbgprintf("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis));
+			DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis));
 			pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED);
 
 			if(pWtp->toWrkShutdown == -1) {
@@ -400,7 +400,7 @@ wtiWorker(wti_t *pThis)
 			} else {
 				timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */
 				if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) {
-					dbgprintf("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis));
+					DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis));
 					bInactivityTOOccured = 1; /* indicate we had a timeout */
 				}
 			}
diff --git a/tools/syslogd.c b/tools/syslogd.c
index e794e2d1..1a26333d 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -128,6 +128,7 @@
 #include "vm.h"
 #include "errmsg.h"
 #include "datetime.h"
+#include "parser.h"
 #include "sysvar.h"
 
 /* definitions for objects we access */
@@ -249,15 +250,15 @@ typedef struct legacyOptsLL_s {
 legacyOptsLL_t *pLegacyOptsLL = NULL;
 
 /* global variables for config file state */
-static int	bDropTrailingLF = 1; /* drop trailing LF's on reception? */
+int	bDropTrailingLF = 1; /* drop trailing LF's on reception? */
 int	iCompatibilityMode = 0;		/* version we should be compatible with; 0 means sysklogd. It is
 					   the default, so if no -c<n> option is given, we make ourselvs
 					   as compatible to sysklogd as possible. */
 static int	bDebugPrintTemplateList = 1;/* output template list in debug mode? */
 static int	bDebugPrintCfSysLineHandlerList = 1;/* output cfsyslinehandler list in debug mode? */
 static int	bDebugPrintModuleList = 1;/* output module list in debug mode? */
-static uchar	cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */
-static int 	bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */
+uchar	cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */
+int 	bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */
 static int	bErrMsgToStderr = 1; /* print error messages to stderr (in addition to everything else)? */
 int 	bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */
 int	bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */
@@ -596,6 +597,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b
 	int pri;
 	msg_t *pMsg;
 
+	pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */
 	/* Now it is time to create the message object (rgerhards) */
 	if(stTime == NULL) {
 		CHKiRet(msgConstruct(&pMsg));
@@ -1190,6 +1192,9 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr)
 
 	assert(pMsg != NULL);
 
+	if(pMsg->bIsParsed == 0) {
+		parseMsg(pMsg);
+	}
 	processMsg(pMsg);
 	msgDestruct(&pMsg);
 
@@ -1311,7 +1316,7 @@ static int parseRFCStructuredData(char **pp2parse, char *pResult)
  *
  * rger, 2005-11-24
  */
-static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
+int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 {
 	char *p2parse;
 	char *pBuf;
@@ -1407,7 +1412,7 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
  * but I thought I log it in this comment.
  * rgerhards, 2006-01-10
  */
-static int parseLegacySyslogMsg(msg_t *pMsg, int flags)
+int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 {
 	char *p2parse;
 	char *pBuf;
@@ -2908,6 +2913,8 @@ InitGlobalClasses(void)
 	CHKiRet(actionClassInit());
 	pErrObj = "template";
 	CHKiRet(templateInit());
+	pErrObj = "parser";
+	CHKiRet(parserClassInit());
 
 	/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */
 	pErrObj = "net";
@@ -3526,6 +3533,5 @@ int main(int argc, char **argv)
 	dbgClassInit();
 	return realMain(argc, argv);
 }
-
 /* vim:set ai:
  */
-- 
cgit 


From 5742f9cdd0da18a3ddfb0a51a981637124a6ab25 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 9 Oct 2008 07:48:22 +0200
Subject: fixing segfault caused by all inputs but imudp

---
 runtime/msg.c   | 1 +
 tools/syslogd.c | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/runtime/msg.c b/runtime/msg.c
index c030fa45..e52c9e3f 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -264,6 +264,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 
 	/* initialize members that are non-zero */
 	pM->iRefCount = 1;
+	pM->bIsParsed = 1; /* first we assume this is parsed. If not, input must re-set to 0 */
 	pM->iSeverity = -1;
 	pM->iFacility = -1;
 	objConstructSetObjInfo(pM);
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 1a26333d..a45942fa 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -597,13 +597,13 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b
 	int pri;
 	msg_t *pMsg;
 
-	pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */
 	/* Now it is time to create the message object (rgerhards) */
 	if(stTime == NULL) {
 		CHKiRet(msgConstruct(&pMsg));
 	} else {
 		CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime));
 	}
+	pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */
 	if(pszInputName != NULL)
 		MsgSetInputName(pMsg, (char*) pszInputName);
 	MsgSetFlowControlType(pMsg, flowCtlType);
@@ -1192,6 +1192,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr)
 
 	assert(pMsg != NULL);
 
+RUNLOG_VAR("%d", pMsg->bIsParsed);
 	if(pMsg->bIsParsed == 0) {
 		parseMsg(pMsg);
 	}
-- 
cgit 


From 39f2afc8b8a60f09570fcf288e6899cdd42209ba Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 9 Oct 2008 09:16:31 +0200
Subject: bumped interface version number to reflect change to message parsing

---
 configure.ac      | 2 +-
 runtime/modules.h | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 6b6a699d..e1bc248c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[3.21.5],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[3.21.6-Test1],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 AC_CONFIG_SRCDIR([ChangeLog])
 AC_CONFIG_HEADERS([config.h])
diff --git a/runtime/modules.h b/runtime/modules.h
index 7d34bcf7..96016082 100644
--- a/runtime/modules.h
+++ b/runtime/modules.h
@@ -44,8 +44,11 @@
  * rgerhards, 2008-03-04
  * version 3 adds modInfo_t ptr to call of modInit -- rgerhards, 2008-03-10
  * version 4 removes needUDPSocket OM callback -- rgerhards, 2008-03-22
+ * version 5 changes the way parsing works for input modules. This is
+ *           an important change, parseAndSubmitMessage() goes away. Other
+ *           module types are not affected. -- rgerhards, 2008-10-09
  */
-#define CURR_MOD_IF_VERSION 4
+#define CURR_MOD_IF_VERSION 5
 
 typedef enum eModType_ {
 	eMOD_IN,	/* input module */
-- 
cgit 


From 634cfdbe8ec7182b7fb74b184d269697e5fcab6c Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 9 Oct 2008 10:01:30 +0200
Subject: restoring msg parsing for imudp

I tried to work too quick this morning. A side-effect of an
earlier change was that no UDP messages were parsed, which
lead to their loss, because no PRI was set in this case.
---
 plugins/imudp/imudp.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index aab201a9..114b433c 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -207,6 +207,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted,
 			/* first trim the buffer to what we have actually received */
 			CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf));
 			memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf);
+			pMsg->bIsParsed = 0; /* indicate message needs to be parsed */
 			pMsg->iLenRawMsg = lenRcvBuf;
 			MsgSetInputName(pMsg, "imudp");
 			MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY);
-- 
cgit 


From af3c944563b8651c14b2b9087ea76f7eeacfc065 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 9 Oct 2008 11:21:24 +0200
Subject: added experimental pthread_yield()

which so far seems to increase performance. There is also
reason for it to do so, see http://kb.monitorware.com/post14216.html#p14216
---
 runtime/queue.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/runtime/queue.c b/runtime/queue.c
index 25c0bd5f..3ef6d5a0 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -2185,6 +2185,11 @@ finalize_it:
 		/* and release the mutex */
 		d_pthread_mutex_unlock(pThis->mut);
 		pthread_setcancelstate(iCancelStateSave, NULL);
+		/* the following pthread_yield is experimental, but brought us performance
+		 * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216
+		 * rgerhards, 2008-10-09
+		 */
+		pthread_yield();
 	}
 
 	RETiRet;
-- 
cgit 


From 1229143ca3d194bb0daaa5252809d59ee0c3a0bf Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 9 Oct 2008 12:56:28 +0200
Subject: minor: reorder to slightly reduce size of critical section

---
 runtime/queue.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/queue.c b/runtime/queue.c
index 3ef6d5a0..2d1bf7e3 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -2181,10 +2181,10 @@ finalize_it:
 	if(pThis->qType != QUEUETYPE_DIRECT) {
 		/* make sure at least one worker is running. */
 		queueAdviseMaxWorkers(pThis);
-		dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n");
 		/* and release the mutex */
 		d_pthread_mutex_unlock(pThis->mut);
 		pthread_setcancelstate(iCancelStateSave, NULL);
+		dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n");
 		/* the following pthread_yield is experimental, but brought us performance
 		 * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216
 		 * rgerhards, 2008-10-09
-- 
cgit 


From 6c6e9a0f3f7d454ba9553a750b195d7f99c7299a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 9 Oct 2008 13:45:56 +0200
Subject: moved bParseHostname and bIsParsed to msgFlags

This enables us to use more efficient calling conventions and
also helps us keep the on-disk structure of a msg object more
consistent in future releases.
---
 dirty.h                     | 13 +------------
 plugins/immark/immark.c     |  1 +
 plugins/imrelp/imrelp.c     |  5 +++--
 plugins/imudp/imudp.c       |  4 +---
 plugins/imuxsock/imuxsock.c |  4 +++-
 runtime/atomic.h            |  2 ++
 runtime/debug.h             |  1 +
 runtime/msg.c               |  1 -
 runtime/msg.h               | 13 ++++++++++++-
 runtime/parser.c            |  3 +--
 runtime/queue.c             |  5 +++--
 tcps_sess.c                 | 11 ++++++-----
 tools/syslogd.c             | 21 ++++++++++++---------
 13 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/dirty.h b/dirty.h
index f5d0415e..4b13adcd 100644
--- a/dirty.h
+++ b/dirty.h
@@ -27,20 +27,9 @@
 #ifndef	DIRTY_H_INCLUDED
 #define	DIRTY_H_INCLUDED 1
 
-/* Flags to logmsg().
- */
-#define NOFLAG		0x000	/* no flag is set (to be used when a flag must be specified and none is required) */
-#define INTERNAL_MSG	0x001	/* msg generated by logmsgInternal() --> special handling */
-/* NO LONGER USED: #define SYNC_FILE	0x002	/ * do fsync on file after printing */
-#define IGNDATE		0x004	/* ignore, if given, date in message and use date of reception as msg date */
-#define MARK		0x008	/* this message is a mark */
-
-#define MSG_PARSE_HOSTNAME 1
-#define MSG_DONT_PARSE_HOSTNAME 0
-
 rsRetVal submitMsg(msg_t *pMsg);
 rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags);
-rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime);
+rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime);
 
 /* TODO: the following 2 need to go in conf obj interface... */
 rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName);
diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c
index 323da3fe..8504f872 100644
--- a/plugins/immark/immark.c
+++ b/plugins/immark/immark.c
@@ -41,6 +41,7 @@
 #include "cfsysline.h"
 #include "module-template.h"
 #include "errmsg.h"
+#include "msg.h"
 
 MODULE_TYPE_INPUT
 
diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c
index 4515acd7..2255e643 100644
--- a/plugins/imrelp/imrelp.c
+++ b/plugins/imrelp/imrelp.c
@@ -42,6 +42,7 @@
 #include "cfsysline.h"
 #include "module-template.h"
 #include "net.h"
+#include "msg.h"
 
 MODULE_TYPE_INPUT
 
@@ -83,8 +84,8 @@ static relpRetVal
 onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, size_t lenMsg)
 {
 	DEFiRet;
-	parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME,
-			      NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0);
+	parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, PARSE_HOSTNAME,
+			      eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0);
 
 	RETiRet;
 }
diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c
index 114b433c..a49378cf 100644
--- a/plugins/imudp/imudp.c
+++ b/plugins/imudp/imudp.c
@@ -207,12 +207,10 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted,
 			/* first trim the buffer to what we have actually received */
 			CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf));
 			memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf);
-			pMsg->bIsParsed = 0; /* indicate message needs to be parsed */
 			pMsg->iLenRawMsg = lenRcvBuf;
 			MsgSetInputName(pMsg, "imudp");
 			MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY);
-			pMsg->bParseHOSTNAME  = MSG_PARSE_HOSTNAME;
-			pMsg->msgFlags  = NOFLAG;
+			pMsg->msgFlags  = NEEDS_PARSING | PARSE_HOSTNAME;
 			MsgSetRcvFrom(pMsg, (char*)fromHost);
 			CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP));
 			CHKiRet(submitMsg(pMsg));
diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c
index efa0365d..1d88a2b5 100644
--- a/plugins/imuxsock/imuxsock.c
+++ b/plugins/imuxsock/imuxsock.c
@@ -42,6 +42,7 @@
 #include "errmsg.h"
 #include "net.h"
 #include "glbl.h"
+#include "msg.h"
 
 MODULE_TYPE_INPUT
 
@@ -221,7 +222,8 @@ static rsRetVal readSocket(int fd, int iSock)
 	if (iRcvd > 0) {
 		parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock],
 				      (uchar*)"127.0.0.1", pRcv,
-			 	      iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0);
+			 	      iRcvd, funixParseHost[iSock] ? (funixFlags[iSock] | PARSE_HOSTNAME) : funixFlags[iSock],
+				      funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0);
 	} else if (iRcvd < 0 && errno != EINTR) {
 		char errStr[1024];
 		rs_strerror_r(errno, errStr, sizeof(errStr));
diff --git a/runtime/atomic.h b/runtime/atomic.h
index 2dbe7f52..7ad8e2e4 100644
--- a/runtime/atomic.h
+++ b/runtime/atomic.h
@@ -42,12 +42,14 @@
  */
 #ifdef HAVE_ATOMIC_BUILTINS
 #	define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1))
+#	define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1))
 #	define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1)
 #	define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff))
 #	define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1)
 #else
 #	warning "atomic builtins not available, using nul operations"
 #	define ATOMIC_INC(data) (++(data))
+#	define ATOMIC_DEC(data) (--(data))
 #	define ATOMIC_DEC_AND_FETCH(data) (--(data))
 #	define ATOMIC_FETCH_32BIT(data) (data)
 #	define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1
diff --git a/runtime/debug.h b/runtime/debug.h
index 7ac29765..59b3e776 100644
--- a/runtime/debug.h
+++ b/runtime/debug.h
@@ -101,6 +101,7 @@ void dbgPrintAllDebugInfo(void);
 
 /* macros */
 #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); }
+#define DBGOPRINT(...) if(Debug) { dbgoprint(__VA_ARGS__); }
 #ifdef RTINST
 #	define BEGINfunc static dbgFuncDB_t *pdbgFuncDB; int dbgCALLStaCK_POP_POINT = dbgEntrFunc(&pdbgFuncDB, __FILE__, __func__, __LINE__);
 #	define ENDfunc dbgExitFunc(pdbgFuncDB, dbgCALLStaCK_POP_POINT, RS_RET_NO_IRET);
diff --git a/runtime/msg.c b/runtime/msg.c
index e52c9e3f..c030fa45 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -264,7 +264,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 
 	/* initialize members that are non-zero */
 	pM->iRefCount = 1;
-	pM->bIsParsed = 1; /* first we assume this is parsed. If not, input must re-set to 0 */
 	pM->iSeverity = -1;
 	pM->iFacility = -1;
 	objConstructSetObjInfo(pM);
diff --git a/runtime/msg.h b/runtime/msg.h
index 98635f85..d98111a8 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -55,7 +55,6 @@ struct msg {
 	flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because
 				        once data has entered the queue, this property is no longer needed. */
 	short	iRefCount;	/* reference counter (0 = unused) */
-	short	bIsParsed;	/* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */
 	short	bParseHOSTNAME;	/* should the hostname be parsed from the message? */
 	   /* background: the hostname is not present on "regular" messages
 	    * received via UNIX domain sockets from the same machine. However,
@@ -122,6 +121,18 @@ struct msg {
 	int msgFlags;		/* flags associated with this message */
 };
 
+
+/* message flags (msgFlags), not an enum for historical reasons
+ */
+#define NOFLAG		0x000	/* no flag is set (to be used when a flag must be specified and none is required) */
+#define INTERNAL_MSG	0x001	/* msg generated by logmsgInternal() --> special handling */
+/* 0x002 not used because it was previously a known value - rgerhards, 2008-10-09 */
+#define IGNDATE		0x004	/* ignore, if given, date in message and use date of reception as msg date */
+#define MARK		0x008	/* this message is a mark */
+#define NEEDS_PARSING	0x010	/* raw message, must be parsed before processing can be done */
+#define PARSE_HOSTNAME	0x020	/* parse the hostname during message parsing */
+
+
 /* function prototypes
  */
 PROTOTYPEObjClassInit(msg);
diff --git a/runtime/parser.c b/runtime/parser.c
index 8c4272a0..fbdeebeb 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -41,7 +41,6 @@
 /* some defines */
 #define DEFUPRI		(LOG_USER|LOG_NOTICE)
 
-#warning "msg object must be updated with new property for persisting the queue!"
 /* definitions for objects we access */
 DEFobjStaticHelpers
 DEFobjCurrIf(glbl)
@@ -306,7 +305,7 @@ rsRetVal parseMsg(msg_t *pMsg)
 	}
 
 	/* finalize message object */
-	pMsg->bIsParsed = 1; /* this message is now parsed */
+	pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */
 	MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */
 
 finalize_it:
diff --git a/runtime/queue.c b/runtime/queue.c
index b0043ef5..42b8137d 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -49,6 +49,7 @@
 #include "obj.h"
 #include "wtp.h"
 #include "wti.h"
+#include "atomic.h"
 
 /* static data */
 DEFobjStaticHelpers
@@ -996,7 +997,7 @@ queueAdd(queue_t *pThis, void *pUsr)
 	CHKiRet(pThis->qAdd(pThis, pUsr));
 
 	if(pThis->qType != QUEUETYPE_DIRECT) {
-		++pThis->iQueueSize;
+		ATOMIC_INC(pThis->iQueueSize);
 		dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize);
 	}
 
@@ -1025,7 +1026,7 @@ queueDel(queue_t *pThis, void *pUsr)
 		iRet = queueGetUngottenObj(pThis, (obj_t**) pUsr);
 	} else {
 		iRet = pThis->qDel(pThis, pUsr);
-		--pThis->iQueueSize;
+		ATOMIC_DEC(pThis->iQueueSize);
 	}
 
 	dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n",
diff --git a/tcps_sess.c b/tcps_sess.c
index 13644f45..e8bef5b1 100644
--- a/tcps_sess.c
+++ b/tcps_sess.c
@@ -42,6 +42,7 @@
 #include "obj.h"
 #include "errmsg.h"
 #include "netstrm.h"
+#include "msg.h"
 
 
 /* static data */
@@ -229,8 +230,8 @@ PrepareClose(tcps_sess_t *pThis)
 		 * this case.
 		 */
 		dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n");
-		parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME,
-				      NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
+		parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
+				      PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 		pThis->bAtStrtOfFram = 1;
 	}
 
@@ -313,7 +314,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 			/* emergency, we now need to flush, no matter if we are at end of message or not... */
 			dbgprintf("error: message received is larger than max msg size, we split it\n");
 			parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
+					      PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 			pThis->iMsg = 0;
 			/* we might think if it is better to ignore the rest of the
 			 * message than to treat it as a new one. Maybe this is a good
@@ -324,7 +325,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 
 		if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */
 			parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
+					      PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 			pThis->iMsg = 0;
 			pThis->inputState = eAtStrtFram;
 		} else {
@@ -343,7 +344,7 @@ processDataRcvd(tcps_sess_t *pThis, char c)
 			if(pThis->iOctetsRemain < 1) {
 				/* we have end of frame! */
 				parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg,
-					      MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
+					      PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */
 				pThis->iMsg = 0;
 				pThis->inputState = eAtStrtFram;
 			}
diff --git a/tools/syslogd.c b/tools/syslogd.c
index a45942fa..13696955 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -588,8 +588,11 @@ void untty(void)
  * a timestamp that is to be used as timegenerated instead of the current system time.
  * This is meant to facilitate performance optimization. Some inputs support such modes.
  * If stTime is NULL, the current system time is used.
+ *
+ * rgerhards, 2008-10-09:
+ * interface change: bParseHostname removed, now in flags
  */
-static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType,
+static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int flags, flowControl_t flowCtlType,
 	uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime)
 {
 	DEFiRet;
@@ -603,13 +606,11 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b
 	} else {
 		CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime));
 	}
-	pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */
 	if(pszInputName != NULL)
 		MsgSetInputName(pMsg, (char*) pszInputName);
 	MsgSetFlowControlType(pMsg, flowCtlType);
 	MsgSetRawMsg(pMsg, (char*)msg);
 	
-	pMsg->bParseHOSTNAME  = bParseHost;
 	/* test for special codes */
 	pri = DEFUPRI;
 	p = msg;
@@ -634,7 +635,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b
 	 * the message was received from (that, for obvious reasons,
 	 * being the local host).  rgerhards 2004-11-16
 	 */
-	if(bParseHost == 0)
+	if((pMsg->msgFlags & PARSE_HOSTNAME) == 0)
 		MsgSetHOSTNAME(pMsg, (char*)hname);
 	MsgSetRcvFrom(pMsg, (char*)hname);
 	CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP));
@@ -702,9 +703,12 @@ finalize_it:
  * a timestamp that is to be used as timegenerated instead of the current system time.
  * This is meant to facilitate performance optimization. Some inputs support such modes.
  * If stTime is NULL, the current system time is used.
+ *
+ * rgerhards, 2008-10-09:
+ * interface change: bParseHostname removed, now in flags
  */
 rsRetVal
-parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType,
+parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType,
 	uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime)
 {
 	DEFiRet;
@@ -816,7 +820,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 			 */
 			if(iMsg == iMaxLine) {
 				*(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */
-				printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime);
+				printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime);
 			} else {
 				/* This case in theory never can happen. If it happens, we have
 				 * a logic error. I am checking for it, because if I would not,
@@ -868,7 +872,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa
 	*(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */
 
 	/* typically, we should end up here! */
-	printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime);
+	printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime);
 
 finalize_it:
 	if(tmpline != NULL)
@@ -1192,8 +1196,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr)
 
 	assert(pMsg != NULL);
 
-RUNLOG_VAR("%d", pMsg->bIsParsed);
-	if(pMsg->bIsParsed == 0) {
+	if((pMsg->msgFlags & NEEDS_PARSING) != 0) {
 		parseMsg(pMsg);
 	}
 	processMsg(pMsg);
-- 
cgit 


From a27e249e445deecb1d9ef57fbbb203d71bf061dd Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 21 Oct 2008 10:55:33 +0200
Subject: bugfix: (potentially big) memory leak on HUP

This occured if queues could not be drained before timeout.
Thanks to David Lang for pointing this out.
---
 ChangeLog       |  2 ++
 runtime/queue.c | 45 ++++++++++++++++++++++++++++++++-------------
 2 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 2a3c9acf..9a02d675 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -25,6 +25,8 @@ version before switching to this one.
   and consistency
 - bugfix: subsecond time properties generated by imfile, imklog and
   internal messages could be slightly inconsistent
+- bugfix: (potentially big) memory leak on HUP if queues could not be
+  drained before timeout - thanks to David Lang for pointing this out
 ---------------------------------------------------------------------------
 Version 3.21.5  [DEVEL] (rgerhards), 2008-09-30
 - performance optimization: unnecessary time() calls during message
diff --git a/runtime/queue.c b/runtime/queue.c
index 42b8137d..7fa2df91 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -88,6 +88,30 @@ ENDfunc
 	return pThis->iQueueSize + pThis->iUngottenObjs;
 }
 
+
+/* This function drains the queue in cases where this needs to be done. The most probable
+ * reason is a HUP which needs to discard data (because the queue is configured to be lossy).
+ * During a shutdown, this is typically not needed, as the OS frees up ressources and does
+ * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21
+ * This function returns void, as it makes no sense to communicate an error back, even if
+ * it happens.
+ */
+static inline void queueDrain(queue_t *pThis)
+{
+	void *pUsr;
+	
+	ASSERT(pThis != NULL);
+
+	/* iQueueSize is not decremented by qDel(), so we need to do it ourselves */
+	while(pThis->iQueueSize-- > 0) {
+		pThis->qDel(pThis, &pUsr);
+		if(pUsr != NULL) {
+			objDestruct(pUsr);
+		}
+	}
+}
+
+
 /* --------------- code for disk-assisted (DA) queue modes -------------------- */
 
 
@@ -196,14 +220,6 @@ queueTurnOffDAMode(queue_t *pThis)
 		}
 	}
 
-	/* TODO: we have a *really biiiiig* memory leak here: if the queue could not be persisted, all of
-	 * its data elements are still in memory. That doesn't really matter if we are terminated, but on
-	 * HUP this memory leaks. We MUST add a loop of destructor calls here. However, this takes time
-	 * (possibly a lot), so it is probably best to have a config variable for that.
-	 * Something for 3.11.1!
-	 * rgerhards, 2008-01-30
-	 */
-
 	RETiRet;
 }
 
@@ -461,12 +477,15 @@ static rsRetVal qDestructFixedArray(queue_t *pThis)
 	
 	ASSERT(pThis != NULL);
 
+	queueDrain(pThis); /* discard any remaining queue entries */
+
 	if(pThis->tVars.farray.pBuf != NULL)
 		free(pThis->tVars.farray.pBuf);
 
 	RETiRet;
 }
 
+
 static rsRetVal qAddFixedArray(queue_t *pThis, void* in)
 {
 	DEFiRet;
@@ -570,11 +589,11 @@ static rsRetVal qConstructLinkedList(queue_t *pThis)
 static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis)
 {
 	DEFiRet;
-	
-	/* with the linked list type, there is nothing to do here. The
-	 * reason is that the Destructor is only called after all entries
-	 * have bene taken off the queue. In this case, there is nothing
-	 * dynamic left with the linked list.
+
+	queueDrain(pThis); /* discard any remaining queue entries */
+
+	/* with the linked list type, there is nothing left to do here. The
+	 * reason is that there are no dynamic elements for the list itself.
 	 */
 
 	RETiRet;
-- 
cgit 


From cf38fc81759b01af5125b1a05e0d6fe8e2e1bc21 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 22 Oct 2008 13:54:40 +0200
Subject: added a setting "$OptimizeForUniprocessor"

...to enable users to turn off pthread_yield calls which are
counter-productive on multiprocessor machines (but have been
shown to be useful on uniprocessors)
---
 ChangeLog             |  3 +++
 doc/rsyslog_conf.html |  6 ++++--
 runtime/glbl.c        |  5 +++++
 runtime/glbl.h        |  1 +
 runtime/queue.c       |  5 ++++-
 runtime/queue.h       |  1 +
 runtime/wti.c         | 15 ++++++++++++++-
 runtime/wti.h         |  1 +
 runtime/wtp.c         | 16 +++++++++++++++-
 runtime/wtp.h         |  1 +
 10 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 41aef49f..6856c3e7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -21,6 +21,9 @@ version before switching to this one.
   receive loop (aka receiving messsages at a high rate)
 - doc bugfix: queue doc had wrong parameter name for setting controlling
   worker thread shutdown period
+- added a setting "$OptimizeForUniprocessor" to enable users to turn off
+  pthread_yield calls which are counter-productive on multiprocessor 
+  machines (but have been shown to be useful on uniprocessors)
 ---------------------------------------------------------------------------
 Version 3.21.6  [DEVEL] (rgerhards), 2008-10-22
 - consolidated time calls during msg object creation, improves performance
diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html
index 2b773476..be543833 100644
--- a/doc/rsyslog_conf.html
+++ b/doc/rsyslog_conf.html
@@ -236,8 +236,10 @@ supported in order to be compliant to the upcoming new syslog RFC series.
 <li><a href="rsconf1_modload.html">$ModLoad</a></li>
 <li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li>
 <li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li>
-<li>$WorkDirectory &lt;name&gt; (directory for spool
-and other work files)</li>
+<li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better
+performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The
+default may change as uniprocessor systems become less common.</li>
+<li>$WorkDirectory &lt;name&gt; (directory for spool and other work files)</li>
 <li>$UDPServerAddress &lt;IP&gt; (imudp) -- local IP
 address (or name) the UDP listens should bind to</li>
 <li>$UDPServerRun &lt;port&gt; (imudp) -- former
diff --git a/runtime/glbl.c b/runtime/glbl.c
index 1114fcd3..c752288f 100644
--- a/runtime/glbl.c
+++ b/runtime/glbl.c
@@ -51,6 +51,7 @@ DEFobjStaticHelpers
  * class...
  */
 static uchar *pszWorkDir = NULL;
+static int bOptimizeUniProc = 1;	/* enable uniprocessor optimizations */
 static int iMaxLine = 2048;		/* maximum length of a syslog message */
 static int iDefPFFamily = PF_UNSPEC;     /* protocol family (IPv4, IPv6 or both) */
 static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */
@@ -85,6 +86,7 @@ static dataType Get##nameFunc(void) \
 	return(nameVar); \
 }
 
+SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int)
 SIMP_PROP(MaxLine, iMaxLine, int)
 SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */
 SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int)
@@ -173,6 +175,7 @@ CODESTARTobjQueryInterface(glbl)
 	pIf->Get##name = Get##name; \
 	pIf->Set##name = Set##name;
 	SIMP_PROP(MaxLine);
+	SIMP_PROP(OptimizeUniProc);
 	SIMP_PROP(DefPFFamily);
 	SIMP_PROP(DropMalPTRMsgs);
 	SIMP_PROP(Option_DisallowWarning);
@@ -216,6 +219,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 		pszWorkDir = NULL;
 	}
 	bDropMalPTRMsgs = 0;
+	bOptimizeUniProc = 1;
 	return RS_RET_OK;
 }
 
@@ -235,6 +239,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */
 	CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL));
 ENDObjClassInit(glbl)
 
diff --git a/runtime/glbl.h b/runtime/glbl.h
index 0c83bdd5..0b871b3c 100644
--- a/runtime/glbl.h
+++ b/runtime/glbl.h
@@ -41,6 +41,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */
 	dataType (*Get##name)(void); \
 	rsRetVal (*Set##name)(dataType);
 	SIMP_PROP(MaxLine, int)
+	SIMP_PROP(OptimizeUniProc, int)
 	SIMP_PROP(DefPFFamily, int)
 	SIMP_PROP(DropMalPTRMsgs, int)
 	SIMP_PROP(Option_DisallowWarning, int)
diff --git a/runtime/queue.c b/runtime/queue.c
index 7fa2df91..a3e29a96 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1285,6 +1285,7 @@ rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads,
 
 	/* we have an object, so let's fill the properties */
 	objConstructSetObjInfo(pThis);
+	pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc();
 	if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL)
 		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
 
@@ -2208,8 +2209,10 @@ finalize_it:
 		/* the following pthread_yield is experimental, but brought us performance
 		 * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216
 		 * rgerhards, 2008-10-09
+		 * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22
 		 */
-		pthread_yield();
+		if(pThis->bOptimizeUniProc)
+			pthread_yield();
 	}
 
 	RETiRet;
diff --git a/runtime/queue.h b/runtime/queue.h
index 9e75b31b..a2dd594f 100644
--- a/runtime/queue.h
+++ b/runtime/queue.h
@@ -58,6 +58,7 @@ typedef struct qWrkThrd_s {
 typedef struct queue_s {
 	BEGINobjInstance;
 	queueType_t	qType;
+	int	bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */
 	int	bEnqOnly;	/* does queue run in enqueue-only mode (1) or not (0)? */
 	int	bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */
 	int	bQueueStarted;	/* has queueStart() been called on this queue? 1-yes, 0-no */
diff --git a/runtime/wti.c b/runtime/wti.c
index 4abca090..e8a77474 100644
--- a/runtime/wti.c
+++ b/runtime/wti.c
@@ -45,9 +45,11 @@
 #include "wtp.h"
 #include "wti.h"
 #include "obj.h"
+#include "glbl.h"
 
 /* static data */
 DEFobjStaticHelpers
+DEFobjCurrIf(glbl)
 
 /* forward-definitions */
 
@@ -202,6 +204,7 @@ ENDobjDestruct(wti)
 /* Standard-Constructor for the wti object
  */
 BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */
+	pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc();
 	pthread_cond_init(&pThis->condExitDone, NULL);
 	pthread_mutex_init(&pThis->mut, NULL);
 ENDobjConstruct(wti)
@@ -367,7 +370,8 @@ wtiWorker(wti_t *pThis)
 		wtpProcessThrdChanges(pWtp);
 		pthread_testcancel(); /* see big comment in function header */
 #		if !defined(__hpux) /* pthread_yield is missing there! */
-		pthread_yield(); /* see big comment in function header */
+		if(pThis->bOptimizeUniProc)
+			pthread_yield(); /* see big comment in function header */
 #		endif
 
 		/* if we have a rate-limiter set for this worker pool, let's call it. Please
@@ -466,6 +470,14 @@ finalize_it:
 /* dummy */
 rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; }
 
+/* exit our class
+ */
+BEGINObjClassExit(wti, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */
+CODESTARTObjClassExit(nsdsel_gtls)
+	/* release objects we no longer need */
+	objRelease(glbl, CORE_COMPONENT);
+ENDObjClassExit(wti)
+
 
 /* Initialize the wti class. Must be called as the very first method
  * before anything else is called inside this class.
@@ -473,6 +485,7 @@ rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; }
  */
 BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most important for persisting) */
 	/* request objects we use */
+	CHKiRet(objUse(glbl, CORE_COMPONENT));
 ENDObjClassInit(wti)
 
 /*
diff --git a/runtime/wti.h b/runtime/wti.h
index 0cd6744e..6b60b833 100644
--- a/runtime/wti.h
+++ b/runtime/wti.h
@@ -31,6 +31,7 @@
 /* the worker thread instance class */
 typedef struct wti_s {
 	BEGINobjInstance;
+	int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */
 	pthread_t thrdID;  /* thread ID */
 	qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */
 	obj_t *pUsrp;		/* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */
diff --git a/runtime/wtp.c b/runtime/wtp.c
index 45034fa7..06173e04 100644
--- a/runtime/wtp.c
+++ b/runtime/wtp.c
@@ -46,9 +46,11 @@
 #include "wtp.h"
 #include "wti.h"
 #include "obj.h"
+#include "glbl.h"
 
 /* static data */
 DEFobjStaticHelpers
+DEFobjCurrIf(glbl)
 
 /* forward-definitions */
 
@@ -75,6 +77,7 @@ static rsRetVal NotImplementedDummy() { return RS_RET_OK; }
 /* Standard-Constructor for the wtp object
  */
 BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */
+	pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc();
 	pthread_mutex_init(&pThis->mut, NULL);
 	pthread_cond_init(&pThis->condThrdTrm, NULL);
 	/* set all function pointers to "not implemented" dummy so that we can safely call them */
@@ -484,7 +487,8 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex)
  	 * hold the queue's mutex, but at least it has a chance to start on a single-CPU system.
  	 */
 #	if !defined(__hpux) /* pthread_yield is missing there! */
-	pthread_yield();
+	if(pThis->bOptimizeUniProc)
+		pthread_yield();
 #	endif
 
 	/* indicate we just started a worker and would like to see it running */
@@ -617,12 +621,22 @@ finalize_it:
 /* dummy */
 rsRetVal wtpQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; }
 
+/* exit our class
+ */
+BEGINObjClassExit(wtp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */
+CODESTARTObjClassExit(nsdsel_gtls)
+	/* release objects we no longer need */
+	objRelease(glbl, CORE_COMPONENT);
+ENDObjClassExit(wtp)
+
+
 /* Initialize the stream class. Must be called as the very first method
  * before anything else is called inside this class.
  * rgerhards, 2008-01-09
  */
 BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE)
 	/* request objects we use */
+	CHKiRet(objUse(glbl, CORE_COMPONENT));
 ENDObjClassInit(wtp)
 
 /*
diff --git a/runtime/wtp.h b/runtime/wtp.h
index 13ebe536..88d88afd 100644
--- a/runtime/wtp.h
+++ b/runtime/wtp.h
@@ -52,6 +52,7 @@ typedef enum {
 /* the worker thread pool (wtp) object */
 typedef struct wtp_s {
 	BEGINobjInstance;
+	int	bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */
 	wtpState_t wtpState;
 	int 	iNumWorkerThreads;/* number of worker threads to use */
 	int 	iCurNumWrkThrd;/* current number of active worker threads */
-- 
cgit 


From 6334d335d89ae5df344f833c5095e9dea2abf6fb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 23 Oct 2008 14:46:47 +0200
Subject: added configuration directive "HUPisRestart"

...which enables to configure HUP to be either a full restart
or "just" a leightweight way to close open files
---
 ChangeLog                 |  12 ++++--
 action.c                  |  33 +++++++++++++++
 action.h                  |   1 +
 configure.ac              |   2 +-
 doc/manual.html           |   2 +-
 doc/rsyslog_conf.html     |  20 +++++----
 runtime/glbl.c            |   5 +++
 runtime/glbl.h            |   1 +
 runtime/module-template.h |  31 +++++++++++++-
 runtime/modules.c         |   5 +++
 runtime/modules.h         |   1 +
 tools/omfile.c            | 102 +++++++++++++++++++++++++++++-----------------
 tools/syslogd.c           |  59 ++++++++++++++++++++++-----
 13 files changed, 212 insertions(+), 62 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6856c3e7..a11ce27d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,5 @@
 ---------------------------------------------------------------------------
-Version 3.23.1  [DEVEL] (rgerhards), 2008-10-??
+Version 4.1.0  [DEVEL] (rgerhards), 2008-10-??
 
 ********************************* WARNING *********************************
 This version has a slightly different on-disk format for message entries.
@@ -9,6 +9,13 @@ the output driver. It is recommended to drain queues with the previous
 version before switching to this one.
 ********************************* WARNING *********************************
 
+- greatly enhanced performance when compared to v3.
+- added configuration directive "HUPisRestart" which enables to configure
+  HUP to be either a full restart or "just" a leightweight way to
+  close open files.
+- added a setting "$OptimizeForUniprocessor" to enable users to turn off
+  pthread_yield calls which are counter-productive on multiprocessor 
+  machines (but have been shown to be useful on uniprocessors)
 - reordered imudp processing. Message parsing is now done as part of main
   message queue worker processing (was part of the input thread)
   This should also improve performance, as potentially more work is
@@ -21,9 +28,6 @@ version before switching to this one.
   receive loop (aka receiving messsages at a high rate)
 - doc bugfix: queue doc had wrong parameter name for setting controlling
   worker thread shutdown period
-- added a setting "$OptimizeForUniprocessor" to enable users to turn off
-  pthread_yield calls which are counter-productive on multiprocessor 
-  machines (but have been shown to be useful on uniprocessors)
 ---------------------------------------------------------------------------
 Version 3.21.6  [DEVEL] (rgerhards), 2008-10-22
 - consolidated time calls during msg object creation, improves performance
diff --git a/action.c b/action.c
index ec8732bc..dee46f16 100644
--- a/action.c
+++ b/action.c
@@ -498,6 +498,39 @@ finalize_it:
 }
 #pragma GCC diagnostic warning "-Wempty-body"
 
+
+/* call the HUP handler for a given action, if such a handler is defined. The
+ * action mutex is locked, because the HUP handler most probably needs to modify
+ * some internal state information.
+ * rgerhards, 2008-10-22
+ */
+#pragma GCC diagnostic ignored "-Wempty-body"
+rsRetVal
+actionCallHUPHdlr(action_t *pAction)
+{
+	DEFiRet;
+	int iCancelStateSave;
+
+	ASSERT(pAction != NULL);
+	DBGPRINTF("Action %p checks HUP hdlr: %p\n", pAction, pAction->pMod->doHUP);
+
+	if(pAction->pMod->doHUP == NULL) {
+		FINALIZE;	/* no HUP handler, so we are done ;) */
+	}
+
+	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
+	d_pthread_mutex_lock(&pAction->mutActExec);
+	pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec);
+	pthread_setcancelstate(iCancelStateSave, NULL);
+	CHKiRet(pAction->pMod->doHUP(pAction->pModData));
+	pthread_cleanup_pop(1); /* unlock mutex */
+
+finalize_it:
+	RETiRet;
+}
+#pragma GCC diagnostic warning "-Wempty-body"
+
+
 /* set the action message queue mode
  * TODO: probably move this into queue object, merge with MainMsgQueue!
  * rgerhards, 2008-01-28
diff --git a/action.h b/action.h
index d26d15b6..8d9d5102 100644
--- a/action.h
+++ b/action.h
@@ -85,6 +85,7 @@ rsRetVal actionSetGlobalResumeInterval(int iNewVal);
 rsRetVal actionDoAction(action_t *pAction);
 rsRetVal actionCallAction(action_t *pAction, msg_t *pMsg);
 rsRetVal actionWriteToAction(action_t *pAction);
+rsRetVal actionCallHUPHdlr(action_t *pAction);
 rsRetVal actionClassInit(void);
 rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended);
 
diff --git a/configure.ac b/configure.ac
index 2f17e575..981ef193 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[3.23.1],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[4.1.0],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 AC_CONFIG_SRCDIR([ChangeLog])
 AC_CONFIG_HEADERS([config.h])
diff --git a/doc/manual.html b/doc/manual.html
index 884c43c0..f1b7d657 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -16,7 +16,7 @@ relay chains while at the same time being very easy to setup for the
 novice user. And as we know what enterprise users really need, there is
 also <a href="professional_support.html">professional
 rsyslog support</a> available directly from the source!</p>
-<p><b>This documentation is for version 3.21.6 (devel branch) of rsyslog.</b>
+<p><b>This documentation is for version 4.1.0 (devel branch) of rsyslog.</b>
 Visit the <i> <a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> to obtain current
 version information and project status.
 </p><p><b>If you like rsyslog, you might
diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html
index be543833..cd3db405 100644
--- a/doc/rsyslog_conf.html
+++ b/doc/rsyslog_conf.html
@@ -59,17 +59,19 @@ remember, that a modules configuration directive (and functionality) is
 only available if it has been loaded (using $ModLoad).</p>
 <h2>Lines</h2>
 Lines can be continued by specifying a backslash ("\") as the last
-character of the line.<br>
+character of the line. There is a hard-coded maximum line length of 4K.
+If you need lines larger than that, you need to change compile-time
+settings inside rsyslog and recompile.
 <h2>Global Directives</h2>
 <p>All global directives need to be specified on a line by their
 own and must start with a dollar-sign. Here is a list in alphabetical
 order. Follow links for a description.</p>
+<p>Please note that not all directives here are actually global. Some affect
+only the next action. This documentation will be changed soon.
 <p>Not all directives have an in-depth description right now.
 Default values for them are in bold. A more in-depth description will
-appear as implementation progresses. Directives may change during that
-process, we will NOT try hard to maintain backwards compatibility
-(after all, v3 is still very early in development and quite
-unstable...). So you have been warned ;)</p>
+appear as implementation progresses.
+</p>
 <p><b>Be sure to read information about <a href="queues.html">queues in rsyslog</a></b> -
 many parameter settings modify queue parameters. If in doubt, use the
 default, it is usually well-chosen and applicable in most cases.</p>
@@ -175,12 +177,16 @@ default 60000 (1 minute)]</li>
 <li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li>
 <li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li>
 <li><a href="rsconf1_gssmode.html">$GssMode</a></li>
+<li>$HUPisRestart [<b>on</b>/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending
+on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is
+not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for
+compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually
+the only one that is needed e.g. for log rotation. <b>It is recommended to set the setting to "off".</b></li>
 <li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval &lt;number&gt;</li>
 <li>$MainMsgQueueDequeueSlowdown &lt;number&gt; [number
 is timeout in <i> micro</i>seconds (1000000us is 1sec!),
 default 0 (no delay). Simple rate-limiting!]</li>
-<li>$MainMsgQueueDiscardMark &lt;number&gt; [default
-9750]</li>
+<li>$MainMsgQueueDiscardMark &lt;number&gt; [default 9750]</li>
 <li>$MainMsgQueueDiscardSeverity &lt;severity&gt;
 [either a textual or numerical severity! default 4 (warning)]</li>
 <li>$MainMsgQueueFileName &lt;name&gt;</li>
diff --git a/runtime/glbl.c b/runtime/glbl.c
index c752288f..2a6bfb11 100644
--- a/runtime/glbl.c
+++ b/runtime/glbl.c
@@ -52,6 +52,7 @@ DEFobjStaticHelpers
  */
 static uchar *pszWorkDir = NULL;
 static int bOptimizeUniProc = 1;	/* enable uniprocessor optimizations */
+static int bHUPisRestart = 1;		/* should SIGHUP cause a full system restart? */
 static int iMaxLine = 2048;		/* maximum length of a syslog message */
 static int iDefPFFamily = PF_UNSPEC;     /* protocol family (IPv4, IPv6 or both) */
 static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */
@@ -87,6 +88,7 @@ static dataType Get##nameFunc(void) \
 }
 
 SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int)
+SIMP_PROP(HUPisRestart, bHUPisRestart, int)
 SIMP_PROP(MaxLine, iMaxLine, int)
 SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */
 SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int)
@@ -176,6 +178,7 @@ CODESTARTobjQueryInterface(glbl)
 	pIf->Set##name = Set##name;
 	SIMP_PROP(MaxLine);
 	SIMP_PROP(OptimizeUniProc);
+	SIMP_PROP(HUPisRestart);
 	SIMP_PROP(DefPFFamily);
 	SIMP_PROP(DropMalPTRMsgs);
 	SIMP_PROP(Option_DisallowWarning);
@@ -220,6 +223,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	}
 	bDropMalPTRMsgs = 0;
 	bOptimizeUniProc = 1;
+	bHUPisRestart = 1;
 	return RS_RET_OK;
 }
 
@@ -240,6 +244,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */
 	CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL));
 ENDObjClassInit(glbl)
 
diff --git a/runtime/glbl.h b/runtime/glbl.h
index 0b871b3c..44e41e3e 100644
--- a/runtime/glbl.h
+++ b/runtime/glbl.h
@@ -42,6 +42,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */
 	rsRetVal (*Set##name)(dataType);
 	SIMP_PROP(MaxLine, int)
 	SIMP_PROP(OptimizeUniProc, int)
+	SIMP_PROP(HUPisRestart, int)
 	SIMP_PROP(DefPFFamily, int)
 	SIMP_PROP(DropMalPTRMsgs, int)
 	SIMP_PROP(Option_DisallowWarning, int)
diff --git a/runtime/module-template.h b/runtime/module-template.h
index eb39b587..6f7d877c 100644
--- a/runtime/module-template.h
+++ b/runtime/module-template.h
@@ -481,6 +481,33 @@ static rsRetVal afterRun(void)\
 }
 
 
-/*
- * vi:set ai:
+/* doHUP()
+ * This function is optional. Currently, it is available to output plugins
+ * only, but may be made available to other types of plugins in the future.
+ * A plugin does not need to define this entry point. If if does, it gets
+ * called when a non-restart type of HUP is done. A plugin should register
+ * this function so that it can close files, connection or other ressources
+ * on HUP - if it can be assume the user wanted to do this as a part of HUP
+ * processing. Note that the name "HUP" has historical reasons, it stems back
+ * to the infamous SIGHUP which was sent to restart a syslogd. We still retain
+ * that legacy, but may move this to a different signal.
+ * rgerhards, 2008-10-22
+ */
+#define CODEqueryEtryPt_doHUP \
+	else if(!strcmp((char*) name, "doHUP")) {\
+		*pEtryPoint = doHUP;\
+	}
+#define BEGINdoHUP \
+static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\
+{\
+	DEFiRet;
+
+#define CODESTARTdoHUP 
+
+#define ENDdoHUP \
+	RETiRet;\
+}
+
+
+/* vim:set ai:
  */
diff --git a/runtime/modules.c b/runtime/modules.c
index d5730ede..169d234b 100644
--- a/runtime/modules.c
+++ b/runtime/modules.c
@@ -347,6 +347,7 @@ static rsRetVal
 doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr)
 {
 	DEFiRet;
+	rsRetVal localRet;
 	modInfo_t *pNew = NULL;
 	rsRetVal (*modGetType)(eModType_t *pType);
 
@@ -391,6 +392,10 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_
 			CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct));
 			CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature));
 			CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume));
+			/* try load optional interfaces */
+			localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP);
+			if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND)
+				ABORT_FINALIZE(localRet);
 			break;
 		case eMOD_LIB:
 			break;
diff --git a/runtime/modules.h b/runtime/modules.h
index 96016082..372529ee 100644
--- a/runtime/modules.h
+++ b/runtime/modules.h
@@ -91,6 +91,7 @@ typedef struct modInfo_s {
 	rsRetVal (*tryResume)(void*);/* called to see if module actin can be resumed now */
 	rsRetVal (*modExit)(void);		/* called before termination or module unload */
 	rsRetVal (*modGetID)(void **);		/* get its unique ID from module */
+	rsRetVal (*doHUP)(void *);		/* non-restart type HUP handler */
 	/* below: parse a configuration line - return if processed
 	 * or not. If not, must be parsed to next module.
 	 */
diff --git a/tools/omfile.c b/tools/omfile.c
index 8144386f..00a82933 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -174,7 +174,7 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal)
 	}
 
 	iDynaFileCacheSize = iNewVal;
-	dbgprintf("DynaFileCacheSize changed to %d.\n", iNewVal);
+	DBGPRINTF("DynaFileCacheSize changed to %d.\n", iNewVal);
 
 	RETiRet;
 }
@@ -244,7 +244,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR
 	 */
 	pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit;
 
-RUNLOG_VAR("%p", pszTplName);
 	iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts,
 				       (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName);
 
@@ -327,7 +326,7 @@ static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int b
 	if(pCache[iEntry] == NULL)
 		FINALIZE;
 
-	dbgprintf("Removed entry %d for file '%s' from dynaCache.\n", iEntry,
+	DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry,
 		pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName);
 	/* if the name is NULL, this is an improperly initilized entry which
 	 * needs to be discarded. In this case, neither the file is to be closed
@@ -349,9 +348,11 @@ finalize_it:
 }
 
 
-/* This function frees the dynamic file name cache.
+/* This function frees all dynamic file name cache entries and closes the
+ * relevant files. Part of Shutdown and HUP processing.
+ * rgerhards, 2008-10-23
  */
-static void dynaFileFreeCache(instanceData *pData)
+static inline void dynaFileFreeCacheEntries(instanceData *pData)
 {
 	register int i;
 	ASSERT(pData != NULL);
@@ -360,17 +361,36 @@ static void dynaFileFreeCache(instanceData *pData)
 	for(i = 0 ; i < pData->iCurrCacheSize ; ++i) {
 		dynaFileDelCacheEntry(pData->dynCache, i, 1);
 	}
+	ENDfunc;
+}
+
+
+/* This function frees the dynamic file name cache.
+ */
+static void dynaFileFreeCache(instanceData *pData)
+{
+	ASSERT(pData != NULL);
 
+	BEGINfunc;
+	dynaFileFreeCacheEntries(pData);
 	if(pData->dynCache != NULL)
 		d_free(pData->dynCache);
 	ENDfunc;
 }
 
 
-/* This is a shared code for both static and dynamic files.
+/* This is now shared code for all types of files. It simply prepares
+ * file access, which, among others, means the the file wil be opened
+ * and any directories in between will be created (based on config, of
+ * course). -- rgerhards, 2008-10-22
  */
 static void prepareFile(instanceData *pData, uchar *newFileName)
 {
+	if(pData->fileType == eTypePIPE) {
+		pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK);
+		FINALIZE; /* we are done in this case */
+	}
+
 	if(access((char*)newFileName, F_OK) == 0) {
 		/* file already exists */
 		pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY,
@@ -392,8 +412,7 @@ static void prepareFile(instanceData *pData, uchar *newFileName)
 					/* check and set uid/gid */
 					if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) {
 						/* we need to set owner/group */
-						if(fchown(pData->fd, pData->fileUID,
-						          pData->fileGID) != 0) {
+						if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) {
 							if(pData->bFailOnChown) {
 								int eSave = errno;
 								close(pData->fd);
@@ -409,6 +428,12 @@ static void prepareFile(instanceData *pData, uchar *newFileName)
 			}
 		}
 	}
+finalize_it:
+	if((pData->fd) != 0 && isatty(pData->fd)) {
+		DBGPRINTF("file %d is a tty file\n", pData->fd);
+		pData->fileType = eTypeTTY;
+		untty();
+	}
 }
 
 
@@ -482,7 +507,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 		/* we need to allocate memory for the cache structure */
 		pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry));
 		if(pCache[iFirstFree] == NULL) {
-			dbgprintf("prepareDynfile(): could not alloc mem, discarding this request\n");
+			DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n");
 			return -1;
 		}
 	}
@@ -496,10 +521,11 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 		 * message. Otherwise, we could run into a never-ending loop. The bad
 		 * news is that we also lose errors on startup messages, but so it is.
 		 */
-		if(iMsgOpts & INTERNAL_MSG)
-			dbgprintf("Could not open dynaFile, discarding message\n");
-		else
+		if(iMsgOpts & INTERNAL_MSG) {
+			DBGPRINTF("Could not open dynaFile, discarding message\n");
+		} else {
 			errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName);
+		}
 		dynaFileDelCacheEntry(pCache, iFirstFree, 1);
 		pData->iCurrElt = -1;
 		return -1;
@@ -509,8 +535,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 	pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */
 	pCache[iFirstFree]->lastUsed = time(NULL);
 	pData->iCurrElt = iFirstFree;
-	dbgprintf("Added new entry %d for file cache, file '%s'.\n",
-		iFirstFree, newFileName);
+	DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName);
 
 	return 0;
 }
@@ -533,6 +558,8 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa
 	if(pData->bDynamicName) {
 		if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0)
 			ABORT_FINALIZE(RS_RET_ERR);
+	} else if(pData->fd == -1) {
+		prepareFile(pData, pData->f_fname);
 	}
 
 	/* create the message based on format specified */
@@ -584,11 +611,10 @@ again:
 			ABORT_FINALIZE(RS_RET_OK);
 
 		(void) close(pData->fd);
-		/*
-		 * Check for EBADF on TTY's due to vhangup()
+		/* Check for EBADF on TTY's due to vhangup()
 		 * Linux uses EIO instead (mrn 12 May 96)
 		 */
-		if ((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE)
+		if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE)
 #ifdef linux
 			&& e == EIO) {
 #else
@@ -637,13 +663,8 @@ ENDtryResume
 
 BEGINdoAction
 CODESTARTdoAction
-	dbgprintf(" (%s)\n", pData->f_fname);
-	/* pData->fd == -1 is an indicator that the we couldn't
-	 * open the file at startup. For dynaFiles, this is ok,
-	 * all others are doomed.
-	 */
-	if(pData->bDynamicName || (pData->fd != -1))
-		iRet = writeFile(ppString, iMsgOpts, pData);
+	DBGPRINTF(" (%s)\n", pData->f_fname);
+	iRet = writeFile(ppString, iMsgOpts, pData);
 ENDdoAction
 
 
@@ -726,7 +747,7 @@ CODESTARTparseSelectorAct
 		if((pData->dynCache = (dynaFileCacheEntry**)
 		    calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) {
 			iRet = RS_RET_OUT_OF_MEMORY;
-			dbgprintf("Could not allocate memory for dynaFileCache - selector disabled.\n");
+			DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n");
 		}
 		break;
 
@@ -760,23 +781,15 @@ CODESTARTparseSelectorAct
 		pData->dirUID = dirUID;
 		pData->dirGID = dirGID;
 
-		if(pData->fileType == eTypePIPE) {
-			pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK);
-	        } else {
-			prepareFile(pData, pData->f_fname);
-		}
+		prepareFile(pData, pData->f_fname);
 		        
-	  	if ( pData->fd < 0 ){
+	  	if(pData->fd < 0 ) {
 			pData->fd = -1;
-			dbgprintf("Error opening log file: %s\n", pData->f_fname);
+			DBGPRINTF("Error opening log file: %s\n", pData->f_fname);
 			errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname);
 			break;
 		}
-		if (isatty(pData->fd)) {
-			pData->fileType = eTypeTTY;
-			untty();
-		}
-		if (strcmp((char*) p, _PATH_CONSOLE) == 0)
+		if(strcmp((char*) p, _PATH_CONSOLE) == 0)
 			pData->fileType = eTypeCONSOLE;
 		break;
 	default:
@@ -811,6 +824,20 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 }
 
 
+BEGINdoHUP
+CODESTARTdoHUP
+	if(pData->bDynamicName) {
+		dynaFileFreeCacheEntries(pData);
+		pData->iCurrElt = -1; /* invalidate current element */
+	} else {
+		if(pData->fd != -1) {
+			close(pData->fd);
+			pData->fd = -1;
+		}
+	}
+ENDdoHUP
+
+
 BEGINmodExit
 CODESTARTmodExit
 	if(pszTplName != NULL)
@@ -821,6 +848,7 @@ ENDmodExit
 BEGINqueryEtryPt
 CODESTARTqueryEtryPt
 CODEqueryEtryPt_STD_OMOD_QUERIES
+CODEqueryEtryPt_doHUP
 ENDqueryEtryPt
 
 
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 13696955..7145779d 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -220,7 +220,7 @@ static char	*PidFile = _PATH_LOGPID; /* read-only after startup */
 
 static pid_t myPid;	/* our pid for use in self-generated messages, e.g. on startup */
 /* mypid is read-only after the initial fork() */
-static int restart = 0; /* do restart (config read) - multithread safe */
+static int bHadHUP = 0; /* did we have a HUP? */
 
 static int bParseHOSTNAMEandTAG = 1; /* global config var: should the hostname and tag be
                                       * parsed inside message - rgerhards, 2006-03-13 */
@@ -2543,20 +2543,18 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz
  * The following function is resposible for handling a SIGHUP signal.  Since
  * we are now doing mallocs/free as part of init we had better not being
  * doing this during a signal handler.  Instead this function simply sets
- * a flag variable which will tell the main loop to go through a restart.
+ * a flag variable which will tells the main loop to do "the right thing".
  */
 void sighup_handler()
 {
 	struct sigaction sigAct;
 	
-	restart = 1;
+	bHadHUP = 1;
 
 	memset(&sigAct, 0, sizeof (sigAct));
 	sigemptyset(&sigAct.sa_mask);
 	sigAct.sa_handler = sighup_handler;
 	sigaction(SIGHUP, &sigAct, NULL);
-
-	return;
 }
 
 
@@ -2578,6 +2576,49 @@ static void processImInternal(void)
 }
 
 
+/* helper to doHUP(), this "HUPs" each action. The necessary locking
+ * is done inside the action class and nothing we need to take care of.
+ * rgerhards, 2008-10-22
+ */
+DEFFUNC_llExecFunc(doHUPActions)
+{
+	BEGINfunc
+	actionCallHUPHdlr((action_t*) pData);
+	ENDfunc
+	return RS_RET_OK; /* we ignore errors, we can not do anything either way */
+}
+
+
+/* This function processes a HUP after one has been detected. Note that this
+ * is *NOT* the sighup handler. The signal is recorded by the handler, that record
+ * detected inside the mainloop and then this function is called to do the
+ * real work. -- rgerhards, 2008-10-22
+ */
+static inline void
+doHUP(void)
+{
+	selector_t *f;
+	char buf[512];
+
+	snprintf(buf, sizeof(buf) / sizeof(char),
+		 " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION
+		 "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type '%s'.",
+		 (int) myPid, glbl.GetHUPisRestart() ? "restart" : "lightweight");
+		errno = 0;
+	logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0);
+
+	if(glbl.GetHUPisRestart()) {
+		DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n");
+		init(); /* main queue is stopped as part of init() */
+	} else {
+		DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n");
+		for(f = Files; f != NULL ; f = f->f_next) {
+			llExecFunc(&f->llActList, doHUPActions, NULL);
+		}
+	}
+}
+
+
 /* This is the main processing loop. It is called after successful initialization.
  * When it returns, the syslogd terminates.
  * Its sole function is to provide some housekeeping things. The real work is done
@@ -2634,11 +2675,9 @@ mainloop(void)
 		if(bReduceRepeatMsgs == 1)
 			doFlushRptdMsgs();
 
-		if(restart) {
-			dbgprintf("\nReceived SIGHUP, reloading rsyslogd.\n");
-			/* main queue is stopped as part of init() */
-			init();
-			restart = 0;
+		if(bHadHUP) {
+			doHUP();
+			bHadHUP = 0;
 			continue;
 		}
 	}
-- 
cgit 


From 4f5ab3342c262309ffaed769a5aa9584b7c0e6a3 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 28 Oct 2008 14:42:06 +0100
Subject: updated support options

---
 doc/professional_support.html | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/doc/professional_support.html b/doc/professional_support.html
index 2cb6c1e1..7724ede8 100644
--- a/doc/professional_support.html
+++ b/doc/professional_support.html
@@ -49,6 +49,30 @@ configuration file, but will place easy to read comments in the places
 where you need to put them in. The agreement is governed under German
 law. You may also purchase this service if you would like to have your
 own configuration file reviewed, e.g. for auditing purposes.</p>
+<h3>Local Installation Support</h3>
+<p>If you intend to install rsyslog on your system but would like
+to do so with minimal effort and according to your specification, you
+can ask us to perform the installation for you. You get a perfect
+installation, exactly like you needed, but without a need to
+touch the system. This is a perfect choice for the busy administrator!
+<p>In order to perform this work, we just need ssh access to your
+system and the proper permissions.
+<p>We charge a low one-time fee for this service. For details, please
+contact <a href="mailto:info@adiscon.com">info@adiscon.com</a>.
+<h3>Local Installation Maintenance</h3>
+<p>If you used our services to set up the system, why not keep it
+running perfectly with maintenance support? Under this contract, we 
+assure you run a recent build that does not interfere with your 
+environment and we even carry out change requests you may have. So this
+is a hassle-free, everything cared about solution.
+<p>Again, all we need to have is ssh access and the proper permissions
+to your machine. Of course, work will only be carried out when you
+expect us to do so. You are always in control of what happens. This
+is a perfect outsourcing solution for those who would like to run
+a great logging system but can not afford the time to keep it
+in perfect shape!
+<p>We charge a low monthly fee for this service. For details, please
+contact <a href="mailto:info@adiscon.com">info@adiscon.com</a>.
 <h3>Custom Development</h3>
 <p>Do you need an exotic feature that otherwise would not be implemented?
 Do you need something really quick, quicker than it is available via
@@ -85,4 +109,4 @@ Copyright&nbsp;
 Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>.
 Released under the GNU GPL version 3 or higher.</font></p>
-</body></html>
\ No newline at end of file
+</body></html>
-- 
cgit 


From 47bbd838bdaa200d89d8210a6a0ba9c322129bca Mon Sep 17 00:00:00 2001
From: Tom Bergfeld <tbergfeld@hq.adiscon.com>
Date: Fri, 31 Oct 2008 14:47:30 +0100
Subject: restructured rsyslog.conf documentation

Signed-off-by: Rainer Gerhards <rgerhards@adiscon.com>
---
 ChangeLog                                          |    1 +
 doc/expression.html                                |    3 +-
 doc/features.html                                  |   13 +-
 doc/im3195.html                                    |    2 +
 doc/imfile.html                                    |    2 +
 doc/imgssapi.html                                  |    2 +
 doc/imklog.html                                    |    2 +
 doc/imrelp.html                                    |    2 +
 doc/imtcp.html                                     |    2 +
 doc/imuxsock.html                                  |    2 +
 doc/log_rotation_fix_size.html                     |   10 +
 doc/netstream.html                                 |    4 +-
 doc/omlibdbi.html                                  |    2 +
 doc/ommail.html                                    |    3 +-
 doc/ommysql.html                                   |    2 +
 doc/omrelp.html                                    |    4 +-
 doc/omsnmp.html                                    |    1 +
 doc/property_replacer.html                         |   10 +
 doc/queues.html                                    |    9 +
 ...onf1_actionexeconlywhenpreviousissuspended.html |    2 +
 doc/rsconf1_actionresumeinterval.html              |    4 +-
 doc/rsconf1_allowedsender.html                     |    2 +
 doc/rsconf1_controlcharacterescapeprefix.html      |    2 +
 doc/rsconf1_debugprintcfsyslinehandlerlist.html    |    4 +-
 doc/rsconf1_debugprintmodulelist.html              |    3 +-
 doc/rsconf1_debugprinttemplatelist.html            |    4 +-
 doc/rsconf1_dircreatemode.html                     |    4 +-
 doc/rsconf1_dirgroup.html                          |    4 +-
 doc/rsconf1_dirowner.html                          |    4 +-
 ...rsconf1_dropmsgswithmaliciousdnsptrrecords.html |    4 +-
 doc/rsconf1_droptrailinglfonreception.html         |    2 +
 doc/rsconf1_dynafilecachesize.html                 |    4 +-
 doc/rsconf1_escapecontrolcharactersonreceive.html  |    2 +
 doc/rsconf1_failonchownfailure.html                |    4 +-
 doc/rsconf1_filecreatemode.html                    |    2 +
 doc/rsconf1_filegroup.html                         |    4 +-
 doc/rsconf1_fileowner.html                         |    4 +-
 doc/rsconf1_gssforwardservicename.html             |    2 +
 doc/rsconf1_gsslistenservicename.html              |    2 +
 doc/rsconf1_gssmode.html                           |    2 +
 doc/rsconf1_includeconfig.html                     |    4 +-
 doc/rsconf1_mainmsgqueuesize.html                  |    2 +
 doc/rsconf1_markmessageperiod.html                 |    4 +-
 doc/rsconf1_moddir.html                            |    4 +-
 doc/rsconf1_modload.html                           |    2 +
 doc/rsconf1_repeatedmsgreduction.html              |    4 +-
 doc/rsconf1_resetconfigvariables.html              |    4 +-
 doc/rsconf1_umask.html                             |    4 +-
 doc/rsyslog_conf.html                              | 1235 +-------------------
 doc/rsyslog_high_database_rate.html                |    9 +
 doc/rsyslog_mysql.html                             |   11 +-
 doc/rsyslog_ng_comparison.html                     |   10 +
 doc/rsyslog_stunnel.html                           |   11 +-
 doc/rsyslog_tls.html                               |   10 +
 doc/syslog_protocol.html                           |    9 +
 55 files changed, 225 insertions(+), 1239 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a11ce27d..0bf28fc4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -28,6 +28,7 @@ version before switching to this one.
   receive loop (aka receiving messsages at a high rate)
 - doc bugfix: queue doc had wrong parameter name for setting controlling
   worker thread shutdown period
+- restructured rsyslog.conf documentation
 ---------------------------------------------------------------------------
 Version 3.21.6  [DEVEL] (rgerhards), 2008-10-22
 - consolidated time calls during msg object creation, improves performance
diff --git a/doc/expression.html b/doc/expression.html
index e7eb7842..9e37cb7a 100644
--- a/doc/expression.html
+++ b/doc/expression.html
@@ -2,6 +2,7 @@
 <html><head>
 <meta http-equiv="Content-Language" content="en"><title>Expressions</title></head>
 <body>
+<a href="rsyslog_conf_filter.html">back</a>
 <h1>Expressions</h1>
 <p>Rsyslog supports expressions at a growing number of places. So
 far, they are supported for filtering messages.</p><p>Expression support is provided by RainerScript. For now, please see the formal expression definition in <a href="rainerscript.html">RainerScript ABNF</a>. It is the "expr" node.</p><p>C-like comments (/* some comment */) are supported <span style="font-weight: bold;">inside</span> the expression, but not yet in the rest of the configuration file.</p><p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
@@ -13,4 +14,4 @@ Copyright 
 Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>.
 Released under the GNU GPL version 3 or higher.</font></p>
-</body></html>
\ No newline at end of file
+</body></html>
diff --git a/doc/features.html b/doc/features.html
index d221eb77..89796a41 100644
--- a/doc/features.html
+++ b/doc/features.html
@@ -1,8 +1,8 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head><title>rsyslog features</title>
-
 </head>
 <body>
+<a href="rsyslog_conf.html">back</a>
 <h1>RSyslog - Features</h1>
 <p><b>This page lists both current features as well as
 those being considered for future versions of rsyslog.</b> If you
@@ -134,4 +134,15 @@ future of RFC 3195 in rsyslog</a>.</li>
 <p>To see when each feature was added, see the
 <a href="http://www.rsyslog.com/Topic4.phtml">rsyslog
 change log</a> (online only).</p>
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+
 </body></html>
+
diff --git a/doc/im3195.html b/doc/im3195.html
index d6f2f2ed..aad9f3d1 100644
--- a/doc/im3195.html
+++ b/doc/im3195.html
@@ -4,6 +4,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>RFC3195 Input Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; im3195</b></p>
 <p><b>Author: </b>Rainer Gerhards
diff --git a/doc/imfile.html b/doc/imfile.html
index 5bdbce5c..af0413dd 100644
--- a/doc/imfile.html
+++ b/doc/imfile.html
@@ -2,6 +2,8 @@
 <html><head>
 <meta http-equiv="Content-Language" content="en"><title>Text File Input Monitor</title></head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>Text File Input Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; imfile</b></p>
 <p><b>Author: </b>Rainer Gerhards
diff --git a/doc/imgssapi.html b/doc/imgssapi.html
index d644303e..ec183fe7 100644
--- a/doc/imgssapi.html
+++ b/doc/imgssapi.html
@@ -4,6 +4,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>GSSAPI Syslog Input Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; imgssapi</b></p>
 <p><b>Author: </b>varmojfekoj</p>
diff --git a/doc/imklog.html b/doc/imklog.html
index b5b21e84..9166bae6 100644
--- a/doc/imklog.html
+++ b/doc/imklog.html
@@ -4,6 +4,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>Kernel Log Input Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; imklog</b></p>
 <p><b>Author: </b>Rainer Gerhards
diff --git a/doc/imrelp.html b/doc/imrelp.html
index bfdaad84..53826ac2 100644
--- a/doc/imrelp.html
+++ b/doc/imrelp.html
@@ -4,6 +4,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>RELP Input Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; imrelp</b></p>
 <p><b>Author: Rainer Gerhards</b></p>
diff --git a/doc/imtcp.html b/doc/imtcp.html
index ecc72748..583cd531 100644
--- a/doc/imtcp.html
+++ b/doc/imtcp.html
@@ -2,6 +2,8 @@
 <html><head>
 <meta http-equiv="Content-Language" content="en"><title>TCP Syslog Input Module</title></head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>TCP Syslog Input Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; imtcp</b></p>
 <p><b>Author: </b>Rainer Gerhards
diff --git a/doc/imuxsock.html b/doc/imuxsock.html
index 77491992..472470a0 100644
--- a/doc/imuxsock.html
+++ b/doc/imuxsock.html
@@ -4,6 +4,8 @@
 <title>Unix Socket Input</title>
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>Unix Socket Input</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; imuxsock</b></p>
 <p><b>Author: </b>Rainer Gerhards
diff --git a/doc/log_rotation_fix_size.html b/doc/log_rotation_fix_size.html
index 0b9a3b2e..190b24cb 100644
--- a/doc/log_rotation_fix_size.html
+++ b/doc/log_rotation_fix_size.html
@@ -3,6 +3,8 @@
 <meta name="KEYWORDS" content="log rotation, howto, guide, fixed-size log">
 </head>
 <body>
+<a href="rsyslog_conf_output.html">back</a>
+
 <h1>Log rotation with rsyslog</h1>
 		<P><small><i>Written by
 		Michael Meckelein</i></small></P>
@@ -54,6 +56,14 @@ file and fill it up with new logs. So the latest logs are always in log_roatatio
 <p>With this approach two files for logging are used, each with a maximum size of 50 MB. So 
 we can say we have successfully configured a log rotation which satisfies our requirement. 
 We keep the logs at a fixed-size level of100 MB.</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
 
 </body>
 </html>
diff --git a/doc/netstream.html b/doc/netstream.html
index e7d54c12..cbfa12ae 100644
--- a/doc/netstream.html
+++ b/doc/netstream.html
@@ -3,6 +3,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h1>Network Stream Drivers</h1><p>Network stream drivers are a layer
 between various parts of rsyslogd (e.g. the imtcp module) and the
 transport layer. They provide sequenced delivery, authentication and
@@ -18,4 +20,4 @@ Copyright 
 Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>.
 Released under the GNU GPL version 3 or higher.</font></p>
-</body></html>
\ No newline at end of file
+</body></html>
diff --git a/doc/omlibdbi.html b/doc/omlibdbi.html
index 8ff74371..ec1d01b6 100644
--- a/doc/omlibdbi.html
+++ b/doc/omlibdbi.html
@@ -4,6 +4,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>Generic Database Output Module (omlibdbi)</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; omlibdbi</b></p>
 <p><b>Author: </b>Rainer Gerhards
diff --git a/doc/ommail.html b/doc/ommail.html
index c18cf3f8..0841dc9f 100644
--- a/doc/ommail.html
+++ b/doc/ommail.html
@@ -1,6 +1,6 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head><title>mail output module - sending syslog messages via mail</title>
-
+<a href="features.html">back</a>
 </head>
 <body>
 <h1>Mail Output Module (ommail)</h1>
@@ -142,4 +142,5 @@ Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer
 Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>.
 Released under the GNU GPL version 3 or higher.</font></p>
+
 </body></html>
diff --git a/doc/ommysql.html b/doc/ommysql.html
index 79d913eb..7a3f5930 100644
--- a/doc/ommysql.html
+++ b/doc/ommysql.html
@@ -5,6 +5,8 @@
 </head>
 
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>MySQL Database Output Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; ommysql</b></p>
 <p><b>Author: </b>Michael Meckelein (Initial Author) / Rainer Gerhards
diff --git a/doc/omrelp.html b/doc/omrelp.html
index 0952cc71..82a62afc 100644
--- a/doc/omrelp.html
+++ b/doc/omrelp.html
@@ -4,6 +4,8 @@
 
 </head>
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
+
 <h1>RELP Output Module (omlibdbi)</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; omrelp</b></p>
 <p><b>Author: </b>Rainer Gerhards
@@ -51,4 +53,4 @@ Copyright 
 Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>.
 Released under the GNU GPL version 3 or higher.</font></p>
-</body></html>
\ No newline at end of file
+</body></html>
diff --git a/doc/omsnmp.html b/doc/omsnmp.html
index 31aaef24..b38a594f 100644
--- a/doc/omsnmp.html
+++ b/doc/omsnmp.html
@@ -4,6 +4,7 @@
 <title>SNMP Output Module</title></head>
 
 <body>
+<a href="rsyslog_conf_modules.html">back</a>
 
 <h1>SNMP Output Module</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; omsnmp</b></p>
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index f666fb76..34e2116c 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -1,6 +1,7 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head><title>The Rsyslogd Property Replacer</title></head>
 <body>
+<a href="rsyslog_conf_templates.html">back</a>
 <h1>The Property Replacer</h1>
 <p><b>The property replacer is a core component in
 rsyslogd's output system.</b> A syslog message has a number of
@@ -398,4 +399,13 @@ to record severity and facility of a message)</li>
 <li><a href="rsyslog_conf.html">Configuration file
 syntax</a>, this is where you actually use the property replacer.</li>
 </ul>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+
 </body></html>
diff --git a/doc/queues.html b/doc/queues.html
index 7461121b..727bc26a 100644
--- a/doc/queues.html
+++ b/doc/queues.html
@@ -3,6 +3,7 @@
 <meta http-equiv="Content-Language" content="de">
 <title>Understanding rsyslog queues</title></head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
 
 <h1>Understanding rsyslog Queues</h1>
 <p>Rsyslog uses queues whenever two activities need to be loosely coupled. With a 
@@ -357,5 +358,13 @@ environment for the next action).</p>
 parameters, because not all are applicable. For example, in current output 
 module design, actions do not support multi-threading. Consequently, the number 
 of worker threads is fixed to one for action queues and can not be changed.</p>
+[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
 
 </body></html>
diff --git a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html
index d5cf8b14..1626b4ca 100644
--- a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html
+++ b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$ActionExecOnlyWhenPreviousIsSuspended</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> off</p>
diff --git a/doc/rsconf1_actionresumeinterval.html b/doc/rsconf1_actionresumeinterval.html
index a854a212..c0365470 100644
--- a/doc/rsconf1_actionresumeinterval.html
+++ b/doc/rsconf1_actionresumeinterval.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$ActionResumeInterval</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> 30</p>
@@ -27,4 +29,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_allowedsender.html b/doc/rsconf1_allowedsender.html
index 4a980b89..ac39e268 100644
--- a/doc/rsconf1_allowedsender.html
+++ b/doc/rsconf1_allowedsender.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$AllowedSender</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> all allowed</p>
diff --git a/doc/rsconf1_controlcharacterescapeprefix.html b/doc/rsconf1_controlcharacterescapeprefix.html
index 6dab1e2e..45cd9230 100644
--- a/doc/rsconf1_controlcharacterescapeprefix.html
+++ b/doc/rsconf1_controlcharacterescapeprefix.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$ControlCharacterEscapePrefix</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> \</p>
diff --git a/doc/rsconf1_debugprintcfsyslinehandlerlist.html b/doc/rsconf1_debugprintcfsyslinehandlerlist.html
index 1aad7552..e158de43 100644
--- a/doc/rsconf1_debugprintcfsyslinehandlerlist.html
+++ b/doc/rsconf1_debugprintcfsyslinehandlerlist.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DebugPrintCFSyslineHandlerList</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> on</p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_debugprintmodulelist.html b/doc/rsconf1_debugprintmodulelist.html
index 4d8e9bff..f25663fb 100644
--- a/doc/rsconf1_debugprintmodulelist.html
+++ b/doc/rsconf1_debugprintmodulelist.html
@@ -3,6 +3,7 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
 <h2>$DebugPrintModuleList</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> on</p>
@@ -19,4 +20,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_debugprinttemplatelist.html b/doc/rsconf1_debugprinttemplatelist.html
index 243530e1..b5f1f28f 100644
--- a/doc/rsconf1_debugprinttemplatelist.html
+++ b/doc/rsconf1_debugprinttemplatelist.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DebugPrintTemplateList</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> on</p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_dircreatemode.html b/doc/rsconf1_dircreatemode.html
index 66a35e18..9a9c61eb 100644
--- a/doc/rsconf1_dircreatemode.html
+++ b/doc/rsconf1_dircreatemode.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DirCreateMode</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> 0644</p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_dirgroup.html b/doc/rsconf1_dirgroup.html
index 868e5ecd..de070126 100644
--- a/doc/rsconf1_dirgroup.html
+++ b/doc/rsconf1_dirgroup.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DirGroup</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_dirowner.html b/doc/rsconf1_dirowner.html
index e85a5122..da8e252d 100644
--- a/doc/rsconf1_dirowner.html
+++ b/doc/rsconf1_dirowner.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DirOwner</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html
index e0a53ae6..95027a70 100644
--- a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html
+++ b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DropMsgsWithMaliciousDnsPTRRecords</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> off</p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_droptrailinglfonreception.html b/doc/rsconf1_droptrailinglfonreception.html
index 1e3aa8af..fb59b871 100644
--- a/doc/rsconf1_droptrailinglfonreception.html
+++ b/doc/rsconf1_droptrailinglfonreception.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DropTrailingLFOnReception</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> on</p>
diff --git a/doc/rsconf1_dynafilecachesize.html b/doc/rsconf1_dynafilecachesize.html
index 3813f981..cacbf6e5 100644
--- a/doc/rsconf1_dynafilecachesize.html
+++ b/doc/rsconf1_dynafilecachesize.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$DynaFileCacheSize</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> 10</p>
@@ -20,4 +22,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_escapecontrolcharactersonreceive.html b/doc/rsconf1_escapecontrolcharactersonreceive.html
index 26917736..178f9a6f 100644
--- a/doc/rsconf1_escapecontrolcharactersonreceive.html
+++ b/doc/rsconf1_escapecontrolcharactersonreceive.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$EscapeControlCharactersOnReceive</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> on</p>
diff --git a/doc/rsconf1_failonchownfailure.html b/doc/rsconf1_failonchownfailure.html
index 0e646e36..d8bbab82 100644
--- a/doc/rsconf1_failonchownfailure.html
+++ b/doc/rsconf1_failonchownfailure.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$FailOnChownFailure</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> on</p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_filecreatemode.html b/doc/rsconf1_filecreatemode.html
index c8440864..10b0317b 100644
--- a/doc/rsconf1_filecreatemode.html
+++ b/doc/rsconf1_filecreatemode.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$FileCreateMode</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> 0644</p>
diff --git a/doc/rsconf1_filegroup.html b/doc/rsconf1_filegroup.html
index b9acaab7..dd5b8ad5 100644
--- a/doc/rsconf1_filegroup.html
+++ b/doc/rsconf1_filegroup.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$FileGroup</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_fileowner.html b/doc/rsconf1_fileowner.html
index 7a9cbbc7..935cfffd 100644
--- a/doc/rsconf1_fileowner.html
+++ b/doc/rsconf1_fileowner.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$FileOwner</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_gssforwardservicename.html b/doc/rsconf1_gssforwardservicename.html
index 9d39dc2a..45d9ba98 100644
--- a/doc/rsconf1_gssforwardservicename.html
+++ b/doc/rsconf1_gssforwardservicename.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$GssForwardServiceName</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> host</p>
diff --git a/doc/rsconf1_gsslistenservicename.html b/doc/rsconf1_gsslistenservicename.html
index cd03dc58..5fdf3edc 100644
--- a/doc/rsconf1_gsslistenservicename.html
+++ b/doc/rsconf1_gsslistenservicename.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$GssListenServiceName</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> host</p>
diff --git a/doc/rsconf1_gssmode.html b/doc/rsconf1_gssmode.html
index 71c50696..2b1d5656 100644
--- a/doc/rsconf1_gssmode.html
+++ b/doc/rsconf1_gssmode.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$GssMode</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> encryption</p>
diff --git a/doc/rsconf1_includeconfig.html b/doc/rsconf1_includeconfig.html
index 24462f77..132cee6f 100644
--- a/doc/rsconf1_includeconfig.html
+++ b/doc/rsconf1_includeconfig.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$IncludeConfig</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -43,4 +45,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_mainmsgqueuesize.html b/doc/rsconf1_mainmsgqueuesize.html
index acf88e94..ffed1c09 100644
--- a/doc/rsconf1_mainmsgqueuesize.html
+++ b/doc/rsconf1_mainmsgqueuesize.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$MainMsgQueueSize</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> 10000</p>
diff --git a/doc/rsconf1_markmessageperiod.html b/doc/rsconf1_markmessageperiod.html
index 9b6590cd..2c833339 100644
--- a/doc/rsconf1_markmessageperiod.html
+++ b/doc/rsconf1_markmessageperiod.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$MarkMessagePeriod</h2>
 <p><b>Type:</b> specific to immark input module</p>
 <p><b>Default:</b> 1800 (20 minutes)</p>
@@ -27,4 +29,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_moddir.html b/doc/rsconf1_moddir.html
index ced07dc9..889de05d 100644
--- a/doc/rsconf1_moddir.html
+++ b/doc/rsconf1_moddir.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$ModDir</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> system default for user libraries, e.g. 
@@ -24,4 +26,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_modload.html b/doc/rsconf1_modload.html
index a2b8087a..ce457ea5 100644
--- a/doc/rsconf1_modload.html
+++ b/doc/rsconf1_modload.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$ModLoad</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
diff --git a/doc/rsconf1_repeatedmsgreduction.html b/doc/rsconf1_repeatedmsgreduction.html
index 20e56f89..248e8343 100644
--- a/doc/rsconf1_repeatedmsgreduction.html
+++ b/doc/rsconf1_repeatedmsgreduction.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$RepeatedMsgReduction</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> depending on -e</p>
@@ -20,4 +22,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_resetconfigvariables.html b/doc/rsconf1_resetconfigvariables.html
index 9794d158..46cf0bdf 100644
--- a/doc/rsconf1_resetconfigvariables.html
+++ b/doc/rsconf1_resetconfigvariables.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$ResetConfigVariables</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -19,4 +21,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsconf1_umask.html b/doc/rsconf1_umask.html
index ee47dbad..8e41e672 100644
--- a/doc/rsconf1_umask.html
+++ b/doc/rsconf1_umask.html
@@ -3,6 +3,8 @@
 <title>rsyslog.conf file</title>
 </head>
 <body>
+<a href="rsyslog_conf_global.html">back</a>
+
 <h2>$UMASK</h2>
 <p><b>Type:</b> global configuration directive</p>
 <p><b>Default:</b> </p>
@@ -21,4 +23,4 @@ Copyright &copy; 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
 version 2 or higher.</font></p>
 </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html
index cd3db405..852d95b5 100644
--- a/doc/rsyslog_conf.html
+++ b/doc/rsyslog_conf.html
@@ -20,257 +20,13 @@ possible. While, for obvious reasons, <a href="features.html">enhanced
 features</a> require a different config file syntax, rsyslogd
 should be able to work with a standard syslog.conf file. This is
 especially useful while you are migrating from syslogd to rsyslogd.</p>
-<h2>Modules</h2>
-<p>Rsyslog has a modular design. Consequently, there is a growing
-number of modules. Here is the entry point to their documentation and
-what they do (list is currently not complete)</p>
-<ul>
-<li><a href="omsnmp.html">omsnmp</a> - SNMP
-trap output module</li>
-<li><a href="omrelp.html">omrelp</a> - RELP
-output module</li>
-<li>omgssapi - output module for GSS-enabled syslog</li>
-<li><a href="ommysql.html">ommysql</a> - output module for MySQL</li>
-<li>ompgsql - output module for PostgreSQL</li>
-<li><a href="omlibdbi.html">omlibdbi</a> -
-generic database output module (Firebird/Interbase, MS SQL, Sybase,
-SQLLite, Ingres, Oracle, mSQL)</li>
-<li><a href="ommail.html">ommail</a> -
-permits rsyslog to alert folks by mail if something important happens</li>
-<li><a href="imfile.html">imfile</a>
--&nbsp; input module for text files</li>
-<li><a href="imrelp.html">imrelp</a> - RELP
-input module</li>
-<li>imudp - udp syslog message input</li>
-<li><a href="imtcp.html">imtcp</a> - input
-plugin for plain tcp syslog</li>
-<li><a href="imgssapi.html">imgssapi</a> -
-input plugin for plain tcp and GSS-enabled syslog</li>
-<li>immark - support for mark messages</li>
-<li><a href="imklog.html">imklog</a> - kernel logging</li>
-<li><a href="imuxsock.html">imuxsock</a> -
-unix sockets, including the system log socket</li>
-<li><a href="im3195.html">im3195</a> -
-accepts syslog messages via RFC 3195</li>
-</ul>
-<p>Please note that each module provides configuration
-directives, which are NOT necessarily being listed below. Also
-remember, that a modules configuration directive (and functionality) is
-only available if it has been loaded (using $ModLoad).</p>
+<h2><a href="rsyslog_conf_modules.html">Modules</a></h2>
 <h2>Lines</h2>
 Lines can be continued by specifying a backslash ("\") as the last
 character of the line. There is a hard-coded maximum line length of 4K.
 If you need lines larger than that, you need to change compile-time
 settings inside rsyslog and recompile.
-<h2>Global Directives</h2>
-<p>All global directives need to be specified on a line by their
-own and must start with a dollar-sign. Here is a list in alphabetical
-order. Follow links for a description.</p>
-<p>Please note that not all directives here are actually global. Some affect
-only the next action. This documentation will be changed soon.
-<p>Not all directives have an in-depth description right now.
-Default values for them are in bold. A more in-depth description will
-appear as implementation progresses.
-</p>
-<p><b>Be sure to read information about <a href="queues.html">queues in rsyslog</a></b> -
-many parameter settings modify queue parameters. If in doubt, use the
-default, it is usually well-chosen and applicable in most cases.</p>
-<ul>
-<li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li>
-<li>$ActionExecOnlyOnceEveryInterval &lt;seconds&gt; -
-execute action only if the last execute is at last
-&lt;seconds&gt; seconds in the past (more info in <a href="ommail.html">ommail</a>,
-but may be used with any action)</li>
-<li><i><b>$ActionExecOnlyEveryNthTime</b> &lt;number&gt;</i> - If configured, the next action will
-only be executed every n-th time. For example, if configured to 3, the first two messages
-that go into the action will be dropped, the 3rd will actually cause the action to execute,
-the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: 
-this setting is automatically re-set when the actual action is defined.</li>
-<li><i><b>$ActionExecOnlyEveryNthTimeTimeout</b> &lt;number-of-seconds&gt;</i> - has a meaning only if
-$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout
-setting specifies after which period the counting of "previous actions" expires and
-a new action count is begun. Specify 0 (the default) to disable timeouts.
-<br>
-<i>Why is this option needed?</i> Consider this case: a message comes in at, eg., 10am. That's
-count 1. Then, nothing happens for the next 10 hours. At 8pm, the next
-one occurs. That's count 2. Another 5 hours later, the next message
-occurs, bringing the total count to 3. Thus, this message now triggers
-the rule.
-<br>
-The question is if this is desired behavior? Or should the rule only be
-triggered if the messages occur within an e.g. 20 minute window? If the
-later is the case, you need a
-<br>
-$ActionExecOnlyEveryNthTimeTimeout 1200
-<br>
-This directive will timeout previous messages seen if they are older
-than 20 minutes. In the example above, the count would now be always 1
-and consequently no rule would ever be triggered.
-
-<li>$ActionFileDefaultTemplate [templateName] - sets a new default template for file actions</li>
-<li>$ActionFileEnableSync [on/<span style="font-weight: bold;">off</span>] - enables file
-syncing capability of omfile</li>
-<li>$ActionForwardDefaultTemplate [templateName] - sets a new
-default template for UDP and plain TCP forwarding action</li>
-<li>$ActionGSSForwardDefaultTemplate [templateName] - sets a
-new default template for GSS-API forwarding action</li>
-<li>$ActionQueueCheckpointInterval &lt;number&gt;</li>
-<li>$ActionQueueDequeueSlowdown &lt;number&gt; [number
-is timeout in <i> micro</i>seconds (1000000us is 1sec!),
-default 0 (no delay). Simple rate-limiting!]</li>
-<li>$ActionQueueDiscardMark &lt;number&gt; [default
-9750]</li>
-<li>$ActionQueueDiscardSeverity &lt;number&gt;
-[*numerical* severity! default 4 (warning)]</li>
-<li>$ActionQueueFileName &lt;name&gt;</li>
-<li>$ActionQueueHighWaterMark &lt;number&gt; [default
-8000]</li>
-<li>$ActionQueueImmediateShutdown [on/<b>off</b>]</li>
-<li>$ActionQueueSize &lt;number&gt;</li>
-<li>$ActionQueueLowWaterMark &lt;number&gt; [default
-2000]</li>
-<li>$ActionQueueMaxFileSize &lt;size_nbr&gt;, default 1m</li>
-<li>$ActionQueueTimeoutActionCompletion &lt;number&gt;
-[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means
-immediate!]</li>
-<li>$ActionQueueTimeoutEnqueue &lt;number&gt; [number
-is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li>
-<li>$ActionQueueTimeoutShutdown &lt;number&gt; [number
-is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li>
-<li>$ActionQueueWorkerTimeoutThreadShutdown
-&lt;number&gt; [number is timeout in ms (1000ms is 1sec!),
-default 60000 (1 minute)]</li>
-<li>$ActionQueueType [FixedArray/LinkedList/<b>Direct</b>/Disk]</li>
-<li>$ActionQueueSaveOnShutdown&nbsp; [on/<b>off</b>]
-</li>
-<li>$ActionQueueWorkerThreads &lt;number&gt;, num worker threads, default 1, recommended 1</li>
-<li>$ActionQueueWorkerThreadMinumumMessages &lt;number&gt;, default 100</li>
-<li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li>
-<li>$ActionResumeRetryCount &lt;number&gt; [default 0, -1 means eternal]</li>
-<li>$ActionSendResendLastMsgOnReconn &lt;[on/<b>off</b>]&gt; specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication.
-<li>$ActionSendStreamDriver &lt;driver basename&gt; just like $DefaultNetstreamDriver, but for the specific action
-</li><li>$ActionSendStreamDriverMode &lt;mode&gt;, default 0, mode to use with the stream driver
-(driver-specific)</li><li>$ActionSendStreamDriverAuthMode &lt;mode&gt;,&nbsp; authentication mode to use with the stream driver
-(driver-specific)</li><li>$ActionSendStreamDriverPermittedPeer &lt;ID&gt;,&nbsp; accepted fingerprint (SHA1) or name of remote peer
-(driver-specific) -<span style="font-weight: bold;"> directive may go away</span>!</li>
-<li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li>
-<li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li>
-<li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li>
-
-<li><a href="rsconf1_debugprintmodulelist.html">$DebugPrintModuleList</a></li>
-<li><a href="rsconf1_debugprinttemplatelist.html">$DebugPrintTemplateList</a></li>
-<li>$DefaultNetstreamDriver &lt;drivername&gt;, the default <a href="netstream.html">network stream driver</a> to use. Defaults to&nbsp;ptcp.$DefaultNetstreamDriverCAFile &lt;/path/to/cafile.pem&gt;</li>
-<li>$DefaultNetstreamDriverCertFile &lt;/path/to/certfile.pem&gt;</li>
-<li>$DefaultNetstreamDriverKeyFile &lt;/path/to/keyfile.pem&gt;</li>
-<li><a href="rsconf1_dircreatemode.html">$DirCreateMode</a></li>
-<li><a href="rsconf1_dirgroup.html">$DirGroup</a></li>
-<li><a href="rsconf1_dirowner.html">$DirOwner</a></li>
-<li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li>
-<li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li>
-<li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li>
-<li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li>
-<li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li>
-<li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li>
-<li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li>
-<li><a href="rsconf1_filegroup.html">$FileGroup</a></li>
-<li><a href="rsconf1_fileowner.html">$FileOwner</a></li>
-<li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li>
-<li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li>
-<li><a href="rsconf1_gssmode.html">$GssMode</a></li>
-<li>$HUPisRestart [<b>on</b>/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending
-on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is
-not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for
-compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually
-the only one that is needed e.g. for log rotation. <b>It is recommended to set the setting to "off".</b></li>
-<li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval &lt;number&gt;</li>
-<li>$MainMsgQueueDequeueSlowdown &lt;number&gt; [number
-is timeout in <i> micro</i>seconds (1000000us is 1sec!),
-default 0 (no delay). Simple rate-limiting!]</li>
-<li>$MainMsgQueueDiscardMark &lt;number&gt; [default 9750]</li>
-<li>$MainMsgQueueDiscardSeverity &lt;severity&gt;
-[either a textual or numerical severity! default 4 (warning)]</li>
-<li>$MainMsgQueueFileName &lt;name&gt;</li>
-<li>$MainMsgQueueHighWaterMark &lt;number&gt; [default
-8000]</li>
-<li>$MainMsgQueueImmediateShutdown [on/<b>off</b>]</li>
-<li><a href="rsconf1_mainmsgqueuesize.html">$MainMsgQueueSize</a></li>
-<li>$MainMsgQueueLowWaterMark &lt;number&gt; [default
-2000]</li>
-<li>$MainMsgQueueMaxFileSize &lt;size_nbr&gt;, default
-1m</li>
-<li>$MainMsgQueueTimeoutActionCompletion
-&lt;number&gt; [number is timeout in ms (1000ms is 1sec!),
-default
-1000, 0 means immediate!]</li>
-<li>$MainMsgQueueTimeoutEnqueue &lt;number&gt; [number
-is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li>
-<li>$MainMsgQueueTimeoutShutdown &lt;number&gt; [number
-is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li>
-<li>$MainMsgQueueWorkerTimeoutThreadShutdown
-&lt;number&gt; [number is timeout in ms (1000ms is 1sec!),
-default 60000 (1 minute)]</li>
-<li>$MainMsgQueueType [<b>FixedArray</b>/LinkedList/Direct/Disk]</li>
-<li>$MainMsgQueueSaveOnShutdown&nbsp; [on/<b>off</b>]
-</li>
-<li>$MainMsgQueueWorkerThreads &lt;number&gt;, num
-worker threads, default 1, recommended 1</li>
-<li>$MainMsgQueueWorkerThreadMinumumMessages &lt;number&gt;, default 100</li>
-<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> (immark)</li>
-<li><b><i>$MaxMessageSize</i></b> &lt;size_nbr&gt;, default 2k - allows to specify maximum supported message size
-(both for sending and receiving). The default
-should be sufficient for almost all cases. Do not set this below 1k, as it would cause
-interoperability problems with other syslog implementations.<br>
-Change the setting to e.g. 32768 if you would like to
-support large message sizes for IHE (32k is the current maximum
-needed for IHE). I was initially tempted to set the default to 32k,
-but there is a some memory footprint with the current
-implementation in rsyslog.
-<br>If you intend to receive Windows Event Log data (e.g. via
-<a href="http://www.eventreporter.com/">EventReporter</a>), you might want to 
-increase this number to an even higher value, as event
-log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea).
-Note: testing showed that 4k seems to be
-the typical maximum for <b>UDP</b> based syslog. This is an IP stack
-restriction. Not always ... but very often. If you go beyond
-that value, be sure to test that rsyslogd actually does what
-you think it should do ;) It is highly suggested to use a TCP based transport
-instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions.
-<br>Note that 2k, the current default, is the smallest size that must be
-supported in order to be compliant to the upcoming new syslog RFC series.
-</li>
-<li><a href="rsconf1_moddir.html">$ModDir</a></li>
-<li><a href="rsconf1_modload.html">$ModLoad</a></li>
-<li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li>
-<li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li>
-<li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better
-performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The
-default may change as uniprocessor systems become less common.</li>
-<li>$WorkDirectory &lt;name&gt; (directory for spool and other work files)</li>
-<li>$UDPServerAddress &lt;IP&gt; (imudp) -- local IP
-address (or name) the UDP listens should bind to</li>
-<li>$UDPServerRun &lt;port&gt; (imudp) -- former
--r&lt;port&gt; option, default 514, start UDP server on this
-port, "*" means all addresses</li>
-<li>$UDPServerTimeRequery &lt;nbr-of-times&gt; (imudp) -- this is a performance 
-optimization. Getting the system time is very costly. With this setting, imudp can
-be instructed to obtain the precise time only once every n-times. This logic is
-only activated if messages come in at a very fast rate, so doing less frequent
-time calls should usually be acceptable. The default value is two, because we have
-seen that even without optimization the kernel often returns twice the identical time.
-You can set this value as high as you like, but do so at your own risk. The higher
-the value, the less precise the timestamp.
-<li><a href="rsconf1_umask.html">$UMASK</a></li>
-</ul>
-<p><b>Where &lt;size_nbr&gt; is specified above,</b>
-modifiers can be used after the number part. For example, 1k means
-1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa).
-Lower case letters refer to the traditional binary defintion (e.g. 1m
-equals 1,048,576) whereas upper case letters refer to their new
-1000-based definition (e.g 1M equals 1,000,000).</p>
-<p>Numbers may include '.' and ',' for readability. So you can
-for example specify either "1000" or "1,000" with the same result.
-Please note that rsyslogd simply ignores the punctuation. Form it's
-point of view, "1,,0.0.,.,0" also has the value 1000. </p>
+<h2><a href="rsyslog_conf_global.html">Global Directives</a></h2>
 <h2>Basic Structure</h2>
 <p>Rsyslog supports standard sysklogd's configuration file format
 and extends it. So in general, you can take a "normal" syslog.conf and
@@ -289,975 +45,15 @@ priorities belonging to the specified action.<br>
 <br>
 Lines starting with a hash mark ("#'') and empty lines are ignored.
 </p>
-<h2>Templates</h2>
-<p>Templates are a key feature of rsyslog. They allow to specify
-any
-format a user might want. They are also used for dynamic file name
-generation. Every output in rsyslog uses templates - this holds true
-for files, user messages and so on. The database writer expects its
-template to be a proper SQL statement - so this is highly customizable
-too. You might ask how does all of this work when no templates at all
-are specified. Good question ;) The answer is simple, though. Templates
-compatible with the stock syslogd formats are hardcoded into rsyslogd.
-So if no template is specified, we use one of these hardcoded
-templates. Search for "template_" in syslogd.c and you will find the
-hardcoded ones.</p>
-<p>A template consists of a template directive, a name, the
-actual template text and optional options. A sample is:</p>
-<blockquote><code>$template MyTemplateName,"\7Text
-%property% some more text\n",&lt;options&gt;</code></blockquote>
-<p>The "$template" is the template directive. It tells rsyslog
-that this line contains a template. "MyTemplateName" is the template
-name. All
-other config lines refer to this name. The text within quotes is the
-actual template text. The backslash is an escape character, much as it
-is in C. It does all these "cool" things. For example, \7 rings the
-bell (this is an ASCII value), \n is a new line. C programmers and perl
-coders have the advantage of knowing this, but the set in rsyslog is a
-bit restricted currently.
-</p>
-<p>All text in the template is used literally, except for things
-within percent signs. These are properties and allow you access to the
-contents of the syslog message. Properties are accessed via the
-property replacer (nice name, huh) and it can do cool things, too. For
-example, it can pick a substring or do date-specific formatting. More
-on this is below, on some lines of the property replacer.<br>
-<br>
-The &lt;options&gt; part is optional. It carries options
-influencing the template as whole. See details below. Be sure NOT to
-mistake template options with property options - the later ones are
-processed by the property replacer and apply to a SINGLE property, only
-(and not the whole template).<br>
-<br>
-Template options are case-insensitive. Currently defined are: </p>
-<p><b>sql</b> - format the string suitable for a SQL
-statement in MySQL format. This will replace single quotes ("'") and
-the backslash character by their backslash-escaped counterpart ("\'"
-and "\\") inside each field. Please note that in MySQL configuration,
-the <code class="literal">NO_BACKSLASH_ESCAPES</code>
-mode must be turned off for this format to work (this is the default).</p>
-<p><b>stdsql</b> - format the string suitable for a
-SQL statement that is to be sent to a standards-compliant sql server.
-This will replace single quotes ("'") by two single quotes ("''")
-inside each field. You must use stdsql together with MySQL if in MySQL
-configuration the
-<code class="literal">NO_BACKSLASH_ESCAPES</code> is
-turned on.</p>
-<p>Either the <b>sql</b> or <b>stdsql</b>&nbsp;
-option <b>must</b> be specified when a template is used
-for writing to a database, otherwise injection might occur. Please note
-that due to the unfortunate fact that several vendors have violated the
-sql standard and introduced their own escape methods, it is impossible
-to have a single option doing all the work.&nbsp; So you yourself
-must make sure you are using the right format. <b>If you choose
-the wrong one, you are still vulnerable to sql injection.</b><br>
-<br>
-Please note that the database writer *checks* that the sql option is
-present in the template. If it is not present, the write database
-action is disabled. This is to guard you against accidental forgetting
-it and then becoming vulnerable to SQL injection. The sql option can
-also be useful with files - especially if you want to import them into
-a database on another machine for performance reasons. However, do NOT
-use it if you do not have a real need for it - among others, it takes
-some toll on the processing time. Not much, but on a really busy system
-you might notice it ;)</p>
-<p>The default template for the write to database action has the
-sql option set. As we currently support only MySQL and the sql option
-matches the default MySQL configuration, this is a good choice.
-However, if you have turned on
-<code class="literal">NO_BACKSLASH_ESCAPES</code> in
-your MySQL config, you need to supply a template with the stdsql
-option. Otherwise you will become vulnerable to SQL injection. <br>
-<br>
-To escape:<br>
-% = \%<br>
-\ = \\ --&gt; '\' is used to escape (as in C)<br>
-$template TraditionalFormat,%timegenerated% %HOSTNAME%
-%syslogtag%%msg%\n"<br>
-<br>
-Properties can be accessed by the <a href="property_replacer.html">property
-replacer</a> (see there for details).</p>
-<p><b>Please note that templates can also by
-used to generate selector lines with dynamic file names.</b> For
-example, if you would like to split syslog messages from different
-hosts to different files (one per host), you can define the following
-template:</p>
-<blockquote><code>$template
-DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote>
-<p>This template can then be used when defining an output
-selector line. It will result in something like
-"/var/log/system-localhost.log"</p>
-<p>Template
-names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT
-use them if, otherwise you may receive a conflict in the future (and
-quite unpredictable behaviour). There is a small set of pre-defined
-templates that you can use without the need to define it:</p>
-<ul>
-<li><span style="font-weight: bold;">RSYSLOG_TraditionalFileFormat</span>
-- the "old style" default log file format with low-precision timestamps</li>
-<li><span style="font-weight: bold;">RSYSLOG_FileFormat</span>
-- a modern-style logfile format similar to TraditionalFileFormat, buth
-with high-precision timestamps and timezone information</li>
-<li><span style="font-weight: bold;">RSYSLOG_TraditionalForwardFormat</span>
-- the traditional forwarding format with low-precision timestamps. Most
-useful if you send&nbsp;messages to other syslogd's or rsyslogd
-below
-version 3.12.5.</li>
-<li><span style="font-weight: bold;">RSYSLOG_ForwardFormat</span>
-- a new high-precision forwarding format very similar to the
-traditional one, but with high-precision timestamps and timezone
-information. Recommended to be used when sending messages to rsyslog
-3.12.5 or above.</li>
-<li><span style="font-weight: bold;">RSYSLOG_SyslogProtocol23Format</span>
-- the format specified in IETF's internet-draft
-ietf-syslog-protocol-23, which is assumed to be come the new syslog
-standard RFC. This format includes several improvements. The rsyslog
-message parser understands this format, so you can use it together with
-all relatively recent versions of rsyslog. Other syslogd's may get
-hopelessly confused if receiving that format, so check before you use
-it. Note that the format is unlikely to change when the final RFC comes
-out, but this may happen.</li>
-<li><span style="font-weight: bold;">RSYSLOG_DebugFormat</span>
-- a special format used for troubleshooting property problems. This format
-is meant to be written to a log file. Do <b>not</b> use for production or remote
-forwarding.</li>
-</ul>
-<h2>Output Channels</h2>
-<p>Output Channels are a new concept first introduced in rsyslog
-0.9.0. <b>As of this writing, it is most likely that they will
-be replaced by something different in the future.</b> So if you
-use them, be prepared to change you configuration file syntax when you
-upgrade to a later release.<br>
-<br>
-The idea behind output channel definitions is that it shall provide an
-umbrella for any type of output that the user might want. In essence,<br>
-this is the "file" part of selector lines (and this is why we are not
-sure output channel syntax will stay after the next review). There is a<br>
-difference, though: selector channels both have filter conditions
-(currently facility and severity) as well as the output destination.
-Output channels define the output definition, only. As of this build,
-they can only be used to write to files - not pipes, ttys or whatever
-else. If we stick with output channels, this will change over time.</p>
-<p>In concept, an output channel includes everything needed to
-know about an output actions. In practice, the current implementation
-only carries<br>
-a filename, a maximum file size and a command to be issued when this
-file size is reached. More things might be present in future version,
-which might also change the syntax of the directive.</p>
-<p>Output channels are defined via an $outchannel directive. It's
-syntax is as follows:<br>
-<br>
-$outchannel name,file-name,max-size,action-on-max-size<br>
-<br>
-name is the name of the output channel (not the file), file-name is the
-file name to be written to, max-size the maximum allowed size and
-action-on-max-size a command to be issued when the max size is reached.
-This command always has exactly one parameter. The binary is that part
-of action-on-max-size before the first space, its parameter is
-everything behind that space.<br>
-<br>
-Please note that max-size is queried BEFORE writing the log message to
-the file. So be sure to set this limit reasonably low so that any
-message might fit. For the current release, setting it 1k lower than
-you expected is helpful. The max-size must always be specified in bytes
-- there are no special symbols (like 1k, 1m,...) at this point of
-development.<br>
-<br>
-Keep in mind that $outchannel just defines a channel with "name". It
-does not activate it. To do so, you must use a selector line (see
-below). That selector line includes the channel name plus an $ sign in
-front of it. A sample might be:<br>
-<br>
-*.* $mychannel<br>
-<br>
-In its current form, output channels primarily provide the ability to
-size-limit an output file. To do so, specify a maximum size. When this
-size is reached, rsyslogd will execute the action-on-max-size command
-and then reopen the file and retry. The command should be something
-like a <a href="log_rotation_fix_size.html">log rotation
-script</a> or a similar thing.</p>
-<p>If there is no action-on-max-size command or the command did
-not resolve the situation, the file is closed and never reopened by
-rsyslogd (except, of course, by huping it). This logic was integrated
-when we first experienced severe issues with files larger 2gb, which
-could lead to rsyslogd dumping core. In such cases, it is more
-appropriate to stop writing to a single file. Meanwhile, rsyslogd has
-been fixed to support files larger 2gb, but obviously only on file
-systems and operating system versions that do so. So it can still make
-sense to enforce a 2gb file size limit.</p>
-<h2>Filter Conditions</h2>
-<p>Rsyslog offers four different types "filter conditions":</p>
-<ul>
-<li>BSD-style blocks</li>
-<li>"traditional" severity and facility based selectors</li>
-<li>property-based filters</li>
-<li>expression-based filters</li>
-</ul>
-<h3>Blocks</h3>
-<p>Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each
-block of lines is separated from the previous block by a program or
-hostname specification. A block will only log messages corresponding to
-the most recent program and hostname specifications given. Thus, a
-block which selects &#8216;ppp&#8217; as the program, directly followed by a block
-that selects messages from the hostname &#8216;dialhost&#8217;, then the second
-block will only log messages from the ppp program on dialhost.
-</p>
-<p>A program specification is a line beginning with &#8216;!prog&#8217; and
-the following blocks will be associated with calls to syslog from that
-specific program. A program specification for &#8216;foo&#8217; will also match any
-message logged by the kernel with the prefix &#8216;foo: &#8217;. Alternatively, a
-program specification &#8216;-foo&#8217; causes the following blocks to be applied
-to messages from any program but the one specified. A hostname
-specification of the form &#8216;+hostname&#8217; and the following blocks will be
-applied to messages received from the specified hostname.
-Alternatively, a hostname specification &#8216;-hostname&#8217; causes the
-following blocks to be applied to messages from any host but the one
-specified. If the hostname is given as &#8216;@&#8217;, the local hostname will be
-used. (NOT YET IMPLEMENTED) A program or hostname specification may be
-reset by giving the program or hostname as &#8216;*&#8217;.</p>
-<p>Please note that the "#!prog", "#+hostname" and "#-hostname"
-syntax available in BSD syslogd is not supported by rsyslogd. By
-default, no hostname or program is set.</p>
-<h3>Selectors</h3>
-<p><b>Selectors are the traditional way of filtering syslog
-messages.</b> They have been kept in rsyslog with their original
-syntax, because it is well-known, highly effective and also needed for
-compatibility with stock syslogd configuration files. If you just need
-to filter based on priority and facility, you should do this with
-selector lines. They are <b>not</b> second-class citizens
-in rsyslog and offer the best performance for this job.</p>
-<p>The selector field itself again consists of two parts, a
-facility and a priority, separated by a period (".''). Both parts are
-case insensitive and can also be specified as decimal numbers, but
-don't do that, you have been warned. Both facilities and priorities are
-described in rsyslog(3). The names mentioned below correspond to the
-similar LOG_-values in /usr/include/rsyslog.h.<br>
-<br>
-The facility is one of the following keywords: auth, authpriv, cron,
-daemon, kern, lpr, mail, mark, news, security (same as auth), syslog,
-user, uucp and local0 through local7. The keyword security should not
-be used anymore and mark is only for internal use and therefore should
-not be used in applications. Anyway, you may want to specify and
-redirect these messages here. The facility specifies the subsystem that
-produced the message, i.e. all mail programs log with the mail facility
-(LOG_MAIL) if they log using syslog.<br>
-<br>
-The priority is one of the following keywords, in ascending order:
-debug, info, notice, warning, warn (same as warning), err, error (same
-as err), crit, alert, emerg, panic (same as emerg). The keywords error,
-warn and panic are deprecated and should not be used anymore. The
-priority defines the severity of the message.<br>
-<br>
-The behavior of the original BSD syslogd is that all messages of the
-specified priority and higher are logged according to the given action.
-Rsyslogd behaves the same, but has some extensions.<br>
-<br>
-In addition to the above mentioned names the rsyslogd(8) understands
-the following extensions: An asterisk ("*'') stands for all facilities
-or all priorities, depending on where it is used (before or after the
-period). The keyword none stands for no priority of the given facility.<br>
-<br>
-You can specify multiple facilities with the same priority pattern in
-one statement using the comma (",'') operator. You may specify as much
-facilities as you want. Remember that only the facility part from such
-a statement is taken, a priority part would be skipped.</p>
-<p>Multiple selectors may be specified for a single action using
-the semicolon (";'') separator. Remember that each selector in the
-selector field is capable to overwrite the preceding ones. Using this
-behavior you can exclude some priorities from the pattern.</p>
-<p>Rsyslogd has a syntax extension to the original BSD source,
-that makes its use more intuitively. You may precede every priority
-with an equation sign ("='') to specify only this single priority and
-not any of the above. You may also (both is valid, too) precede the
-priority with an exclamation mark ("!'') to ignore all that
-priorities, either exact this one or this and any higher priority. If
-you use both extensions than the exclamation mark must occur before the
-equation sign, just use it intuitively.</p>
-<h3>Property-Based Filters</h3>
-<p>Property-based filters are unique to rsyslogd. They allow to
-filter on any property, like HOSTNAME, syslogtag and msg. A list of all
-currently-supported properties can be found in the <a href="property_replacer.html">property replacer documentation</a>
-(but keep in mind that only the properties, not the replacer is
-supported). With this filter, each properties can be checked against a
-specified value, using a specified compare operation. Currently, there
-is only a single compare operation (contains) available, but additional
-operations will be added in the future.</p>
-<p>A property-based filter must start with a colon in column 0.
-This tells rsyslogd that it is the new filter type. The colon must be
-followed by the property name, a comma, the name of the compare
-operation to carry out, another comma and then the value to compare
-against. This value must be quoted. There can be spaces and tabs
-between the commas. Property names and compare operations are
-case-sensitive, so "msg" works, while "MSG" is an invalid property
-name. In brief, the syntax is as follows:</p>
-<p><code><b>:property, [!]compare-operation, "value"</b></code></p>
-<p>The following <b>compare-operations</b> are
-currently supported:</p>
-<table id="table1" border="1" width="100%">
-<tbody>
-<tr>
-<td>contains</td>
-<td>Checks if the string provided in value is contained in
-the property. There must be an exact match, wildcards are not supported.</td>
-</tr>
-<tr>
-<td>isequal</td>
-<td>Compares the "value" string provided and the property
-contents. These two values must be exactly equal to match. The
-difference to contains is that contains searches for the value anywhere
-inside the property value, whereas all characters must be identical for
-isequal. As such, isequal is most useful for fields like syslogtag or
-FROMHOST, where you probably know the exact contents.</td>
-</tr>
-<tr>
-<td>startswith</td>
-<td>Checks if the value is found exactly at the beginning
-of the property value. For example, if you search for "val" with
-<p><code><b>:msg, startswith, "val"</b></code></p>
-<p>it will be a match if msg contains "values are in this
-message" but it won't match if the msg contains "There are values in
-this message" (in the later case, contains would match). Please note
-that "startswith" is by far faster than regular expressions. So even
-once they are implemented, it can make very much sense
-(performance-wise) to use "startswith".</p>
-</td>
-</tr>
-<tr>
-<td>regex</td>
-<td>Compares the property against the provided POSIX
-regular
-expression.</td>
-</tr>
-</tbody>
-</table>
-<p>You can use the bang-character (!) immediately in front of a
-compare-operation, the outcome of this operation is negated. For
-example, if msg contains "This is an informative message", the
-following sample would not match:</p>
-<p><code><b>:msg, contains, "error"</b></code></p>
-<p>but this one matches:</p>
-<p><code><b>:msg, !contains, "error"</b></code></p>
-<p>Using negation can be useful if you would like to do some
-generic processing but exclude some specific events. You can use the
-discard action in conjunction with that. A sample would be:</p>
-<p><code><b>*.*
-/var/log/allmsgs-including-informational.log<br>
-:msg, contains, "informational"&nbsp; <font color="#ff0000" size="4">~</font>
-<br>
-*.* /var/log/allmsgs-but-informational.log</b></code></p>
-<p>Do not overlook the red tilde in line 2! In this sample, all
-messages are written to the file allmsgs-including-informational.log.
-Then, all messages containing the string "informational" are discarded.
-That means the config file lines below the "discard line" (number 2 in
-our sample) will not be applied to this message. Then, all remaining
-lines will also be written to the file allmsgs-but-informational.log.</p>
-<p><b>Value</b> is a quoted string. It supports some
-escape sequences:</p>
-<p>\" - the quote character (e.g. "String with \"Quotes\"")<br>
-\\ - the backslash character (e.g. "C:\\tmp")</p>
-<p>Escape sequences always start with a backslash. Additional
-escape sequences might be added in the future. Backslash characters <b>must</b>
-be escaped. Any other sequence then those outlined above is invalid and
-may lead to unpredictable results.</p>
-<p>Probably, "msg" is the most prominent use case of property
-based filters. It is the actual message text. If you would like to
-filter based on some message content (e.g. the presence of a specific
-code), this can be done easily by:</p>
-<p><code><b>:msg, contains, "ID-4711"</b></code></p>
-<p>This filter will match when the message contains the string
-"ID-4711". Please note that the comparison is case-sensitive, so it
-would not match if "id-4711" would be contained in the message.</p>
-<p><code><b>:msg, regex, "fatal .* error"</b></code></p>
-<p>This filter uses a POSIX regular expression. It matches when
-the
-string contains the words "fatal" and "error" with anything in between
-(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as
-two spaces are required by the regular expression!).</p>
-<p>Getting property-based filters right can sometimes be
-challenging. In order to help you do it with as minimal effort as
-possible, rsyslogd spits out debug information for all property-based
-filters during their evaluation. To enable this, run rsyslogd in
-foreground and specify the "-d" option.</p>
-<p>Boolean operations inside property based filters (like
-'message contains "ID17" or message contains "ID18"') are currently not
-supported (except for "not" as outlined above). Please note that while
-it is possible to query facility and severity via property-based
-filters, it is far more advisable to use classic selectors (see above)
-for those cases.</p>
-<h3>Expression-Based Filters</h3>
-Expression based filters allow
-filtering on arbitrary complex expressions, which can include boolean,
-arithmetic and string operations. Expression filters will evolve into a
-full configuration scripting language. Unfortunately, their syntax will
-slightly change during that process. So if you use them now, you need
-to be prepared to change your configuration files some time later.
-However, we try to implement the scripting facility as soon as possible
-(also in respect to stage work needed). So the window of exposure is
-probably not too long.<br>
-<br>
-Expression based filters are indicated by the keyword "if" in column 1
-of a new line. They have this format:<br>
-<br>
-if expr then action-part-of-selector-line<br>
-<br>
-"If" and "then" are fixed keywords that mus be present. "expr" is a
-(potentially quite complex) expression. So the <a href="expression.h">expression documentation</a> for
-details. "action-part-of-selector-line" is an action, just as you know
-it (e.g. "/var/log/logfile" to write to that file).<br>
-<br>
-A few quick samples:<br>
-<br>
-<code>
-*.* /var/log/file1 # the traditional way<br>
-if $msg contains 'error' /var/log/errlog # the expression-based way<br>
-</code>
-<br>
-Right now, you need to specify numerical values if you would like to
-check for facilities and severity. These can be found in <a href="http://www.ietf.org/rfc/rfc3164.txt">RFC 3164</a>.
-If you don't like that, you can of course also use the textual property
-- just be sure to use the right one. As expression support is enhanced,
-this will change. For example, if you would like to filter on message
-that have facility local0, start with "DEVNAME" and have either
-"error1" or "error0" in their message content, you could use the
-following filter:<br>
-<br>
-<code>
-if $syslogfacility-text == 'local0' and $msg
-startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains
-'error0') then /var/log/somelog<br>
-</code>
-<br>
-Please note that the above <span style="font-weight: bold;">must
-all be on one line</span>! And if you would like to store all
-messages except those that contain "error1" or "error0", you just need
-to add a "not":<br>
-<br>
-<code>
-if $syslogfacility-text == 'local0' and $msg
-startswith 'DEVNAME' and <span style="font-weight: bold;">not</span>
-($msg contains 'error1' or $msg contains
-'error0') then /var/log/somelog<br>
-</code>
-<br>
-If you would like to do case-insensitive comparisons, use
-"contains_i" instead of "contains" and "startswith_i" instead of
-"startswith".<br>
-<br>
-Note that regular expressions are currently NOT
-supported in expression-based filters. These will be added later when
-function support is added to the expression engine (the reason is that
-regular expressions will be a separate loadable module, which requires
-some more prequisites before it can be implemented).<br>
-<h2>ACTIONS</h2>
-<p>The action field of a rule describes what to do with the
-message. In general, message content is written to a kind of "logfile".
-But also other actions might be done, like writing to a database table
-or forwarding to another host.<br>
-<br>
-Templates can be used with all actions. If used, the specified template
-is used to generate the message content (instead of the default
-template). To specify a template, write a semicolon after the action
-value immediately followed by the template name.<br>
-<br>
-Beware: templates MUST be defined BEFORE they are used. It is OK to
-define some templates, then use them in selector lines, define more
-templates and use use them in the following selector lines. But it is
-NOT permitted to use a template in a selector line that is above its
-definition. If you do this, the action will be ignored.</p>
-<p><b>You can have multiple actions for a single selector </b>&nbsp;(or
-more precisely a single filter of such a selector line). Each action
-must be on its own line and the line must start with an ampersand
-('&amp;') character and have no filters. An example would be</p>
-<p><code><b>*.=crit rger<br>
-&amp; root<br>
-&amp; /var/log/critmsgs</b></code></p>
-<p>These three lines send critical messages to the user rger and
-root and also store them in /var/log/critmsgs. <b>Using multiple
-actions per selector is</b> convenient and also <b>offers
-a performance benefit</b>. As the filter needs to be evaluated
-only once, there is less computation required to process the directive
-compared to the otherwise-equal config directives below:</p>
-<p><code><b>*.=crit rger<br>
-*.=crit root<br>
-*.=crit /var/log/critmsgs</b></code></p>
-<p>&nbsp;</p>
-<h3>Regular File</h3>
-<p>Typically messages are logged to real files. The file has to
-be specified with full pathname, beginning with a slash "/''.<br>
-<br>
-You may prefix each entry with the minus "-'' sign to omit syncing the
-file after every logging. Note that you might lose information if the
-system crashes right behind a write attempt. Nevertheless this might
-give you back some performance, especially if you run programs that use
-logging in a very verbose manner.</p>
-<p>If your system is connected to a reliable UPS and you receive
-lots of log data (e.g. firewall logs), it might be a very good idea to
-turn of
-syncing by specifying the "-" in front of the file name. </p>
-<p><b>The filename can be either static </b>(always
-the same) or <b>dynamic</b> (different based on message
-received). The later is useful if you would automatically split
-messages into different files based on some message criteria. For
-example, dynamic file name selectors allow you to split messages into
-different files based on the host that sent them. With dynamic file
-names, everything is automatic and you do not need any filters. </p>
-<p>It works via the template system. First, you define a template
-for the file name. An example can be seen above in the description of
-template. We will use the "DynFile" template defined there. Dynamic
-filenames are indicated by specifying a questions mark "?" instead of a
-slash, followed by the template name. Thus, the selector line for our
-dynamic file name would look as follows:</p>
-<blockquote>
-<code>*.* ?DynFile</code>
-</blockquote>
-<p>That's all you need to do. Rsyslog will now automatically
-generate file names for you and store the right messages into the right
-files. Please note that the minus sign also works with dynamic file
-name selectors. Thus, to avoid syncing, you may use</p>
-<blockquote>
-<code>*.* -?DynFile</code></blockquote>
-<p>And of course you can use templates to specify the output
-format:</p>
-<blockquote>
-<code>*.* ?DynFile;MyTemplate</code></blockquote>
-<p><b>A word of caution:</b> rsyslog creates files as
-needed. So if a new host is using your syslog server, rsyslog will
-automatically create a new file for it.</p>
-<p><b>Creating directories is also supported</b>. For
-example you can use the hostname as directory and the program name as
-file name:</p>
-<blockquote>
-<code>$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"</code></blockquote>
-<h3>Named Pipes</h3>
-<p>This version of rsyslogd(8) has support for logging output to
-named pipes (fifos). A fifo or named pipe can be used as a destination
-for log messages by prepending a pipe symbol ("|'') to the name of the
-file. This is handy for debugging. Note that the fifo must be created
-with the mkfifo(1) command before rsyslogd(8) is started.</p>
-<h3>Terminal and Console</h3>
-<p>If the file you specified is a tty, special tty-handling is
-done, same with /dev/console.</p>
-<h3>Remote Machine</h3>
-<p>Rsyslogd provides full remote logging, i.e. is able to send
-messages to a remote host running rsyslogd(8) and to receive messages
-from remote hosts. Using this feature you're able to control all syslog
-messages on one host, if all other machines will log remotely to that.
-This tears down<br>
-administration needs.<br>
-<br>
-<b>Please note that this version of rsyslogd by default does NOT
-forward messages it has received from the network to another host.
-Specify the "-h" option to enable this.</b></p>
-<p>To forward messages to another host, prepend the hostname with
-the at sign ("@"). A single at sign means that messages will
-be forwarded via UDP protocol (the standard for syslog). If you prepend
-two at signs ("@@"), the messages will be transmitted via TCP. Please
-note that plain TCP based syslog is not officially standardized, but
-most major syslogds support it (e.g. syslog-ng or WinSyslog). The
-forwarding action indicator (at-sign) can be followed by one or more
-options. If they are given, they must be immediately (without a space)
-following the final at sign and be enclosed in parenthesis. The
-individual options must be separated by commas. The following options
-are right now defined:</p>
-<table id="table2" border="1" width="100%">
-<tbody>
-<tr>
-<td>
-<p align="center"><b>z&lt;number&gt;</b></p>
-</td>
-<td>Enable zlib-compression for the message. The
-&lt;number&gt; is the compression level. It can be 1 (lowest
-gain, lowest CPU overhead) to 9 (maximum compression, highest CPU
-overhead). The level can also be 0, which means "no compression". If
-given, the "z" option is ignored. So this does not make an awful lot of
-sense. There is hardly a difference between level 1 and 9 for typical
-syslog messages. You can expect a compression gain between 0% and 30%
-for typical messages. Very chatty messages may compress up to 50%, but
-this is seldom seen with typically traffic. Please note that rsyslogd
-checks the compression gain. Messages with 60 bytes or less will never
-be compressed. This is because compression gain is pretty unlikely and
-we prefer to save CPU cycles. Messages over that size are always
-compressed. However, it is checked if there is a gain in compression
-and only if there is, the compressed message is transmitted. Otherwise,
-the uncompressed messages is transmitted. This saves the receiver CPU
-cycles for decompression. It also prevents small message to actually
-become larger in compressed form.
-<p><b>Please note that when a TCP transport is used,
-compression will also turn on syslog-transport-tls framing. See the "o"
-option for important information on the implications.</b></p>
-<p>Compressed messages are automatically detected and
-decompressed by the receiver. There is nothing that needs to be
-configured on the receiver side.</p>
-</td>
-</tr>
-<tr>
-<td>
-<p align="center"><b>o</b></p>
-</td>
-<td><b>This option is experimental. Use at your own
-risk and only if you know why you need it! If in doubt, do NOT turn it
-on.</b>
-<p>This option is only valid for plain TCP based
-transports. It selects a different framing based on IETF internet draft
-syslog-transport-tls-06. This framing offers some benefits over
-traditional LF-based framing. However, the standardization effort is
-not yet complete. There may be changes in upcoming versions of this
-standard. Rsyslog will be kept in line with the standard. There is some
-chance that upcoming changes will be incompatible to the current
-specification. In this case, all systems using -transport-tls framing
-must be upgraded. There will be no effort made to retain compatibility
-between different versions of rsyslog. The primary reason for that is
-that it seems technically impossible to provide compatibility between
-some of those changes. So you should take this note very serious. It is
-not something we do not *like* to do (and may change our mind if enough
-people beg...), it is something we most probably *can not* do for
-technical reasons (aka: you can beg as much as you like, it won't
-change anything...).</p>
-<p>The most important implication is that compressed syslog
-messages via TCP must be considered with care. Unfortunately, it is
-technically impossible to transfer compressed records over traditional
-syslog plain tcp transports, so you are left with two evil choices...</p>
-</td>
-</tr>
-</tbody>
-</table>
-<p><br>
-The hostname may be followed by a colon and the destination port.</p>
-<p>The following is an example selector line with forwarding:</p>
-<p>*.*&nbsp;&nbsp;&nbsp; @@(o,z9)192.168.0.1:1470</p>
-<p>In this example, messages are forwarded via plain TCP with
-experimental framing and maximum compression to the host 192.168.0.1 at
-port 1470.</p>
-<p>*.* @192.168.0.1</p>
-<p>In the example above, messages are forwarded via UDP to the
-machine 192.168.0.1, the destination port defaults to 514. Messages
-will not be compressed.</p>
-<p>Note that IPv6 addresses contain colons. So if an IPv6 address is specified
-in the hostname part, rsyslogd could not detect where the IP address ends
-and where the port starts. There is a syntax extension to support this:
-put squary brackets around the address (e.g. "[2001::1]"). Square
-brackets also work with real host names and IPv4 addresses, too.
-<p>A valid sample to send messages to the IPv6 host 2001::1 at port 515
-is as follows:
-<p>*.* @[2001::1]:515
-<p>This works with TCP, too.
-<p><b>Note to sysklogd users:</b> sysklogd does <b>not</b>
-support RFC 3164 format, which is the default forwarding template in
-rsyslog. As such, you will experience duplicate hostnames if rsyslog is
-the sender and sysklogd is the receiver. The fix is simple: you need to
-use a different template. Use that one:</p>
-<p class="MsoPlainText">$template
-sysklogd,"&lt;%PRI%&gt;%TIMESTAMP% %syslogtag%%msg%\""<br>
-*.* @192.168.0.1;sysklogd</p>
-<h3>List of Users</h3>
-<p>Usually critical messages are also directed to "root'' on
-that machine. You can specify a list of users that shall get the
-message by simply writing the login. You may specify more than one user
-by separating them with commas (",''). If they're logged in they get
-the message. Don't think a mail would be sent, that might be too late.</p>
-<h3>Everyone logged on</h3>
-<p>Emergency messages often go to all users currently online to
-notify them that something strange is happening with the system. To
-specify this wall(1)-feature use an asterisk ("*'').</p>
-<h3>Call Plugin</h3>
-<p>This is a generic way to call an output plugin. The plugin
-must support this functionality. Actual parameters depend on the
-module, so see the module's doc on what to supply. The general syntax
-is as follows:</p>
-<p>:modname:params;template</p>
-<p>Currently, the ommysql database output module supports this
-syntax (in addtion to the "&gt;" syntax it traditionally
-supported). For ommysql, the module name is "ommysql" and the params
-are the traditional ones. The ;template part is not module specific, it
-is generic rsyslog functionality available to all modules.</p>
-<p>As an example, the ommysql module may be called as follows:</p>
-<p>:ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate</p>
-<p>For details, please see the "Database Table" section of this
-documentation.</p>
-<p>Note: as of this writing, the ":modname:" part is hardcoded
-into the module. So the name to use is not necessarily the name the
-module's plugin file is called.</p>
-<h3>Database Table</h3>
-<p>This allows logging of the message to a database table.
-Currently, only MySQL databases are supported. However, other database
-drivers will most probably be developed as plugins. By default, a <a href="http://www.monitorware.com/">MonitorWare</a>-compatible
-schema is required for this to work. You can create that schema with
-the createDB.SQL file that came with the rsyslog package. You can also<br>
-use any other schema of your liking - you just need to define a proper
-template and assign this template to the action.<br>
-<br>
-The database writer is called by specifying a greater-then sign
-("&gt;") in front of the database connect information. Immediately
-after that<br>
-sign the database host name must be given, a comma, the database name,
-another comma, the database user, a comma and then the user's password.
-If a specific template is to be used, a semicolon followed by the
-template name can follow the connect information. This is as follows:<br>
-<br>
-&gt;dbhost,dbname,dbuser,dbpassword;dbtemplate</p>
-<p><b>Important: to use the database functionality, the
-MySQL output module must be loaded in the config file</b> BEFORE
-the first database table action is used. This is done by placing the</p>
-<p><code><b>$ModLoad ommysql</b></code></p>
-<p>directive some place above the first use of the database write
-(we recommend doing at the the beginning of the config file).</p>
-<h3>Discard</h3>
-<p>If the discard action is carried out, the received message is
-immediately discarded. No further processing of it occurs. Discard has
-primarily been added to filter out messages before carrying on any
-further processing. For obvious reasons, the results of "discard" are
-depending on where in the configuration file it is being used. Please
-note that once a message has been discarded there is no way to retrieve
-it in later configuration file lines.</p>
-<p>Discard can be highly effective if you want to filter out some
-annoying messages that otherwise would fill your log files. To do that,
-place the discard actions early in your log files. This often plays
-well with property-based filters, giving you great freedom in
-specifying what you do not want.</p>
-<p>Discard is just the single tilde character with no further
-parameters:</p>
-<p>~</p>
-<p>For example,</p>
-<p>*.*&nbsp;&nbsp; ~</p>
-<p>discards everything (ok, you can achive the same by not
-running rsyslogd at all...).</p>
-<h3>Output Channel</h3>
-<p>Binds an output channel definition (see there for details) to
-this action. Output channel actions must start with a $-sign, e.g. if
-you would like to bind your output channel definition "mychannel" to
-the action, use "$mychannel". Output channels support template
-definitions like all all other actions.</p>
-<h3>Shell Execute</h3>
-<p>This executes a program in a subshell. The program is passed
-the template-generated message as the only command line parameter.
-Rsyslog waits until the program terminates and only then continues to
-run.</p>
-<p>^program-to-execute;template</p>
-<p>The program-to-execute can be any valid executable. It
-receives the template string as a single parameter (argv[1]).</p>
-<p><b>WARNING:</b> The Shell Execute action was added
-to serve an urgent need. While it is considered reasonable save when
-used with some thinking, its implications must be considered. The
-current implementation uses a system() call to execute the command.
-This is not the best way to do it (and will hopefully changed in
-further releases). Also, proper escaping of special characters is done
-to prevent command injection. However, attackers always find smart ways
-to circumvent escaping, so we can not say if the escaping applied will
-really safe you from all hassles. Lastly, rsyslog will wait until the
-shell command terminates. Thus, a program error in it (e.g. an infinite
-loop) can actually disable rsyslog. Even without that, during the
-programs run-time no messages are processed by rsyslog. As the IP
-stacks buffers are quickly overflowed, this bears an increased risk of
-message loss. You must be aware of these implications. Even though they
-are severe, there are several cases where the "shell execute" action is
-very useful. This is the reason why we have included it in its current
-form. To mitigate its risks, always a) test your program thoroughly, b)
-make sure its runtime is as short as possible (if it requires a longer
-run-time, you might want to spawn your own sub-shell asynchronously),
-c) apply proper firewalling so that only known senders can send syslog
-messages to rsyslog. Point c) is especially important: if rsyslog is
-accepting message from any hosts, chances are much higher that an
-attacker might try to exploit the "shell execute" action.</p>
-<h2>TEMPLATE NAME</h2>
-<p>Every ACTION can be followed by a template name. If so, that
-template is used for message formatting. If no name is given, a
-hard-coded default template is used for the action. There can only be
-one template name for each given action. The default template is
-specific to each action. For a description of what a template is and
-what you can do with it, see "TEMPLATES" at the top of this document.</p>
-<h2>EXAMPLES</h2>
-<p>Below are example for templates and selector lines. I hope
+<h2><a href="rsyslog_conf_templates.html">Templates</a></h2>
+<h2><a href="rsyslog_conf_output.html">Output Channels</a></h2>
+<h2><a href="rsyslog_conf_filter.html">Filter Conditions</a></h2>
+<h2><a href="rsyslog_conf_actions.html">Actions</a></h2>
+<h2><a href="rsyslog_conf_examples.html">Examples</a></h2>
+<p>Here you will find examples for templates and selector lines. I hope
 they are self-explanatory. If not, please see
 www.monitorware.com/rsyslog/ for advise.</p>
-<h3>TEMPLATES</h3>
-<p>Please note that the samples are split across multiple lines.
-A template MUST NOT actually be split across multiple lines.<br>
-<br>
-A template that resembles traditional syslogd file output:<br>
-$template TraditionalFormat,"%timegenerated% %HOSTNAME%<br>
-%syslogtag%%msg:::drop-last-lf%\n"<br>
-<br>
-A template that tells you a little more about the message:<br>
-$template
-precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,<br>
-%syslogtag%,%msg%\n"<br>
-<br>
-A template for RFC 3164 format:<br>
-$template RFC3164fmt,"&lt;%PRI%&gt;%TIMESTAMP% %HOSTNAME%
-%syslogtag%%msg%"<br>
-<br>
-A template for the format traditonally used for user messages:<br>
-$template usermsg," XXXX%syslogtag%%msg%\n\r"<br>
-<br>
-And a template with the traditonal wall-message format:<br>
-$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at
-%timegenerated%<br>
-<br>
-A template that can be used for the database write (please note the SQL<br>
-template option)<br>
-$template MySQLInsert,"insert iut, message, receivedat values<br>
-('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')<br>
-into systemevents\r\n", SQL<br>
-<br>
-The following template emulates <a href="http://www.winsyslog.com/en/">WinSyslog</a>
-format (it's an <a href="http://www.adiscon.com/en/">Adiscon</a>
-format, you do not feel bad if you don't know it ;)). It's interesting
-to see how it takes different parts out of the date stamps. What
-happens is that the date stamp is split into the actual date and time
-and the these two are combined with just a comma in between them.<br>
-<br>
-$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,<br>
-%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,<br>
-%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,<br>
-%syslogtag%%msg%\n"</p>
-<h3>SELECTOR LINES</h3>
-<p># Store critical stuff in critical<br>
-#<br>
-*.=crit;kern.none /var/adm/critical<br>
-<br>
-This will store all messages with the priority crit in the file
-/var/adm/critical, except for any kernel message.<br>
-<br>
-<br>
-# Kernel messages are first, stored in the kernel<br>
-# file, critical messages and higher ones also go<br>
-# to another host and to the console. Messages to<br>
-# the host finlandia are forwarded in RFC 3164<br>
-# format (using the template defined above).<br>
-#<br>
-kern.* /var/adm/kernel<br>
-kern.crit @finlandia;RFC3164fmt<br>
-kern.crit /dev/console<br>
-kern.info;kern.!err /var/adm/kernel-info<br>
-<br>
-The first rule direct any message that has the kernel facility to the
-file /var/adm/kernel.<br>
-<br>
-The second statement directs all kernel messages of the priority crit
-and higher to the remote host finlandia. This is useful, because if the
-host crashes and the disks get irreparable errors you might not be able
-to read the stored messages. If they're on a remote host, too, you
-still can try to find out the reason for the crash.<br>
-<br>
-The third rule directs these messages to the actual console, so the
-person who works on the machine will get them, too.<br>
-<br>
-The fourth line tells rsyslogd to save all kernel messages that come
-with priorities from info up to warning in the file
-/var/adm/kernel-info. Everything from err and higher is excluded.<br>
-<br>
-<br>
-# The tcp wrapper loggs with mail.info, we display<br>
-# all the connections on tty12<br>
-#<br>
-mail.=info /dev/tty12<br>
-<br>
-This directs all messages that uses mail.info (in source LOG_MAIL |
-LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper
-tcpd(8) uses this as it's default.<br>
-<br>
-<br>
-# Store all mail concerning stuff in a file<br>
-#<br>
-mail.*;mail.!=info /var/adm/mail<br>
-<br>
-This pattern matches all messages that come with the mail facility,
-except for the info priority. These will be stored in the file
-/var/adm/mail.<br>
-<br>
-<br>
-# Log all mail.info and news.info messages to info<br>
-#<br>
-mail,news.=info /var/adm/info<br>
-<br>
-This will extract all messages that come either with mail.info or with
-news.info and store them in the file /var/adm/info.<br>
-<br>
-<br>
-# Log info and notice messages to messages file<br>
-#<br>
-*.=info;*.=notice;\<br>
-mail.none /var/log/messages<br>
-<br>
-This lets rsyslogd log all messages that come with either the info or
-the notice facility into the file /var/log/messages, except for all<br>
-messages that use the mail facility.<br>
-<br>
-<br>
-# Log info messages to messages file<br>
-#<br>
-*.=info;\<br>
-mail,news.none /var/log/messages<br>
-<br>
-This statement causes rsyslogd to log all messages that come with the
-info priority to the file /var/log/messages. But any message coming
-either with the mail or the news facility will not be stored.<br>
-<br>
-<br>
-# Emergency messages will be displayed using wall<br>
-#<br>
-*.=emerg *<br>
-<br>
-This rule tells rsyslogd to write all emergency messages to all
-currently logged in users. This is the wall action.<br>
-<br>
-<br>
-# Messages of the priority alert will be directed<br>
-# to the operator<br>
-#<br>
-*.alert root,rgerhards<br>
-<br>
-This rule directs all messages with a priority of alert or higher to
-the terminals of the operator, i.e. of the users "root'' and
-"rgerhards'' if they're logged in.<br>
-<br>
-<br>
-*.* @finlandia<br>
-<br>
-This rule would redirect all messages to a remote host called
-finlandia. This is useful especially in a cluster of machines where all
-syslog messages will be stored on only one machine.<br>
-<br>
-In the format shown above, UDP is used for transmitting the message.
-The destination port is set to the default auf 514. Rsyslog is also
-capable of using much more secure and reliable TCP sessions for message
-forwarding. Also, the destination port can be specified. To select TCP,
-simply add one additional @ in front of the host name (that is, @host
-is UPD, @@host is TCP). For example:<br>
-<br>
-<br>
-*.* @@finlandia<br>
-<br>
-To specify the destination port on the remote machine, use a colon
-followed by the port number after the machine name. The following
-forwards to port 1514 on finlandia:<br>
-<br>
-<br>
-*.* @@finlandia:1514<br>
-<br>
-This syntax works both with TCP and UDP based syslog. However, you will
-probably primarily need it for TCP, as there is no well-accepted port
-for this transport (it is non-standard). For UDP, you can usually stick
-with the default auf 514, but might want to modify it for security rea-<br>
-sons. If you would like to do that, it's quite easy:<br>
-<br>
-<br>
-*.* @finlandia:1514<br>
-<br>
-<br>
-<br>
-*.* &gt;dbhost,dbname,dbuser,dbpassword;dbtemplate<br>
-<br>
-This rule writes all message to the database "dbname" hosted on
-"dbhost". The login is done with user "dbuser" and password
-"dbpassword". The actual table that is updated is specified within the
-template (which contains the insert statement). The template is called
-"dbtemplate" in this case.</p>
-<p>:msg,contains,"error" @errorServer</p>
-<p>This rule forwards all messages that contain the word "error"
-in the msg part to the server "errorServer". Forwarding is via UDP.
-Please note the colon in fron</p>
-<h2>CONFIGURATION FILE SYNTAX DIFFERENCES</h2>
+<h2>Configuration File Syntax Differences</h2>
 <p>Rsyslogd uses a slightly different syntax for its
 configuration file than the original BSD sources. Originally all
 messages of a specific priority and above were forwarded to the log
@@ -1272,4 +68,15 @@ additional
 <a href="features.html">features</a> (like template
 and database support). For obvious reasons, the syntax for defining
 such features is available in rsyslogd, only.</p>
-</body></html>
+
+<p>[<a href="rsyslog_conf.html">back to top</a>]
+[<a href="manual.html">manual index</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+>
diff --git a/doc/rsyslog_high_database_rate.html b/doc/rsyslog_high_database_rate.html
index 158a4df6..2bae58c6 100644
--- a/doc/rsyslog_high_database_rate.html
+++ b/doc/rsyslog_high_database_rate.html
@@ -7,6 +7,7 @@
 </head>
 
 <body>
+<a href="features.html">back</a>
 
 <h1>Handling a massive syslog database insert rate with Rsyslog</h1>
 
@@ -171,6 +172,14 @@ comments or find bugs (I *do* bugs - no way... ;)), please
 http://www.gnu.org/copyleft/fdl.html</a>.</p>
 
 
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
 
 </body>
 
diff --git a/doc/rsyslog_mysql.html b/doc/rsyslog_mysql.html
index 753c86ec..a27bd59e 100644
--- a/doc/rsyslog_mysql.html
+++ b/doc/rsyslog_mysql.html
@@ -1,6 +1,6 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head><title>Writing syslog Data to MySQL</title>
-
+<a href="features.html">back</a>
 <meta name="KEYWORDS" content="syslog, mysql, syslog to mysql, howto"></head>
 <body>
 <h1>Writing syslog messages to MySQL</h1>
@@ -259,4 +259,13 @@ document under the terms of the GNU Free Documentation License, Version
 with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
 Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html">
 http://www.gnu.org/copyleft/fdl.html</a>.</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+
 </body></html>
diff --git a/doc/rsyslog_ng_comparison.html b/doc/rsyslog_ng_comparison.html
index 2f383f78..8e121a8d 100644
--- a/doc/rsyslog_ng_comparison.html
+++ b/doc/rsyslog_ng_comparison.html
@@ -1,6 +1,7 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head><title>rsyslog vs. syslog-ng - a comparison</title></head>
 <body>
+<a href="features.html">back</a>
 <h1>rsyslog vs. syslog-ng</h1>
 <p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a>
 (2008-05-06)</i></small></p>
@@ -584,4 +585,13 @@ feature sheet. I have not yet been able to fully work through it. In
 the mean time, you may want to read it in parallel. It is available at
 <a href="http://www.balabit.com/network-security/syslog-ng/features/detailed/">Balabit's
 site</a>.</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+
 </body></html>
diff --git a/doc/rsyslog_stunnel.html b/doc/rsyslog_stunnel.html
index 104a672e..f4f82cd0 100644
--- a/doc/rsyslog_stunnel.html
+++ b/doc/rsyslog_stunnel.html
@@ -1,5 +1,6 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head>
+<a href="features.html">back</a>
 
 <title>SSL Encrypting syslog with stunnel</title><meta name="KEYWORDS" content="syslog encryption, rsyslog, stunnel, secure syslog, tcp, reliable, howto, ssl"></head><body>
 <h1>SSL Encrypting Syslog with Stunnel</h1>
@@ -236,5 +237,13 @@ comments or find bugs (I *do* bugs - no way... ;)), please
 	 Texts.  A copy of the license can be viewed at
 <a href="http://www.gnu.org/copyleft/fdl.html">
 http://www.gnu.org/copyleft/fdl.html</a>.</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
 
-</body></html>
\ No newline at end of file
+</body></html>
diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html
index 7d156c3a..ebb08ebe 100644
--- a/doc/rsyslog_tls.html
+++ b/doc/rsyslog_tls.html
@@ -1,5 +1,6 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
 <html><head><title>TLS (SSL) Encrypting syslog</title>
+<a href="features.html">back</a>
 
 <meta name="KEYWORDS" content="syslog encryption, rsyslog, secure syslog, tcp, reliable, howto, ssl, tls">
 </head>
@@ -304,4 +305,13 @@ document under the terms of the GNU Free Documentation License, Version
 with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
 Texts. A copy of the license can be viewed at
 <a href="http://www.gnu.org/copyleft/fdl.html">http://www.gnu.org/copyleft/fdl.html</a>.</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+
 </body></html>
diff --git a/doc/syslog_protocol.html b/doc/syslog_protocol.html
index 72de5c27..57eb9ffe 100644
--- a/doc/syslog_protocol.html
+++ b/doc/syslog_protocol.html
@@ -3,6 +3,7 @@
 <title>syslog-protocol support in rsyslog</title>
 </head>
 <body>
+<a href="features.html">back</a>
 <h1>syslog-protocol support in rsyslog</h1>
 <p><b><a href="http://www.rsyslog.com/">Rsyslog</a>&nbsp; provides a trial 
 implementation of the proposed
@@ -191,6 +192,14 @@ discussed ;)</p>
 	syslog-protocol should be further evaluated and be fully understood</li>
 </ul>
 <p>&nbsp;</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
 </body>
 </html>
 
-- 
cgit 


From 3c615c60beb851f3a42cea3fcc31f4a2243cedaa Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Fri, 7 Nov 2008 14:49:24 +0100
Subject: doc: added link to regular expression checker online tool

---
 doc/manual.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual.html b/doc/manual.html
index f1b7d657..1d09d460 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -33,6 +33,7 @@ the links below for the</b><br></p><ul>
 
 <li><a href="troubleshoot.html">troubleshooting rsyslog problems</a></li>
 <li><a href="rsyslog_conf.html">configuration file syntax (rsyslog.conf)</a></li>
+<li><a href="/tool-regex">a regular expression checker/generator tool for rsyslog</a></li>
 <li> <a href="property_replacer.html">property replacer, an important core component</a></li>
 <li>a commented <a href="sample.conf.html">sample rsyslog.conf</a></li>
 <li><a href="bugs.html">rsyslog bug list</a></li>
-- 
cgit 


From 170d0d6f375241e0d0ca85a1327df82165fec439 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 11 Nov 2008 11:00:37 +0100
Subject: added forgotten files

they were new after restructuring the doc...
---
 doc/Makefile.am                 |   7 +
 doc/rsyslog_conf_actions.html   | 336 ++++++++++++++++++++++++++++++++++++++++
 doc/rsyslog_conf_examples.html  | 209 +++++++++++++++++++++++++
 doc/rsyslog_conf_filter.html    | 280 +++++++++++++++++++++++++++++++++
 doc/rsyslog_conf_global.html    | 227 +++++++++++++++++++++++++++
 doc/rsyslog_conf_modules.html   |  53 +++++++
 doc/rsyslog_conf_output.html    |  81 ++++++++++
 doc/rsyslog_conf_templates.html | 150 ++++++++++++++++++
 8 files changed, 1343 insertions(+)
 create mode 100644 doc/rsyslog_conf_actions.html
 create mode 100644 doc/rsyslog_conf_examples.html
 create mode 100644 doc/rsyslog_conf_filter.html
 create mode 100644 doc/rsyslog_conf_global.html
 create mode 100644 doc/rsyslog_conf_modules.html
 create mode 100644 doc/rsyslog_conf_output.html
 create mode 100644 doc/rsyslog_conf_templates.html

diff --git a/doc/Makefile.am b/doc/Makefile.am
index 22d368e0..fef1e44c 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -98,6 +98,13 @@ html_files = \
 	omrelp.html \
 	status.html \
 	troubleshoot.html \
+	rsyslog_conf_actions.html \
+	rsyslog_conf_examples.html \
+	rsyslog_conf_filter.html \
+	rsyslog_conf_global.html \
+	rsyslog_conf_modules.html \
+	rsyslog_conf_output.html \
+	rsyslog_conf_templates.html \
 	src/classes.dia
 
 EXTRA_DIST = $(html_files)
diff --git a/doc/rsyslog_conf_actions.html b/doc/rsyslog_conf_actions.html
new file mode 100644
index 00000000..2ef3f4b0
--- /dev/null
+++ b/doc/rsyslog_conf_actions.html
@@ -0,0 +1,336 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Actions - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Actions</h2>
+<p>The action field of a rule describes what to do with the
+message. In general, message content is written to a kind of "logfile".
+But also other actions might be done, like writing to a database table
+or forwarding to another host.<br>
+<br>
+Templates can be used with all actions. If used, the specified template
+is used to generate the message content (instead of the default
+template). To specify a template, write a semicolon after the action
+value immediately followed by the template name.<br>
+<br>
+Beware: templates MUST be defined BEFORE they are used. It is OK to
+define some templates, then use them in selector lines, define more
+templates and use use them in the following selector lines. But it is
+NOT permitted to use a template in a selector line that is above its
+definition. If you do this, the action will be ignored.</p>
+<p><b>You can have multiple actions for a single selector </b>&nbsp;(or
+more precisely a single filter of such a selector line). Each action
+must be on its own line and the line must start with an ampersand
+('&amp;') character and have no filters. An example would be</p>
+<p><code><b>*.=crit rger<br>
+&amp; root<br>
+&amp; /var/log/critmsgs</b></code></p>
+<p>These three lines send critical messages to the user rger and
+root and also store them in /var/log/critmsgs. <b>Using multiple
+actions per selector is</b> convenient and also <b>offers
+a performance benefit</b>. As the filter needs to be evaluated
+only once, there is less computation required to process the directive
+compared to the otherwise-equal config directives below:</p>
+<p><code><b>*.=crit rger<br>
+*.=crit root<br>
+*.=crit /var/log/critmsgs</b></code></p>
+<p>&nbsp;</p>
+<h3>Regular File</h3>
+<p>Typically messages are logged to real files. The file has to
+be specified with full pathname, beginning with a slash "/''.<br>
+<br>
+<br>
+You may prefix each entry with the minus "-'' sign to omit syncing the
+file after every logging. Note that you might lose information if the
+system crashes right behind a write attempt. Nevertheless this might
+give you back some performance, especially if you run programs that use
+logging in a very verbose manner.</p>
+<p>If your system is connected to a reliable UPS and you receive
+lots of log data (e.g. firewall logs), it might be a very good idea to
+turn of
+syncing by specifying the "-" in front of the file name. </p>
+<p><b>The filename can be either static </b>(always
+the same) or <b>dynamic</b> (different based on message
+received). The later is useful if you would automatically split
+messages into different files based on some message criteria. For
+example, dynamic file name selectors allow you to split messages into
+different files based on the host that sent them. With dynamic file
+names, everything is automatic and you do not need any filters. </p>
+<p>It works via the template system. First, you define a template
+for the file name. An example can be seen above in the description of
+template. We will use the "DynFile" template defined there. Dynamic
+filenames are indicated by specifying a questions mark "?" instead of a
+slash, followed by the template name. Thus, the selector line for our
+dynamic file name would look as follows:</p>
+<blockquote>
+<code>*.* ?DynFile</code>
+</blockquote>
+<p>That's all you need to do. Rsyslog will now automatically
+generate file names for you and store the right messages into the right
+files. Please note that the minus sign also works with dynamic file
+name selectors. Thus, to avoid syncing, you may use</p>
+<blockquote>
+<code>*.* -?DynFile</code></blockquote>
+<p>And of course you can use templates to specify the output
+format:</p>
+<blockquote>
+<code>*.* ?DynFile;MyTemplate</code></blockquote>
+<p><b>A word of caution:</b> rsyslog creates files as
+needed. So if a new host is using your syslog server, rsyslog will
+automatically create a new file for it.</p>
+<p><b>Creating directories is also supported</b>. For
+example you can use the hostname as directory and the program name as
+file name:</p>
+<blockquote>
+<code>$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"</code></blockquote>
+<h3>Named Pipes</h3>
+<p>This version of rsyslogd(8) has support for logging output to
+named pipes (fifos). A fifo or named pipe can be used as a destination
+for log messages by prepending a pipe symbol ("|'') to the name of the
+file. This is handy for debugging. Note that the fifo must be created
+with the mkfifo(1) command before rsyslogd(8) is started.</p>
+<h3>Terminal and Console</h3>
+<p>If the file you specified is a tty, special tty-handling is
+done, same with /dev/console.</p>
+<h3>Remote Machine</h3>
+<p>Rsyslogd provides full remote logging, i.e. is able to send
+messages to a remote host running rsyslogd(8) and to receive messages
+from remote hosts. Using this feature you're able to control all syslog
+messages on one host, if all other machines will log remotely to that.
+This tears down<br>
+administration needs.<br>
+<br>
+<b>Please note that this version of rsyslogd by default does NOT
+forward messages it has received from the network to another host.
+Specify the "-h" option to enable this.</b></p>
+<p>To forward messages to another host, prepend the hostname with
+the at sign ("@"). A single at sign means that messages will
+be forwarded via UDP protocol (the standard for syslog). If you prepend
+two at signs ("@@"), the messages will be transmitted via TCP. Please
+note that plain TCP based syslog is not officially standardized, but
+most major syslogds support it (e.g. syslog-ng or WinSyslog). The
+forwarding action indicator (at-sign) can be followed by one or more
+options. If they are given, they must be immediately (without a space)
+following the final at sign and be enclosed in parenthesis. The
+individual options must be separated by commas. The following options
+are right now defined:</p>
+<table id="table2" border="1" width="100%">
+<tbody>
+<tr>
+<td>
+<p align="center"><b>z&lt;number&gt;</b></p>
+</td>
+<td>Enable zlib-compression for the message. The
+&lt;number&gt; is the compression level. It can be 1 (lowest
+gain, lowest CPU overhead) to 9 (maximum compression, highest CPU
+overhead). The level can also be 0, which means "no compression". If
+given, the "z" option is ignored. So this does not make an awful lot of
+sense. There is hardly a difference between level 1 and 9 for typical
+syslog messages. You can expect a compression gain between 0% and 30%
+for typical messages. Very chatty messages may compress up to 50%, but
+this is seldom seen with typically traffic. Please note that rsyslogd
+checks the compression gain. Messages with 60 bytes or less will never
+be compressed. This is because compression gain is pretty unlikely and
+we prefer to save CPU cycles. Messages over that size are always
+compressed. However, it is checked if there is a gain in compression
+and only if there is, the compressed message is transmitted. Otherwise,
+the uncompressed messages is transmitted. This saves the receiver CPU
+cycles for decompression. It also prevents small message to actually
+become larger in compressed form.
+<p><b>Please note that when a TCP transport is used,
+compression will also turn on syslog-transport-tls framing. See the "o"
+option for important information on the implications.</b></p>
+<p>Compressed messages are automatically detected and
+decompressed by the receiver. There is nothing that needs to be
+configured on the receiver side.</p>
+</td>
+</tr>
+<tr>
+<td>
+<p align="center"><b>o</b></p>
+</td>
+<td><b>This option is experimental. Use at your own
+risk and only if you know why you need it! If in doubt, do NOT turn it
+on.</b>
+<p>This option is only valid for plain TCP based
+transports. It selects a different framing based on IETF internet draft
+syslog-transport-tls-06. This framing offers some benefits over
+traditional LF-based framing. However, the standardization effort is
+not yet complete. There may be changes in upcoming versions of this
+standard. Rsyslog will be kept in line with the standard. There is some
+chance that upcoming changes will be incompatible to the current
+specification. In this case, all systems using -transport-tls framing
+must be upgraded. There will be no effort made to retain compatibility
+between different versions of rsyslog. The primary reason for that is
+that it seems technically impossible to provide compatibility between
+some of those changes. So you should take this note very serious. It is
+not something we do not *like* to do (and may change our mind if enough
+people beg...), it is something we most probably *can not* do for
+technical reasons (aka: you can beg as much as you like, it won't
+change anything...).</p>
+<p>The most important implication is that compressed syslog
+messages via TCP must be considered with care. Unfortunately, it is
+technically impossible to transfer compressed records over traditional
+syslog plain tcp transports, so you are left with two evil choices...</p>
+</td>
+</tr>
+</tbody>
+</table>
+<p><br>
+The hostname may be followed by a colon and the destination port.</p>
+<p>The following is an example selector line with forwarding:</p>
+<p>*.*&nbsp;&nbsp;&nbsp; @@(o,z9)192.168.0.1:1470</p>
+<p>In this example, messages are forwarded via plain TCP with
+experimental framing and maximum compression to the host 192.168.0.1 at
+port 1470.</p>
+<p>*.* @192.168.0.1</p>
+<p>In the example above, messages are forwarded via UDP to the
+machine 192.168.0.1, the destination port defaults to 514. Messages
+will not be compressed.</p>
+<p>Note that IPv6 addresses contain colons. So if an IPv6 address is specified
+in the hostname part, rsyslogd could not detect where the IP address ends
+and where the port starts. There is a syntax extension to support this:
+put squary brackets around the address (e.g. "[2001::1]"). Square
+brackets also work with real host names and IPv4 addresses, too.
+<p>A valid sample to send messages to the IPv6 host 2001::1 at port 515
+is as follows:
+<p>*.* @[2001::1]:515
+<p>This works with TCP, too.
+<p><b>Note to sysklogd users:</b> sysklogd does <b>not</b>
+support RFC 3164 format, which is the default forwarding template in
+rsyslog. As such, you will experience duplicate hostnames if rsyslog is
+the sender and sysklogd is the receiver. The fix is simple: you need to
+use a different template. Use that one:</p>
+<p class="MsoPlainText">$template
+sysklogd,"&lt;%PRI%&gt;%TIMESTAMP% %syslogtag%%msg%\""<br>
+*.* @192.168.0.1;sysklogd</p>
+<h3>List of Users</h3>
+<p>Usually critical messages are also directed to "root'' on
+that machine. You can specify a list of users that shall get the
+message by simply writing the login. You may specify more than one user
+by separating them with commas (",''). If they're logged in they get
+the message. Don't think a mail would be sent, that might be too late.</p>
+<h3>Everyone logged on</h3>
+<p>Emergency messages often go to all users currently online to
+notify them that something strange is happening with the system. To
+specify this wall(1)-feature use an asterisk ("*'').</p>
+<h3>Call Plugin</h3>
+<p>This is a generic way to call an output plugin. The plugin
+must support this functionality. Actual parameters depend on the
+module, so see the module's doc on what to supply. The general syntax
+is as follows:</p>
+<p>:modname:params;template</p>
+<p>Currently, the ommysql database output module supports this
+syntax (in addtion to the "&gt;" syntax it traditionally
+supported). For ommysql, the module name is "ommysql" and the params
+are the traditional ones. The ;template part is not module specific, it
+is generic rsyslog functionality available to all modules.</p>
+<p>As an example, the ommysql module may be called as follows:</p>
+<p>:ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate</p>
+<p>For details, please see the "Database Table" section of this
+documentation.</p>
+<p>Note: as of this writing, the ":modname:" part is hardcoded
+into the module. So the name to use is not necessarily the name the
+module's plugin file is called.</p>
+<h3>Database Table</h3>
+<p>This allows logging of the message to a database table.
+Currently, only MySQL databases are supported. However, other database
+drivers will most probably be developed as plugins. By default, a <a href="http://www.monitorware.com/">MonitorWare</a>-compatible
+schema is required for this to work. You can create that schema with
+the createDB.SQL file that came with the rsyslog package. You can also<br>
+use any other schema of your liking - you just need to define a proper
+template and assign this template to the action.<br>
+<br>
+The database writer is called by specifying a greater-then sign
+("&gt;") in front of the database connect information. Immediately
+after that<br>
+sign the database host name must be given, a comma, the database name,
+another comma, the database user, a comma and then the user's password.
+If a specific template is to be used, a semicolon followed by the
+template name can follow the connect information. This is as follows:<br>
+<br>
+&gt;dbhost,dbname,dbuser,dbpassword;dbtemplate</p>
+<p><b>Important: to use the database functionality, the
+MySQL output module must be loaded in the config file</b> BEFORE
+the first database table action is used. This is done by placing the</p>
+<p><code><b>$ModLoad ommysql</b></code></p>
+<p>directive some place above the first use of the database write
+(we recommend doing at the the beginning of the config file).</p>
+<h3>Discard</h3>
+<p>If the discard action is carried out, the received message is
+immediately discarded. No further processing of it occurs. Discard has
+primarily been added to filter out messages before carrying on any
+further processing. For obvious reasons, the results of "discard" are
+depending on where in the configuration file it is being used. Please
+note that once a message has been discarded there is no way to retrieve
+it in later configuration file lines.</p>
+<p>Discard can be highly effective if you want to filter out some
+annoying messages that otherwise would fill your log files. To do that,
+place the discard actions early in your log files. This often plays
+well with property-based filters, giving you great freedom in
+specifying what you do not want.</p>
+<p>Discard is just the single tilde character with no further
+parameters:</p>
+<p>~</p>
+<p>For example,</p>
+<p>*.*&nbsp;&nbsp; ~</p>
+<p>discards everything (ok, you can achive the same by not
+running rsyslogd at all...).</p>
+<h3>Output Channel</h3>
+<p>Binds an output channel definition (see there for details) to
+this action. Output channel actions must start with a $-sign, e.g. if
+you would like to bind your output channel definition "mychannel" to
+the action, use "$mychannel". Output channels support template
+definitions like all all other actions.</p>
+<h3>Shell Execute</h3>
+<p>This executes a program in a subshell. The program is passed
+the template-generated message as the only command line parameter.
+Rsyslog waits until the program terminates and only then continues to
+run.</p>
+<p>^program-to-execute;template</p>
+<p>The program-to-execute can be any valid executable. It
+receives the template string as a single parameter (argv[1]).</p>
+<p><b>WARNING:</b> The Shell Execute action was added
+to serve an urgent need. While it is considered reasonable save when
+used with some thinking, its implications must be considered. The
+current implementation uses a system() call to execute the command.
+This is not the best way to do it (and will hopefully changed in
+further releases). Also, proper escaping of special characters is done
+to prevent command injection. However, attackers always find smart ways
+to circumvent escaping, so we can not say if the escaping applied will
+really safe you from all hassles. Lastly, rsyslog will wait until the
+shell command terminates. Thus, a program error in it (e.g. an infinite
+loop) can actually disable rsyslog. Even without that, during the
+programs run-time no messages are processed by rsyslog. As the IP
+stacks buffers are quickly overflowed, this bears an increased risk of
+message loss. You must be aware of these implications. Even though they
+are severe, there are several cases where the "shell execute" action is
+very useful. This is the reason why we have included it in its current
+form. To mitigate its risks, always a) test your program thoroughly, b)
+make sure its runtime is as short as possible (if it requires a longer
+run-time, you might want to spawn your own sub-shell asynchronously),
+c) apply proper firewalling so that only known senders can send syslog
+messages to rsyslog. Point c) is especially important: if rsyslog is
+accepting message from any hosts, chances are much higher that an
+attacker might try to exploit the "shell execute" action.</p>
+<h3>Template Name</h3>
+<p>Every ACTION can be followed by a template name. If so, that
+template is used for message formatting. If no name is given, a
+hard-coded default template is used for the action. There can only be
+one template name for each given action. The default template is
+specific to each action. For a description of what a template is and
+what you can do with it, see "TEMPLATES" at the top of this document.</p>
+
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
diff --git a/doc/rsyslog_conf_examples.html b/doc/rsyslog_conf_examples.html
new file mode 100644
index 00000000..b46460e5
--- /dev/null
+++ b/doc/rsyslog_conf_examples.html
@@ -0,0 +1,209 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Examples - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Examples</h2>
+<p>Below are example for templates and selector lines. I hope
+they are self-explanatory. If not, please see
+www.monitorware.com/rsyslog/ for advise.</p>
+<h3>TEMPLATES</h3>
+<p>Please note that the samples are split across multiple lines.
+A template MUST NOT actually be split across multiple lines.<br>
+<br>
+A template that resembles traditional syslogd file output:<br>
+$template TraditionalFormat,"%timegenerated% %HOSTNAME%<br>
+%syslogtag%%msg:::drop-last-lf%\n"<br>
+<br>
+A template that tells you a little more about the message:<br>
+$template
+precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,<br>
+%syslogtag%,%msg%\n"<br>
+<br>
+A template for RFC 3164 format:<br>
+$template RFC3164fmt,"&lt;%PRI%&gt;%TIMESTAMP% %HOSTNAME%
+%syslogtag%%msg%"<br>
+<br>
+A template for the format traditonally used for user messages:<br>
+$template usermsg," XXXX%syslogtag%%msg%\n\r"<br>
+<br>
+And a template with the traditonal wall-message format:<br>
+$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at
+%timegenerated%<br>
+<br>
+A template that can be used for the database write (please note the SQL<br>
+template option)<br>
+$template MySQLInsert,"insert iut, message, receivedat values<br>
+('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')<br>
+into systemevents\r\n", SQL<br>
+<br>
+The following template emulates <a href="http://www.winsyslog.com/en/">WinSyslog</a>
+format (it's an <a href="http://www.adiscon.com/en/">Adiscon</a>
+format, you do not feel bad if you don't know it ;)). It's interesting
+to see how it takes different parts out of the date stamps. What
+happens is that the date stamp is split into the actual date and time
+and the these two are combined with just a comma in between them.<br>
+<br>
+$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,<br>
+%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,<br>
+%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,<br>
+%syslogtag%%msg%\n"</p>
+<h3>SELECTOR LINES</h3>
+<p># Store critical stuff in critical<br>
+#<br>
+*.=crit;kern.none /var/adm/critical<br>
+<br>
+This will store all messages with the priority crit in the file
+/var/adm/critical, except for any kernel message.<br>
+<br>
+<br>
+# Kernel messages are first, stored in the kernel<br>
+# file, critical messages and higher ones also go<br>
+# to another host and to the console. Messages to<br>
+# the host finlandia are forwarded in RFC 3164<br>
+# format (using the template defined above).<br>
+#<br>
+kern.* /var/adm/kernel<br>
+kern.crit @finlandia;RFC3164fmt<br>
+kern.crit /dev/console<br>
+kern.info;kern.!err /var/adm/kernel-info<br>
+<br>
+The first rule direct any message that has the kernel facility to the
+file /var/adm/kernel.<br>
+<br>
+The second statement directs all kernel messages of the priority crit
+and higher to the remote host finlandia. This is useful, because if the
+host crashes and the disks get irreparable errors you might not be able
+to read the stored messages. If they're on a remote host, too, you
+still can try to find out the reason for the crash.<br>
+<br>
+The third rule directs these messages to the actual console, so the
+person who works on the machine will get them, too.<br>
+<br>
+The fourth line tells rsyslogd to save all kernel messages that come
+with priorities from info up to warning in the file
+/var/adm/kernel-info. Everything from err and higher is excluded.<br>
+<br>
+<br>
+# The tcp wrapper loggs with mail.info, we display<br>
+# all the connections on tty12<br>
+#<br>
+mail.=info /dev/tty12<br>
+<br>
+This directs all messages that uses mail.info (in source LOG_MAIL |
+LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper
+tcpd(8) uses this as it's default.<br>
+<br>
+<br>
+# Store all mail concerning stuff in a file<br>
+#<br>
+mail.*;mail.!=info /var/adm/mail<br>
+<br>
+This pattern matches all messages that come with the mail facility,
+except for the info priority. These will be stored in the file
+/var/adm/mail.<br>
+<br>
+<br>
+# Log all mail.info and news.info messages to info<br>
+#<br>
+mail,news.=info /var/adm/info<br>
+<br>
+This will extract all messages that come either with mail.info or with
+news.info and store them in the file /var/adm/info.<br>
+<br>
+<br>
+# Log info and notice messages to messages file<br>
+#<br>
+*.=info;*.=notice;\<br>
+mail.none /var/log/messages<br>
+<br>
+This lets rsyslogd log all messages that come with either the info or
+the notice facility into the file /var/log/messages, except for all<br>
+messages that use the mail facility.<br>
+<br>
+<br>
+# Log info messages to messages file<br>
+#<br>
+*.=info;\<br>
+mail,news.none /var/log/messages<br>
+<br>
+This statement causes rsyslogd to log all messages that come with the
+info priority to the file /var/log/messages. But any message coming
+either with the mail or the news facility will not be stored.<br>
+<br>
+<br>
+# Emergency messages will be displayed using wall<br>
+#<br>
+*.=emerg *<br>
+<br>
+This rule tells rsyslogd to write all emergency messages to all
+currently logged in users. This is the wall action.<br>
+<br>
+<br>
+# Messages of the priority alert will be directed<br>
+# to the operator<br>
+#<br>
+*.alert root,rgerhards<br>
+<br>
+This rule directs all messages with a priority of alert or higher to
+the terminals of the operator, i.e. of the users "root'' and
+"rgerhards'' if they're logged in.<br>
+<br>
+<br>
+*.* @finlandia<br>
+<br>
+This rule would redirect all messages to a remote host called
+finlandia. This is useful especially in a cluster of machines where all
+syslog messages will be stored on only one machine.<br>
+<br>
+In the format shown above, UDP is used for transmitting the message.
+The destination port is set to the default auf 514. Rsyslog is also
+capable of using much more secure and reliable TCP sessions for message
+forwarding. Also, the destination port can be specified. To select TCP,
+simply add one additional @ in front of the host name (that is, @host
+is UPD, @@host is TCP). For example:<br>
+<br>
+<br>
+*.* @@finlandia<br>
+<br>
+To specify the destination port on the remote machine, use a colon
+followed by the port number after the machine name. The following
+forwards to port 1514 on finlandia:<br>
+<br>
+<br>
+*.* @@finlandia:1514<br>
+<br>
+This syntax works both with TCP and UDP based syslog. However, you will
+probably primarily need it for TCP, as there is no well-accepted port
+for this transport (it is non-standard). For UDP, you can usually stick
+with the default auf 514, but might want to modify it for security rea-<br>
+sons. If you would like to do that, it's quite easy:<br>
+<br>
+<br>
+*.* @finlandia:1514<br>
+<br>
+<br>
+<br>
+*.* &gt;dbhost,dbname,dbuser,dbpassword;dbtemplate<br>
+<br>
+This rule writes all message to the database "dbname" hosted on
+"dbhost". The login is done with user "dbuser" and password
+"dbpassword". The actual table that is updated is specified within the
+template (which contains the insert statement). The template is called
+"dbtemplate" in this case.</p>
+<p>:msg,contains,"error" @errorServer</p>
+<p>This rule forwards all messages that contain the word "error"
+in the msg part to the server "errorServer". Forwarding is via UDP.
+Please note the colon in fron</p>
+
+<p>[<a href="manual.html">manual index</a>] 
+[<a href="rsyslog_conf.html">rsyslog.conf</a>] 
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html
new file mode 100644
index 00000000..55244c15
--- /dev/null
+++ b/doc/rsyslog_conf_filter.html
@@ -0,0 +1,280 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Filter Conditions - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Filter Conditions</h2>
+<p>Rsyslog offers four different types "filter conditions":</p>
+<ul>
+<li>BSD-style blocks</li>
+<li>"traditional" severity and facility based selectors</li>
+<li>property-based filters</li>
+<li>expression-based filters</li>
+</ul>
+<h3>Blocks</h3>
+<p>Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each
+block of lines is separated from the previous block by a program or
+hostname specification. A block will only log messages corresponding to
+the most recent program and hostname specifications given. Thus, a
+block which selects &#8216;ppp&#8217; as the program, directly followed by a block
+that selects messages from the hostname &#8216;dialhost&#8217;, then the second
+block will only log messages from the ppp program on dialhost.
+</p>
+<p>A program specification is a line beginning with &#8216;!prog&#8217; and
+the following blocks will be associated with calls to syslog from that
+specific program. A program specification for &#8216;foo&#8217; will also match any
+message logged by the kernel with the prefix &#8216;foo: &#8217;. Alternatively, a
+program specification &#8216;-foo&#8217; causes the following blocks to be applied
+to messages from any program but the one specified. A hostname
+specification of the form &#8216;+hostname&#8217; and the following blocks will be
+applied to messages received from the specified hostname.
+Alternatively, a hostname specification &#8216;-hostname&#8217; causes the
+following blocks to be applied to messages from any host but the one
+specified. If the hostname is given as &#8216;@&#8217;, the local hostname will be
+used. (NOT YET IMPLEMENTED) A program or hostname specification may be
+reset by giving the program or hostname as &#8216;*&#8217;.</p>
+<p>Please note that the "#!prog", "#+hostname" and "#-hostname"
+syntax available in BSD syslogd is not supported by rsyslogd. By
+default, no hostname or program is set.</p>
+<h3>Selectors</h3>
+<p><b>Selectors are the traditional way of filtering syslog
+messages.</b> They have been kept in rsyslog with their original
+syntax, because it is well-known, highly effective and also needed for
+compatibility with stock syslogd configuration files. If you just need
+to filter based on priority and facility, you should do this with
+selector lines. They are <b>not</b> second-class citizens
+in rsyslog and offer the best performance for this job.</p>
+<p>The selector field itself again consists of two parts, a
+facility and a priority, separated by a period (".''). Both parts are
+case insensitive and can also be specified as decimal numbers, but
+don't do that, you have been warned. Both facilities and priorities are
+described in rsyslog(3). The names mentioned below correspond to the
+similar LOG_-values in /usr/include/rsyslog.h.<br>
+<br>
+The facility is one of the following keywords: auth, authpriv, cron,
+daemon, kern, lpr, mail, mark, news, security (same as auth), syslog,
+user, uucp and local0 through local7. The keyword security should not
+be used anymore and mark is only for internal use and therefore should
+not be used in applications. Anyway, you may want to specify and
+redirect these messages here. The facility specifies the subsystem that
+produced the message, i.e. all mail programs log with the mail facility
+(LOG_MAIL) if they log using syslog.<br>
+<br>
+The priority is one of the following keywords, in ascending order:
+debug, info, notice, warning, warn (same as warning), err, error (same
+as err), crit, alert, emerg, panic (same as emerg). The keywords error,
+warn and panic are deprecated and should not be used anymore. The
+priority defines the severity of the message.<br>
+<br>
+The behavior of the original BSD syslogd is that all messages of the
+specified priority and higher are logged according to the given action.
+Rsyslogd behaves the same, but has some extensions.<br>
+<br>
+In addition to the above mentioned names the rsyslogd(8) understands
+the following extensions: An asterisk ("*'') stands for all facilities
+or all priorities, depending on where it is used (before or after the
+period). The keyword none stands for no priority of the given facility.<br>
+<br>
+You can specify multiple facilities with the same priority pattern in
+one statement using the comma (",'') operator. You may specify as much
+facilities as you want. Remember that only the facility part from such
+a statement is taken, a priority part would be skipped.</p>
+<p>Multiple selectors may be specified for a single action using
+the semicolon (";'') separator. Remember that each selector in the
+selector field is capable to overwrite the preceding ones. Using this
+behavior you can exclude some priorities from the pattern.</p>
+<p>Rsyslogd has a syntax extension to the original BSD source,
+that makes its use more intuitively. You may precede every priority
+with an equation sign ("='') to specify only this single priority and
+not any of the above. You may also (both is valid, too) precede the
+priority with an exclamation mark ("!'') to ignore all that
+priorities, either exact this one or this and any higher priority. If
+you use both extensions than the exclamation mark must occur before the
+equation sign, just use it intuitively.</p>
+<h3>Property-Based Filters</h3>
+<p>Property-based filters are unique to rsyslogd. They allow to
+filter on any property, like HOSTNAME, syslogtag and msg. A list of all
+currently-supported properties can be found in the <a href="property_replacer.html">property replacer documentation</a>
+(but keep in mind that only the properties, not the replacer is
+supported). With this filter, each properties can be checked against a
+specified value, using a specified compare operation. Currently, there
+is only a single compare operation (contains) available, but additional
+operations will be added in the future.</p>
+<p>A property-based filter must start with a colon in column 0.
+This tells rsyslogd that it is the new filter type. The colon must be
+followed by the property name, a comma, the name of the compare
+operation to carry out, another comma and then the value to compare
+against. This value must be quoted. There can be spaces and tabs
+between the commas. Property names and compare operations are
+case-sensitive, so "msg" works, while "MSG" is an invalid property
+name. In brief, the syntax is as follows:</p>
+<p><code><b>:property, [!]compare-operation, "value"</b></code></p>
+<p>The following <b>compare-operations</b> are
+currently supported:</p>
+<table id="table1" border="1" width="100%">
+<tbody>
+<tr>
+<td>contains</td>
+<td>Checks if the string provided in value is contained in
+the property. There must be an exact match, wildcards are not supported.</td>
+</tr>
+<tr>
+<td>isequal</td>
+<td>Compares the "value" string provided and the property
+contents. These two values must be exactly equal to match. The
+difference to contains is that contains searches for the value anywhere
+inside the property value, whereas all characters must be identical for
+isequal. As such, isequal is most useful for fields like syslogtag or
+FROMHOST, where you probably know the exact contents.</td>
+</tr>
+<tr>
+<td>startswith</td>
+<td>Checks if the value is found exactly at the beginning
+of the property value. For example, if you search for "val" with
+<p><code><b>:msg, startswith, "val"</b></code></p>
+<p>it will be a match if msg contains "values are in this
+message" but it won't match if the msg contains "There are values in
+this message" (in the later case, contains would match). Please note
+that "startswith" is by far faster than regular expressions. So even
+once they are implemented, it can make very much sense
+(performance-wise) to use "startswith".</p>
+</td>
+</tr>
+<tr>
+<td>regex</td>
+<td>Compares the property against the provided POSIX
+regular
+expression.</td>
+</tr>
+</tbody>
+</table>
+<p>You can use the bang-character (!) immediately in front of a
+compare-operation, the outcome of this operation is negated. For
+example, if msg contains "This is an informative message", the
+following sample would not match:</p>
+<p><code><b>:msg, contains, "error"</b></code></p>
+<p>but this one matches:</p>
+<p><code><b>:msg, !contains, "error"</b></code></p>
+<p>Using negation can be useful if you would like to do some
+generic processing but exclude some specific events. You can use the
+discard action in conjunction with that. A sample would be:</p>
+<p><code><b>*.*
+/var/log/allmsgs-including-informational.log<br>
+:msg, contains, "informational"&nbsp; <font color="#ff0000" size="4">~</font>
+<br>
+*.* /var/log/allmsgs-but-informational.log</b></code></p>
+<p>Do not overlook the red tilde in line 2! In this sample, all
+messages are written to the file allmsgs-including-informational.log.
+Then, all messages containing the string "informational" are discarded.
+That means the config file lines below the "discard line" (number 2 in
+our sample) will not be applied to this message. Then, all remaining
+lines will also be written to the file allmsgs-but-informational.log.</p>
+<p><b>Value</b> is a quoted string. It supports some
+escape sequences:</p>
+<p>\" - the quote character (e.g. "String with \"Quotes\"")<br>
+\\ - the backslash character (e.g. "C:\\tmp")</p>
+<p>Escape sequences always start with a backslash. Additional
+escape sequences might be added in the future. Backslash characters <b>must</b>
+be escaped. Any other sequence then those outlined above is invalid and
+may lead to unpredictable results.</p>
+<p>Probably, "msg" is the most prominent use case of property
+based filters. It is the actual message text. If you would like to
+filter based on some message content (e.g. the presence of a specific
+code), this can be done easily by:</p>
+<p><code><b>:msg, contains, "ID-4711"</b></code></p>
+<p>This filter will match when the message contains the string
+"ID-4711". Please note that the comparison is case-sensitive, so it
+would not match if "id-4711" would be contained in the message.</p>
+<p><code><b>:msg, regex, "fatal .* error"</b></code></p>
+<p>This filter uses a POSIX regular expression. It matches when
+the
+string contains the words "fatal" and "error" with anything in between
+(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as
+two spaces are required by the regular expression!).</p>
+<p>Getting property-based filters right can sometimes be
+challenging. In order to help you do it with as minimal effort as
+possible, rsyslogd spits out debug information for all property-based
+filters during their evaluation. To enable this, run rsyslogd in
+foreground and specify the "-d" option.</p>
+<p>Boolean operations inside property based filters (like
+'message contains "ID17" or message contains "ID18"') are currently not
+supported (except for "not" as outlined above). Please note that while
+it is possible to query facility and severity via property-based
+filters, it is far more advisable to use classic selectors (see above)
+for those cases.</p>
+<h3>Expression-Based Filters</h3>
+Expression based filters allow
+filtering on arbitrary complex expressions, which can include boolean,
+arithmetic and string operations. Expression filters will evolve into a
+full configuration scripting language. Unfortunately, their syntax will
+slightly change during that process. So if you use them now, you need
+to be prepared to change your configuration files some time later.
+However, we try to implement the scripting facility as soon as possible
+(also in respect to stage work needed). So the window of exposure is
+probably not too long.<br>
+<br>
+Expression based filters are indicated by the keyword "if" in column 1
+of a new line. They have this format:<br>
+<br>
+if expr then action-part-of-selector-line<br>
+<br>
+"If" and "then" are fixed keywords that mus be present. "expr" is a
+(potentially quite complex) expression. So the <a href="expression.html">expression documentation</a> for
+details. "action-part-of-selector-line" is an action, just as you know
+it (e.g. "/var/log/logfile" to write to that file).<br>
+<br>
+A few quick samples:<br>
+<br>
+<code>
+*.* /var/log/file1 # the traditional way<br>
+if $msg contains 'error' /var/log/errlog # the expression-based way<br>
+</code>
+<br>
+Right now, you need to specify numerical values if you would like to
+check for facilities and severity. These can be found in <a href="http://www.ietf.org/rfc/rfc3164.txt">RFC 3164</a>.
+If you don't like that, you can of course also use the textual property
+- just be sure to use the right one. As expression support is enhanced,
+this will change. For example, if you would like to filter on message
+that have facility local0, start with "DEVNAME" and have either
+"error1" or "error0" in their message content, you could use the
+following filter:<br>
+<br>
+<code>
+if $syslogfacility-text == 'local0' and $msg
+startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains
+'error0') then /var/log/somelog<br>
+</code>
+<br>
+Please note that the above <span style="font-weight: bold;">must
+all be on one line</span>! And if you would like to store all
+messages except those that contain "error1" or "error0", you just need
+to add a "not":<br>
+<br>
+<code>
+if $syslogfacility-text == 'local0' and $msg
+startswith 'DEVNAME' and <span style="font-weight: bold;">not</span>
+($msg contains 'error1' or $msg contains
+'error0') then /var/log/somelog<br>
+</code>
+<br>
+If you would like to do case-insensitive comparisons, use
+"contains_i" instead of "contains" and "startswith_i" instead of
+"startswith".<br>
+<br>
+Note that regular expressions are currently NOT
+supported in expression-based filters. These will be added later when
+function support is added to the expression engine (the reason is that
+regular expressions will be a separate loadable module, which requires
+some more prequisites before it can be implemented).<br>
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html
new file mode 100644
index 00000000..bc618dd0
--- /dev/null
+++ b/doc/rsyslog_conf_global.html
@@ -0,0 +1,227 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Global Directives - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Global Directives</h2>
+<p>All global directives need to be specified on a line by their
+own and must start with a dollar-sign. Here is a list in alphabetical
+order. Follow links for a description.</p>
+<p>Please note that not all directives here are actually global. Some affect
+only the next action. This documentation will be changed soon.
+<p>Not all directives have an in-depth description right now.
+Default values for them are in bold. A more in-depth description will
+appear as implementation progresses.
+</p>
+<p><b>Be sure to read information about <a href="queues.html">queues in rsyslog</a></b> -
+many parameter settings modify queue parameters. If in doubt, use the
+default, it is usually well-chosen and applicable in most cases.</p>
+<ul>
+<li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li>
+<li>$ActionExecOnlyOnceEveryInterval &lt;seconds&gt; -
+execute action only if the last execute is at last
+&lt;seconds&gt; seconds in the past (more info in <a href="ommail.html">ommail</a>,
+but may be used with any action)</li>
+<li><i><b>$ActionExecOnlyEveryNthTime</b> &lt;number&gt;</i> - If configured, the next action will
+only be executed every n-th time. For example, if configured to 3, the first two messages
+that go into the action will be dropped, the 3rd will actually cause the action to execute,
+the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note:
+this setting is automatically re-set when the actual action is defined.</li>
+<li><i><b>$ActionExecOnlyEveryNthTimeTimeout</b> &lt;number-of-seconds&gt;</i> - has a meaning only if
+$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout
+setting specifies after which period the counting of "previous actions" expires and
+a new action count is begun. Specify 0 (the default) to disable timeouts.
+<br>
+<i>Why is this option needed?</i> Consider this case: a message comes in at, eg., 10am. That's
+count 1. Then, nothing happens for the next 10 hours. At 8pm, the next
+one occurs. That's count 2. Another 5 hours later, the next message
+occurs, bringing the total count to 3. Thus, this message now triggers
+the rule.
+<br>
+The question is if this is desired behavior? Or should the rule only be
+triggered if the messages occur within an e.g. 20 minute window? If the
+later is the case, you need a
+<br>
+$ActionExecOnlyEveryNthTimeTimeout 1200
+<br>
+This directive will timeout previous messages seen if they are older
+than 20 minutes. In the example above, the count would now be always 1
+and consequently no rule would ever be triggered.
+
+<li>$ActionFileDefaultTemplate [templateName] - sets a new default template for file actions</li>
+<li>$ActionFileEnableSync [on/<span style="font-weight: bold;">off</span>] - enables file
+syncing capability of omfile</li>
+<li>$ActionForwardDefaultTemplate [templateName] - sets a new
+default template for UDP and plain TCP forwarding action</li>
+<li>$ActionGSSForwardDefaultTemplate [templateName] - sets a
+new default template for GSS-API forwarding action</li>
+<li>$ActionQueueCheckpointInterval &lt;number&gt;</li>
+<li>$ActionQueueDequeueSlowdown &lt;number&gt; [number
+is timeout in <i> micro</i>seconds (1000000us is 1sec!),
+default 0 (no delay). Simple rate-limiting!]</li>
+<li>$ActionQueueDiscardMark &lt;number&gt; [default
+9750]</li>
+<li>$ActionQueueDiscardSeverity &lt;number&gt;
+[*numerical* severity! default 4 (warning)]</li>
+<li>$ActionQueueFileName &lt;name&gt;</li>
+<li>$ActionQueueHighWaterMark &lt;number&gt; [default
+8000]</li>
+<li>$ActionQueueImmediateShutdown [on/<b>off</b>]</li>
+<li>$ActionQueueSize &lt;number&gt;</li>
+<li>$ActionQueueLowWaterMark &lt;number&gt; [default
+2000]</li>
+<li>$ActionQueueMaxFileSize &lt;size_nbr&gt;, default 1m</li>
+<li>$ActionQueueTimeoutActionCompletion &lt;number&gt;
+[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means
+immediate!]</li>
+<li>$ActionQueueTimeoutEnqueue &lt;number&gt; [number
+is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li>
+<li>$ActionQueueTimeoutShutdown &lt;number&gt; [number
+is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li>
+<li>$ActionQueueWorkerTimeoutThreadShutdown
+&lt;number&gt; [number is timeout in ms (1000ms is 1sec!),
+default 60000 (1 minute)]</li>
+<li>$ActionQueueType [FixedArray/LinkedList/<b>Direct</b>/Disk]</li>
+<li>$ActionQueueSaveOnShutdown&nbsp; [on/<b>off</b>]
+</li>
+<li>$ActionQueueWorkerThreads &lt;number&gt;, num worker threads, default 1, recommended 1</li>
+<li>$ActionQueueWorkerThreadMinumumMessages &lt;number&gt;, default 100</li>
+<li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li>
+<li>$ActionResumeRetryCount &lt;number&gt; [default 0, -1 means eternal]</li>
+<li>$ActionSendResendLastMsgOnReconn &lt;[on/<b>off</b>]&gt; specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication.
+<li>$ActionSendStreamDriver &lt;driver basename&gt; just like $DefaultNetstreamDriver, but for the specific action
+</li><li>$ActionSendStreamDriverMode &lt;mode&gt;, default 0, mode to use with the stream driver
+(driver-specific)</li><li>$ActionSendStreamDriverAuthMode &lt;mode&gt;,&nbsp; authentication mode to use with the stream driver
+(driver-specific)</li><li>$ActionSendStreamDriverPermittedPeer &lt;ID&gt;,&nbsp; accepted fingerprint (SHA1) or name of remote peer
+(driver-specific) -<span style="font-weight: bold;"> directive may go away</span>!</li>
+<li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li>
+<li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li>
+<li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li>
+
+<li><a href="rsconf1_debugprintmodulelist.html">$DebugPrintModuleList</a></li>
+<li><a href="rsconf1_debugprinttemplatelist.html">$DebugPrintTemplateList</a></li>
+<li>$DefaultNetstreamDriver &lt;drivername&gt;, the default <a href="netstream.html">network stream driver</a> to use. Defaults to&nbsp;ptcp.$DefaultNetstreamDriverCAFile &lt;/path/to/cafile.pem&gt;</li>
+<li>$DefaultNetstreamDriverCertFile &lt;/path/to/certfile.pem&gt;</li>
+<li>$DefaultNetstreamDriverKeyFile &lt;/path/to/keyfile.pem&gt;</li>
+<li><a href="rsconf1_dircreatemode.html">$DirCreateMode</a></li>
+<li><a href="rsconf1_dirgroup.html">$DirGroup</a></li>
+<li><a href="rsconf1_dirowner.html">$DirOwner</a></li>
+<li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li>
+<li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li>
+<li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li>
+<li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li>
+<li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li>
+<li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li>
+<li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li>
+<li><a href="rsconf1_filegroup.html">$FileGroup</a></li>
+<li><a href="rsconf1_fileowner.html">$FileOwner</a></li>
+<li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li>
+<li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li>
+<li><a href="rsconf1_gssmode.html">$GssMode</a></li>
+<li>$HUPisRestart [<b>on</b>/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending
+on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is
+not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for
+compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually
+the only one that is needed e.g. for log rotation. <b>It is recommended to set the setting to "off".</b></li>
+<li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval &lt;number&gt;</li>
+<li>$MainMsgQueueDequeueSlowdown &lt;number&gt; [number
+is timeout in <i> micro</i>seconds (1000000us is 1sec!),
+default 0 (no delay). Simple rate-limiting!]</li>
+<li>$MainMsgQueueDiscardMark &lt;number&gt; [default 9750]</li>
+<li>$MainMsgQueueDiscardSeverity &lt;severity&gt;
+[either a textual or numerical severity! default 4 (warning)]</li>
+<li>$MainMsgQueueFileName &lt;name&gt;</li>
+<li>$MainMsgQueueHighWaterMark &lt;number&gt; [default
+8000]</li>
+<li>$MainMsgQueueImmediateShutdown [on/<b>off</b>]</li>
+<li><a href="rsconf1_mainmsgqueuesize.html">$MainMsgQueueSize</a></li>
+<li>$MainMsgQueueLowWaterMark &lt;number&gt; [default
+2000]</li>
+<li>$MainMsgQueueMaxFileSize &lt;size_nbr&gt;, default
+1m</li>
+<li>$MainMsgQueueTimeoutActionCompletion
+&lt;number&gt; [number is timeout in ms (1000ms is 1sec!),
+default
+1000, 0 means immediate!]</li>
+<li>$MainMsgQueueTimeoutEnqueue &lt;number&gt; [number
+is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li>
+<li>$MainMsgQueueTimeoutShutdown &lt;number&gt; [number
+is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li>
+<li>$MainMsgQueueWorkerTimeoutThreadShutdown
+&lt;number&gt; [number is timeout in ms (1000ms is 1sec!),
+default 60000 (1 minute)]</li>
+<li>$MainMsgQueueType [<b>FixedArray</b>/LinkedList/Direct/Disk]</li>
+<li>$MainMsgQueueSaveOnShutdown&nbsp; [on/<b>off</b>]
+</li>
+<li>$MainMsgQueueWorkerThreads &lt;number&gt;, num
+worker threads, default 1, recommended 1</li>
+<li>$MainMsgQueueWorkerThreadMinumumMessages &lt;number&gt;, default 100</li>
+<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> (immark)</li>
+<li><b><i>$MaxMessageSize</i></b> &lt;size_nbr&gt;, default 2k - allows to specify maximum supported message size
+(both for sending and receiving). The default
+should be sufficient for almost all cases. Do not set this below 1k, as it would cause
+interoperability problems with other syslog implementations.<br>
+Change the setting to e.g. 32768 if you would like to
+support large message sizes for IHE (32k is the current maximum
+needed for IHE). I was initially tempted to set the default to 32k,
+but there is a some memory footprint with the current
+implementation in rsyslog.
+<br>If you intend to receive Windows Event Log data (e.g. via
+<a href="http://www.eventreporter.com/">EventReporter</a>), you might want to
+increase this number to an even higher value, as event
+log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea).
+Note: testing showed that 4k seems to be
+the typical maximum for <b>UDP</b> based syslog. This is an IP stack
+restriction. Not always ... but very often. If you go beyond
+that value, be sure to test that rsyslogd actually does what
+you think it should do ;) It is highly suggested to use a TCP based transport
+instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions.
+<br>Note that 2k, the current default, is the smallest size that must be
+supported in order to be compliant to the upcoming new syslog RFC series.
+</li>
+<li><a href="rsconf1_moddir.html">$ModDir</a></li>
+<li><a href="rsconf1_modload.html">$ModLoad</a></li>
+<li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li>
+<li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li>
+<li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better
+performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The
+default may change as uniprocessor systems become less common.</li>
+<li>$WorkDirectory &lt;name&gt; (directory for spool and other work files)</li>
+<li>$UDPServerAddress &lt;IP&gt; (imudp) -- local IP
+address (or name) the UDP listens should bind to</li>
+<li>$UDPServerRun &lt;port&gt; (imudp) -- former
+-r&lt;port&gt; option, default 514, start UDP server on this
+port, "*" means all addresses</li>
+<li>$UDPServerTimeRequery &lt;nbr-of-times&gt; (imudp) -- this is a performance
+optimization. Getting the system time is very costly. With this setting, imudp can
+be instructed to obtain the precise time only once every n-times. This logic is
+only activated if messages come in at a very fast rate, so doing less frequent
+time calls should usually be acceptable. The default value is two, because we have
+seen that even without optimization the kernel often returns twice the identical time.
+You can set this value as high as you like, but do so at your own risk. The higher
+the value, the less precise the timestamp.
+<li><a href="rsconf1_umask.html">$UMASK</a></li>
+</ul>
+<p><b>Where &lt;size_nbr&gt; is specified above,</b>
+modifiers can be used after the number part. For example, 1k means
+1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa).
+Lower case letters refer to the traditional binary defintion (e.g. 1m
+equals 1,048,576) whereas upper case letters refer to their new
+1000-based definition (e.g 1M equals 1,000,000).</p>
+<p>Numbers may include '.' and ',' for readability. So you can
+for example specify either "1000" or "1,000" with the same result.
+Please note that rsyslogd simply ignores the punctuation. Form it's
+point of view, "1,,0.0.,.,0" also has the value 1000. </p>
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
+
diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html
new file mode 100644
index 00000000..890a55c8
--- /dev/null
+++ b/doc/rsyslog_conf_modules.html
@@ -0,0 +1,53 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Modules - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Modules</h2>
+<p>Rsyslog has a modular design. Consequently, there is a growing
+number of modules. Here is the entry point to their documentation and
+what they do (list is currently not complete)</p>
+<ul>
+<li><a href="omsnmp.html">omsnmp</a> - SNMP
+trap output module</li>
+<li><a href="omrelp.html">omrelp</a> - RELP
+output module</li>
+<li>omgssapi - output module for GSS-enabled syslog</li>
+<li><a href="ommysql.html">ommysql</a> - output module for MySQL</li>
+<li>ompgsql - output module for PostgreSQL</li>
+<li><a href="omlibdbi.html">omlibdbi</a> -
+generic database output module (Firebird/Interbase, MS SQL, Sybase,
+SQLLite, Ingres, Oracle, mSQL)</li>
+<li><a href="ommail.html">ommail</a> -
+permits rsyslog to alert folks by mail if something important happens</li>
+<li><a href="imfile.html">imfile</a>
+-&nbsp; input module for text files</li>
+<li><a href="imrelp.html">imrelp</a> - RELP
+input module</li>
+<li>imudp - udp syslog message input</li>
+<li><a href="imtcp.html">imtcp</a> - input
+plugin for plain tcp syslog</li>
+<li><a href="imgssapi.html">imgssapi</a> -
+input plugin for plain tcp and GSS-enabled syslog</li>
+<li>immark - support for mark messages</li>
+<li><a href="imklog.html">imklog</a> - kernel logging</li>
+<li><a href="imuxsock.html">imuxsock</a> -
+unix sockets, including the system log socket</li>
+<li><a href="im3195.html">im3195</a> -
+accepts syslog messages via RFC 3195</li>
+</ul>
+<p>Please note that each module provides configuration
+directives, which are NOT necessarily being listed below. Also
+remember, that a modules configuration directive (and functionality) is
+only available if it has been loaded (using $ModLoad).</p>
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
diff --git a/doc/rsyslog_conf_output.html b/doc/rsyslog_conf_output.html
new file mode 100644
index 00000000..c52aaa5e
--- /dev/null
+++ b/doc/rsyslog_conf_output.html
@@ -0,0 +1,81 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Output Channels - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Output Channels</h2>
+<p>Output Channels are a new concept first introduced in rsyslog
+0.9.0. <b>As of this writing, it is most likely that they will
+be replaced by something different in the future.</b> So if you
+use them, be prepared to change you configuration file syntax when you
+upgrade to a later release.<br>
+<br>
+The idea behind output channel definitions is that it shall provide an
+umbrella for any type of output that the user might want. In essence,<br>
+this is the "file" part of selector lines (and this is why we are not
+sure output channel syntax will stay after the next review). There is a<br>
+difference, though: selector channels both have filter conditions
+(currently facility and severity) as well as the output destination.
+they can only be used to write to files - not pipes, ttys or whatever
+Output channels define the output definition, only. As of this build,
+else. If we stick with output channels, this will change over time.</p>
+<p>In concept, an output channel includes everything needed to
+know about an output actions. In practice, the current implementation
+only carries<br>
+a filename, a maximum file size and a command to be issued when this
+file size is reached. More things might be present in future version,
+which might also change the syntax of the directive.</p>
+<p>Output channels are defined via an $outchannel directive. It's
+syntax is as follows:<br>
+<br>
+$outchannel name,file-name,max-size,action-on-max-size<br>
+<br>
+name is the name of the output channel (not the file), file-name is the
+file name to be written to, max-size the maximum allowed size and
+action-on-max-size a command to be issued when the max size is reached.
+This command always has exactly one parameter. The binary is that part
+of action-on-max-size before the first space, its parameter is
+everything behind that space.<br>
+<br>
+Please note that max-size is queried BEFORE writing the log message to
+the file. So be sure to set this limit reasonably low so that any
+message might fit. For the current release, setting it 1k lower than
+you expected is helpful. The max-size must always be specified in bytes
+- there are no special symbols (like 1k, 1m,...) at this point of
+development.<br>
+<br>
+Keep in mind that $outchannel just defines a channel with "name". It
+does not activate it. To do so, you must use a selector line (see
+below). That selector line includes the channel name plus an $ sign in
+front of it. A sample might be:<br>
+<br>
+*.* $mychannel<br>
+<br>
+In its current form, output channels primarily provide the ability to
+size-limit an output file. To do so, specify a maximum size. When this
+size is reached, rsyslogd will execute the action-on-max-size command
+and then reopen the file and retry. The command should be something
+like a <a href="log_rotation_fix_size.html">log rotation
+script</a> or a similar thing.</p>
+<p>If there is no action-on-max-size command or the command did
+not resolve the situation, the file is closed and never reopened by
+rsyslogd (except, of course, by huping it). This logic was integrated
+when we first experienced severe issues with files larger 2gb, which
+could lead to rsyslogd dumping core. In such cases, it is more
+appropriate to stop writing to a single file. Meanwhile, rsyslogd has
+been fixed to support files larger 2gb, but obviously only on file
+systems and operating system versions that do so. So it can still make
+sense to enforce a 2gb file size limit.</p>
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
+
diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html
new file mode 100644
index 00000000..90b5fafe
--- /dev/null
+++ b/doc/rsyslog_conf_templates.html
@@ -0,0 +1,150 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>Templates - rsyslog.conf</title></head>
+<body>
+<p>This is a part of the rsyslog.conf - documentation.</p>
+<a href="rsyslog_conf.html">back</a>
+<h2>Templates</h2>
+<p>Templates are a key feature of rsyslog. They allow to specify
+any
+format a user might want. They are also used for dynamic file name
+generation. Every output in rsyslog uses templates - this holds true
+for files, user messages and so on. The database writer expects its
+template to be a proper SQL statement - so this is highly customizable
+too. You might ask how does all of this work when no templates at all
+are specified. Good question ;) The answer is simple, though. Templates
+compatible with the stock syslogd formats are hardcoded into rsyslogd.
+So if no template is specified, we use one of these hardcoded
+templates. Search for "template_" in syslogd.c and you will find the
+hardcoded ones.</p>
+<p>A template consists of a template directive, a name, the
+actual template text and optional options. A sample is:</p>
+<blockquote><code>$template MyTemplateName,"\7Text
+%property% some more text\n",&lt;options&gt;</code></blockquote>
+<p>The "$template" is the template directive. It tells rsyslog
+that this line contains a template. "MyTemplateName" is the template
+name. All
+other config lines refer to this name. The text within quotes is the
+actual template text. The backslash is an escape character, much as it
+is in C. It does all these "cool" things. For example, \7 rings the
+bell (this is an ASCII value), \n is a new line. C programmers and perl
+coders have the advantage of knowing this, but the set in rsyslog is a
+bit restricted currently.
+</p>
+<p>All text in the template is used literally, except for things
+within percent signs. These are properties and allow you access to the
+contents of the syslog message. Properties are accessed via the
+property replacer (nice name, huh) and it can do cool things, too. For
+example, it can pick a substring or do date-specific formatting. More
+on this is below, on some lines of the property replacer.<br>
+<br>
+The &lt;options&gt; part is optional. It carries options
+influencing the template as whole. See details below. Be sure NOT to
+mistake template options with property options - the later ones are
+processed by the property replacer and apply to a SINGLE property, only
+(and not the whole template).<br>
+<br>
+Template options are case-insensitive. Currently defined are: </p>
+<p><b>sql</b> - format the string suitable for a SQL
+statement in MySQL format. This will replace single quotes ("'") and
+the backslash character by their backslash-escaped counterpart ("\'"
+and "\\") inside each field. Please note that in MySQL configuration,
+the <code class="literal">NO_BACKSLASH_ESCAPES</code>
+mode must be turned off for this format to work (this is the default).</p>
+<p><b>stdsql</b> - format the string suitable for a
+SQL statement that is to be sent to a standards-compliant sql server.
+This will replace single quotes ("'") by two single quotes ("''")
+inside each field. You must use stdsql together with MySQL if in MySQL
+configuration the
+<code class="literal">NO_BACKSLASH_ESCAPES</code> is
+turned on.</p>
+<p>Either the <b>sql</b> or <b>stdsql</b>&nbsp;
+option <b>must</b> be specified when a template is used
+for writing to a database, otherwise injection might occur. Please note
+that due to the unfortunate fact that several vendors have violated the
+sql standard and introduced their own escape methods, it is impossible
+to have a single option doing all the work.&nbsp; So you yourself
+must make sure you are using the right format. <b>If you choose
+the wrong one, you are still vulnerable to sql injection.</b><br>
+<br>
+Please note that the database writer *checks* that the sql option is
+present in the template. If it is not present, the write database
+action is disabled. This is to guard you against accidental forgetting
+it and then becoming vulnerable to SQL injection. The sql option can
+also be useful with files - especially if you want to import them into
+a database on another machine for performance reasons. However, do NOT
+use it if you do not have a real need for it - among others, it takes
+some toll on the processing time. Not much, but on a really busy system
+you might notice it ;)</p>
+<p>The default template for the write to database action has the
+sql option set. As we currently support only MySQL and the sql option
+matches the default MySQL configuration, this is a good choice.
+However, if you have turned on
+<code class="literal">NO_BACKSLASH_ESCAPES</code> in
+your MySQL config, you need to supply a template with the stdsql
+option. Otherwise you will become vulnerable to SQL injection. <br>
+<br>
+To escape:<br>
+% = \%<br>
+\ = \\ --&gt; '\' is used to escape (as in C)<br>
+$template TraditionalFormat,%timegenerated% %HOSTNAME%
+%syslogtag%%msg%\n"<br>
+<br>
+Properties can be accessed by the <a href="property_replacer.html">property
+replacer</a> (see there for details).</p>
+<p><b>Please note that templates can also by
+used to generate selector lines with dynamic file names.</b> For
+example, if you would like to split syslog messages from different
+hosts to different files (one per host), you can define the following
+template:</p>
+<blockquote><code>$template
+DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote>
+<p>This template can then be used when defining an output
+selector line. It will result in something like
+"/var/log/system-localhost.log"</p>
+<p>Template
+names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT
+use them if, otherwise you may receive a conflict in the future (and
+quite unpredictable behaviour). There is a small set of pre-defined
+templates that you can use without the need to define it:</p>
+<ul>
+<li><span style="font-weight: bold;">RSYSLOG_TraditionalFileFormat</span>
+- the "old style" default log file format with low-precision timestamps</li>
+<li><span style="font-weight: bold;">RSYSLOG_FileFormat</span>
+- a modern-style logfile format similar to TraditionalFileFormat, buth
+with high-precision timestamps and timezone information</li>
+<li><span style="font-weight: bold;">RSYSLOG_TraditionalForwardFormat</span>
+- the traditional forwarding format with low-precision timestamps. Most
+useful if you send&nbsp;messages to other syslogd's or rsyslogd
+below
+version 3.12.5.</li>
+<li><span style="font-weight: bold;">RSYSLOG_ForwardFormat</span>
+- a new high-precision forwarding format very similar to the
+traditional one, but with high-precision timestamps and timezone
+information. Recommended to be used when sending messages to rsyslog
+3.12.5 or above.</li>
+<li><span style="font-weight: bold;">RSYSLOG_SyslogProtocol23Format</span>
+- the format specified in IETF's internet-draft
+ietf-syslog-protocol-23, which is assumed to be come the new syslog
+standard RFC. This format includes several improvements. The rsyslog
+message parser understands this format, so you can use it together with
+all relatively recent versions of rsyslog. Other syslogd's may get
+hopelessly confused if receiving that format, so check before you use
+it. Note that the format is unlikely to change when the final RFC comes
+out, but this may happen.</li>
+<li><span style="font-weight: bold;">RSYSLOG_DebugFormat</span>
+- a special format used for troubleshooting property problems. This format
+is meant to be written to a log file. Do <b>not</b> use for production or remote
+forwarding.</li>
+</ul>
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
-- 
cgit 


From 249b27952a9faea95662eb230f4c86a0db874fe5 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 11 Nov 2008 11:38:37 +0100
Subject: improved doc on property replacer regular expressions

---
 doc/Makefile.am               |  1 +
 doc/property_replacer.html    | 10 +++-------
 doc/rsyslog_conf_nomatch.html | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 7 deletions(-)
 create mode 100644 doc/rsyslog_conf_nomatch.html

diff --git a/doc/Makefile.am b/doc/Makefile.am
index fef1e44c..5c2f5313 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -105,6 +105,7 @@ html_files = \
 	rsyslog_conf_modules.html \
 	rsyslog_conf_output.html \
 	rsyslog_conf_templates.html \
+	rsyslog_conf_nomatch.html \
 	src/classes.dia
 
 EXTRA_DIST = $(html_files)
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 34e2116c..9ea41aed 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -229,7 +229,7 @@ sequence with a regular expression is: "%msg:R:.*Sev:. \(.*\)
 \[.*--end%"</p>
 <p>It is possible to specify some parametes after the "R". These are
 comma-separated. They are:
-<p>R,&lt;regexp-type&gt;,&lt;submatch&gt;,&lt;nomatch&gt;,&lt;match-number&gt;
+<p>R,&lt;regexp-type&gt;,&lt;submatch&gt;,&lt;<a href="rsyslog_conf_nomatch.html">nomatch</a>&gt;,&lt;match-number&gt;
 <p>regexp-type is either "BRE" for Posix basic regular expressions or
 "ERE" for extended ones. The string must be given in upper case. The
 default is "BRE" to be consistent with earlier versions of rsyslog that
@@ -241,12 +241,8 @@ that the first match is number 0, the second 1 and so on. Up to 10 matches
 (up to number 9) are supported. Please note that it would be more
 natural to have the match-number in front of submatch, but this would break 
 backward-compatibility. So the match-number must be specified after "nomatch".
-<p>nomatch is either "DFLT", "BLANK" or "FIELD" (all upper case!). It tells
-what to use if no match is found. With "DFLT", the strig "**NO MATCH**" is
-used. This was the only supported value up to rsyslog 3.19.5. With "BLANK"
-a blank text is used (""). Finally, "FIELD" uses the full property text
-instead of the expression. Some folks have requested that, so it seems
-to be useful.
+<p><a href="rsyslog_conf_nomatch.html">nomatch</a> specifies what should
+be used in case no match is found.
 <p>The following is a sample of an ERE expression that takes the first
 submatch from the message string and replaces the expression with
 the full field if no match is found:
diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html
new file mode 100644
index 00000000..5c4f3f90
--- /dev/null
+++ b/doc/rsyslog_conf_nomatch.html
@@ -0,0 +1,37 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><title>nomatch mode - property replacer - rsyslog.conf</title></head>
+<body>
+<h1>nomatch mode - property replacer - rsyslog.con</h1>
+<p>This is a part of the <a href="rsyslog_conf.html">rsyslog.conf documentation</a>
+of the <a href="property_replacer.html">property replacer</a>.</p>
+<p><b>The "nomatch-Mode" specifies which string the property replacer
+shall return if a regular expression did not find the search string.</b>. Traditionally,
+the string "**NO MATCH**" was returned, but many people complained this was almost never useful.
+Still, this mode is support as "<b>DFLT</b>" for legacy configurations.
+<p>Two additional and potentially useful modes exist: in one (<b>BLANK</b>) a blank string
+is returned. This is probably useful for inserting values into databases where no
+value shall be inserted if the expression could not be found. A use case may be
+that you record a traffic log based on firewall rules and the "bytes transmitted" counter
+is extracted via a regular expression. If no "bytes transmitted" counter is available
+in the current message, it is probably a good idea to return an empty string, which the
+database layer can turn into a zero.
+<p>The other mode is "<b>FIELD</b>", in which the complete field is returned. This may be useful
+in cases where absense of a match is considered a failure and the message that triggered
+it shall be logged.
+<p>If in doubt, <b>it is highly suggested to use the
+<a href="http://www.rsyslog.com/tool-regex">rsyslog online regular expression
+checker and generator</a> to see these options in action</b>. With that online tool,
+you can craft regular expressions based on samples and try out the different modes.
+
+<p>[<a href="manual.html">manual index</a>]
+[<a href="rsyslog_conf.html">rsyslog.conf</a>]
+[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
+version 2 or higher.</font></p>
+</body>
+</html>
+
+
-- 
cgit 


From 704a1145d64532df36624a3c9850b0c86f38f76f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 11 Nov 2008 12:22:06 +0100
Subject: doc: added new nomatch mode

Note: the actual nomatch mode is not yet available in this code,
this needs to be merged from v3-stable first. This happens soon,
but I wanted to make sure the doc is right.
---
 doc/rsyslog_conf_nomatch.html | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html
index 5c4f3f90..5f25f3e4 100644
--- a/doc/rsyslog_conf_nomatch.html
+++ b/doc/rsyslog_conf_nomatch.html
@@ -8,9 +8,11 @@ of the <a href="property_replacer.html">property replacer</a>.</p>
 shall return if a regular expression did not find the search string.</b>. Traditionally,
 the string "**NO MATCH**" was returned, but many people complained this was almost never useful.
 Still, this mode is support as "<b>DFLT</b>" for legacy configurations.
-<p>Two additional and potentially useful modes exist: in one (<b>BLANK</b>) a blank string
+<p>Three additional and potentially useful modes exist: in one (<b>BLANK</b>) a blank string
 is returned. This is probably useful for inserting values into databases where no
-value shall be inserted if the expression could not be found. A use case may be
+value shall be inserted if the expression could not be found.
+<p>A similar mode is "<b>ZERO</b>" where the string "0" is returned. This is suitable
+for numerical values. A use case may be
 that you record a traffic log based on firewall rules and the "bytes transmitted" counter
 is extracted via a regular expression. If no "bytes transmitted" counter is available
 in the current message, it is probably a good idea to return an empty string, which the
@@ -23,6 +25,15 @@ it shall be logged.
 checker and generator</a> to see these options in action</b>. With that online tool,
 you can craft regular expressions based on samples and try out the different modes.
 
+<h2>Summary of nomatch Modes</h2>
+<table border="1" cellspacing="0">
+<tr><td><b>Mode</b></td><td><b>Returned</b></td></tr>
+<tr><td>DFLT</td><td>"**NO MATCH**"</td></tr>
+<tr><td>BLANK</td><td>"" (empty string)</td></tr>
+<tr><td>ZERO</td><td>"0"</td></tr>
+<tr><td>FIELD</td><td>full content of original field</td></tr>
+<tr><td>&nbsp;</td><td><a href="http://www.rsyslog.com/tool-regex">Interactive Tool</a></td></tr>
+</table>
 <p>[<a href="manual.html">manual index</a>]
 [<a href="rsyslog_conf.html">rsyslog.conf</a>]
 [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
-- 
cgit