Merge branch 'beta'

Conflicts: ChangeLog conf.c doc/Makefile.am doc/manual.html omfwd.c plugins/omgssapi/omgssapi.c This was a bit hard to merge; if there are problems, they may be in the area of the new "comment in action line" code that came from the beta.
author: Rainer Gerhards <rgerhards@adiscon.com> 2008-07-01 15:09:07 +0200
committer: Rainer Gerhards <rgerhards@adiscon.com> 2008-07-01 15:09:07 +0200
commit: 4fd98529572b57f494597a34f04ced96ef1031a2 (patch)
tree: a2abace73c6cfcfa9db49d4c04ed2a1e831cef74
parent: 78543b7e31ea9559108d15fd645862db7dd63913 (diff)
parent: a16cbe655541421fefdcc852087219228bbc5c94 (diff)
download: rsyslog-4fd98529572b57f494597a34f04ced96ef1031a2.tar.gz
rsyslog-4fd98529572b57f494597a34f04ced96ef1031a2.tar.xz
rsyslog-4fd98529572b57f494597a34f04ced96ef1031a2.zip
9 files changed, 185 insertions, 38 deletions
diff --git a/ChangeLog b/ChangeLog
index 115f8282..11fe3b7a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -118,6 +118,11 @@ Version 3.19.0 (rgerhards), 2008-05-06
 - a lot of cleanup in regard to modularization
 - -c option no longer must be the first option - thanks to varmjofekoj
   for the patch
+Version 3.17.5 (rgerhards), 2008-06-27
+- added doc: howto set up a reliable connection to remote server via
+  queued mode (and plain tcp protocol)
+- bugfix: comments after actions were not properly treated. For some
+  actions (e.g. forwarding), this could also lead to invalid configuration
 ---------------------------------------------------------------------------
 Version 3.17.4 (rgerhards), 2008-06-16
 - changed default for $KlogSymbolLookup to "off". The directive is
@@ -184,7 +189,7 @@ Version 3.17.0 (rgerhards), 2008-04-08
 Plus a number of bugfixes that were applied to v3-stable and beta
 branches (not mentioned here in detail).
 ---------------------------------------------------------------------------
-Version 3.16.2 (rgerhards), 2008-05-14
+Version 3.16.2 (rgerhards), 2008-06-25
 - fixed potential segfault due to invalid call to cfsysline
   thanks to varmojfekoj for the patch
 - bugfix: some whitespaces where incorrectly not ignored when parsing
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 4ddb1179..03b18f96 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -22,6 +22,7 @@ html_files = \
 	rsyslog_php_syslog_ng.html \
 	rsyslog_recording_pri.html \
 	rsyslog_tls.html \
+	rsyslog_reliable_forwarding.html \
 	rsyslog_stunnel.html \
 	syslog-protocol.html \
 	version_naming.html \
diff --git a/doc/manual.html b/doc/manual.html
index 1c0ffa7d..d1b670aa 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -50,15 +50,12 @@ modules</a></li><li><a href="man_rsyslogd.html">rsyslogd man page</a>
 <p><b>We have some in-depth papers on</b></p>
 <ul>
 <li><a href="install.html">installing rsyslog</a></li>
-<li><a href="ipv6.html">rsyslog and IPv6</a>
-(which is fully supported)</li>
+<li><a href="ipv6.html">rsyslog and IPv6</a> (which is fully supported)</li>
 <li><a href="rsyslog_tls.html">native TLS encryption for syslog</a></li>
-<li><a href="rsyslog_stunnel.html">ssl-encrypting
-syslog with stunnel</a></li>
-<li><a href="rsyslog_mysql.html">writing syslog
-messages to MySQL (and other databases as well)</a></li>
-<li><a href="rsyslog_high_database_rate.html">writing
-massive amounts of syslog messages to a database</a></li>
+<li><a href="rsyslog_stunnel.html">ssl-encrypting syslog with stunnel</a></li>
+<li><a href="rsyslog_mysql.html">writing syslog messages to MySQL (and other databases as well)</a></li>
+<li><a href="rsyslog_high_database_rate.html">writing massive amounts of syslog messages to a database</a></li>
+<li><a href="rsyslog_reliable_forwarding.html">reliable forwarding to a remote server</a></li>
 <li><a href="rsyslog_php_syslog_ng.html">using
 php-syslog-ng with rsyslog</a></li>
 <li><a href="rsyslog_recording_pri.html">recording
@@ -67,7 +64,7 @@ the syslog priority (severity and facility) to the log file</a></li>
 syslog sender over NAT</a> (online only)</li>
 <li><a href="gssapi.html">an overview and howto of rsyslog gssapi support</a></li>
 <li><a href="debug.html">debug support in rsyslog</a></li>
-<li><a href="dev_queue.html">the rsyslog message queue object</a></li>
+<li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li>
 </ul>
 <p>Our <a href="history.html">rsyslog history</a>
 page is for you if you would like to learn a little more
diff --git a/doc/ommail.html b/doc/ommail.html
index ba06a366..62ded6d0 100644
--- a/doc/ommail.html
+++ b/doc/ommail.html
@@ -5,6 +5,7 @@
 <body>
 <h1>Mail Output Module (ommail)</h1>
 <p><b>Module Name:&nbsp;&nbsp;&nbsp; ommail</b></p>
+<p><b>Available since:&nbsp;&nbsp;&nbsp;</b> 3.17.0</p>
 <p><b>Author: </b>Rainer Gerhards
 &lt;rgerhards@adiscon.com&gt;</p>
 <p><b>Description</b>:</p>
diff --git a/doc/rsyslog_reliable_forwarding.html b/doc/rsyslog_reliable_forwarding.html
new file mode 100644
index 00000000..870ca9b7
--- /dev/null
+++ b/doc/rsyslog_reliable_forwarding.html
@@ -0,0 +1,152 @@
+<html><head>
+<title>Reliable Forwarding of syslog Messages (via plain TCP syslog)</title>
+</head>
+<body>
+<h1>Reliable Forwarding of syslog Messages with Rsyslog</h1>
+		<P><small><i>Written by
+		<a href="http://www.gerhards.net/rainer">Rainer 
+		Gerhards</a> (2008-06-27)</i></small></P>
+<h2>Abstract</h2>
+<p><i><b>In this paper, I describe how to forward
+<a href="http://www.monitorware.com/en/topics/syslog/">syslog</a>
+
+ messages (quite) reliable to a central rsyslog server.</b>
+This depends on rsyslog being installed on the client system and
+it is recommended to have it installed on the server system. Please note
+that industry-standard
+<a href="http://blog.gerhards.net/2008/04/on-unreliability-of-plain-tcp-syslog.html">plain TCP syslog protocol is not fully reliable</a>
+(thus the "quite reliable"). If you need a truely reliable solution, you need
+to look into RELP (natively supported by rsyslog).</i></p>
+
+<h2>The Intention</h2>
+<p>Whenever two systems talk over a network, something can go wrong.
+For example, the communications link may go down, or a client or server may abort.
+Even in regular cases, the server may be offline for a short period of time
+because of routine maintenance.
+<p>A logging system should be capable of avoiding message loss in situations where the
+server is not reachable. To do so, unsent data needs to be buffered at the client while the
+server is offline. Then, once the server is up again, this data is to be sent.
+<p>This can easily be acomplished by rsyslog. In rsyslog, every action runs on its own queue
+and each queue can be set to buffer data if the action is not ready. Of course,
+you must be able to detect that "the action is not ready", which means the remote
+server is offline. This can be detected with plain TCP syslog and RELP, but not with UDP.
+So you need to use either of the two. In this howto, we use plain TCP syslog.
+<p>Please note that we are using rsyslog-specific features. The are required on the
+client, but not on the server. So the client system must run rsyslog (at least version 3.12.0), while on the
+server another syslogd may be running, as long as it supports plain tcp syslog.
+<p><b>The rsyslog queueing subsystem tries to buffer to memory. So even if the
+remote server goes
+offline, no disk file is generated.</b> File on disk are created only if there is
+need to, for example if rsyslog runs out of (configured) memory queue space or needs
+to shutdown (and thus persist yet unsent messages). Using main memory and going to the
+disk when needed is a huge performance benefit. You do not need to care about it,
+because, all of it is handled automatically and transparently by rsyslog.</p>
+<h2>How To Setup</h2>
+<p>First, you need to create a working directory for rsyslog. This is where it
+stores its queue files (should need arise). You may use any location on your 
+local system.
+<p>Next, you need to do is instruct rsyslog to use a 
+disk queue and then configure your action. There is nothing else to do. With the 
+following simple config file, you forward anything you receive to a remote server
+and have buffering applied automatically when it goes down. This must be done on the
+client machine.</p>
+<textarea rows="9" cols="80">
+$ModLoad imuxsock             # local message reception
+
+$WorkDirectory /rsyslog/work  # default location for work (spool) files
+
+$ActionQueueType LinkedList   # use asynchronous processing
+$ActionQueueFileName srvrfwd  # set file name, also enables disk mode
+$ActionResumeRetryCount -1    # infinite retries on insert failure
+$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
+*.*       @@server:port
+</textarea>
+<p>The port given above is optional. It may not be specified, in which case you only
+provide the server name. The "$ActionQueueFileName" is used to create queue files, should need
+arise. This value must be unique inside rsyslog.conf. No two rules must use the same queue file.
+Also, for obvious reasons, it must only contain those characters that can be used inside a
+valid file name. Rsyslog possibly adds some characters in front and/or at the end of that name
+when it creates files. So that name should not be at the file size name length limit (which
+should not be a problem these days).
+<p>Please note that actual spool files are only created if the remote server is down
+<b>and</b> there is no more space in the in-memory queue. By default, a short failure
+of the remote server will never result in the creation of a disk file as a couple of
+hundered messages can be held in memory by default. [These parameters can be fine-tuned. However,
+then you need to either fully understand how the queue works
+(<a href="http://www.rsyslog.com/doc-queues.html">read elaborate doc</a>) or
+use <a href="http://www.rsyslog.com/doc-professional_support.html">professional services</a>
+to have it done based on
+your specs ;) - what that means is that fine-tuning queue parameters is far from
+being trivial...]
+<p>If you would like to test if your buffering scenario works, you need to 
+stop, wait a while and restart you central server. Do <b>not</b> watch for files being created,
+as this usually does not happen and never happens immediately.
+
+<h3>Forwarding to More than One Server</h3>
+<p>If you have more than one server you would like to forward to, that's quickly done.
+Rsyslog has no limit on the number or type of actions, so you can define as many targets
+as you like. What is important to know, however, is that the full set of directives make
+up an action. So you can not simply add (just) a second forwarding rule, but need to
+duplicate the rule configuration as well. Be careful that you use different queue
+file names for the second action, else you will mess up your system.
+<p>A sample for forwarding to two hosts looks like this:
+<p>
+<textarea rows="20" cols="80">
+$ModLoad imuxsock.so # local message reception
+
+$WorkDirectory /rsyslog/work # default location for work (spool) files
+
+# start forwarding rule 1
+$ActionQueueType LinkedList   # use asynchronous processing
+$ActionQueueFileName srvrfwd1 # set file name, also enables disk mode
+$ActionResumeRetryCount -1    # infinite retries on insert failure
+$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
+*.*       @@server1:port
+# end forwarding rule 1
+
+# start forwarding rule 2
+$ActionQueueType LinkedList   # use asynchronous processing
+$ActionQueueFileName srvrfwd2 # set file name, also enables disk mode
+$ActionResumeRetryCount -1    # infinite retries on insert failure
+$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
+*.*       @@server2
+# end forwarding rule 2
+</textarea>
+<p>Note the filename used for the first rule it is "srvrfwd1" and for the second it
+is "srvrfwd2". I have used a server without port name in the second forwarding rule.
+This was just to illustrate how this can be done. You can also specify a port there
+(or drop the port from server1).
+<p>When there are multiple action queues, they all work independently. Thus, if server1 
+goes down, server2 still receives data in real-time. The client will <b>not</b> block
+and wait for server1 to come back online. Similarily, server1's operation will not
+be affected by server2's state.
+
+<h2>Some Final Words on Reliability ...</h2>
+<p>Using plain TCP syslog provides a lot of reliability over UDP syslog. However,
+plain TCP syslog is <b>not</b> a fully reliable transport. In order to get full reliability, 
+you need to use the RELP protocol.
+<p>Folow the next link to learn more about 
+<a href="http://blog.gerhards.net/2008/04/on-unreliability-of-plain-tcp-syslog.html">the
+problems you may encounter with plain tcp syslog</a>.
+<h3>Feedback requested</h3>
+<P>I would appreciate feedback on this tutorial. If you have additional ideas, 
+comments or find bugs (I *do* bugs - no way... ;)), please
+<a href="mailto:rgerhards@adiscon.com">let me know</a>.</P>
+<h2>Revision History</h2>
+<ul>
+	<li>2008-06-27 * 
+	<a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> * Initial Version created</li>
+</ul>
+<h2>Copyright</h2>
+<p>Copyright (c) 2008 
+<a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/en/">Adiscon</a>.</p>
+<p>      Permission is granted to copy, distribute and/or modify this document
+      under the terms of the GNU Free Documentation License, Version 1.2
+      or any later version published by the Free Software Foundation;
+      with no Invariant Sections, no Front-Cover Texts, and no Back-Cover
+	 Texts.  A copy of the license can be viewed at
+<a href="http://www.gnu.org/copyleft/fdl.html">
+http://www.gnu.org/copyleft/fdl.html</a>.</p>
+</body>
+</html>
diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c
index 6573c46a..82fca2db 100644
--- a/plugins/omgssapi/omgssapi.c
+++ b/plugins/omgssapi/omgssapi.c
@@ -535,7 +535,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	/* extract the host first (we do a trick - we replace the ';' or ':' with a '\0')
 	 * now skip to port and then template name. rgerhards 2005-07-06
 	 */
-	for(q = p ; *p && *p != ';' && *p != ':' ; ++p)
+	for(q = p ; *p && *p != ';' && *p != ':' && *p != '#' ; ++p)
 		/* JUST SKIP */;
 
 	pData->port = NULL;
@@ -559,6 +559,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 		}
 	}
 	
+		
 	/* now skip to template */
 	bErr = 0;
 	while(*p && *p != ';') {
@@ -574,10 +575,11 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	}
 
 	/* TODO: make this if go away! */
-	if(*p == ';') {
+	if(*p == ';' || *p == '#' || isspace(*p)) {
+		uchar cTmp = *p;
 		*p = '\0'; /* trick to obtain hostname (later)! */
 		CHKmalloc(pData->f_hname = strdup((char*) q));
-		*p = ';';
+		*p = cTmp;
 	} else {
 		CHKmalloc(pData->f_hname = strdup((char*) q));
 	}
diff --git a/runtime/conf.c b/runtime/conf.c
index a0a2b080..71b2b2da 100644
--- a/runtime/conf.c
+++ b/runtime/conf.c
@@ -480,36 +480,33 @@ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEn
 {
 	uchar *p;
 	uchar *tplName;
-	DEFiRet;
 	cstr_t *pStrB;
+	DEFiRet;
 
 	ASSERT(pp != NULL);
 	ASSERT(*pp != NULL);
 	ASSERT(pOMSR != NULL);
 
 	p =*pp;
-	/* a template must follow - search it and complain, if not found
-	 */
+	/* a template must follow - search it and complain, if not found */
 	skipWhiteSpace(&p);
 	if(*p == ';')
 		++p; /* eat it */
 	else if(*p != '\0' && *p != '#') {
-		errmsg.LogError(0, NO_ERRCODE, "invalid character in selector line - ';template' expected");
-		iRet = RS_RET_ERR;
-		goto finalize_it;
+		errmsg.LogError(0, RS_RET_ERR, "invalid character in selector line - ';template' expected");
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	skipWhiteSpace(&p); /* go to begin of template name */
 
-	if(*p == '\0') {
+	if(*p == '\0' || *p == '#') {
 		/* no template specified, use the default */
 		/* TODO: check NULL ptr */
 		tplName = (uchar*) strdup((char*)dfltTplName);
 	} else {
 		/* template specified, pick it up */
 		if(rsCStrConstruct(&pStrB) != RS_RET_OK) {
-			iRet = RS_RET_OUT_OF_MEMORY;
-			goto finalize_it;
+			ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
 		}
 
 		/* now copy the string */
diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c
index a80a1836..e670da13 100644
--- a/runtime/nsd_gtls.c
+++ b/runtime/nsd_gtls.c
@@ -500,7 +500,7 @@ gtlsRecordRecv(nsd_gtls_t *pThis)
 	}
 
 finalize_it:
-	dbgprintf("gtlsRecordRecv return. nsd %p, iRet %d, lenRcvd %d, lenRcvBuf %d, ptrRcvBuf %d\n", pThis, iRet, lenRcvd, pThis->lenRcvBuf, pThis->ptrRcvBuf);
+	dbgprintf("gtlsRecordRecv return. nsd %p, iRet %d, lenRcvd %d, lenRcvBuf %d, ptrRcvBuf %d\n", pThis, iRet, (int) lenRcvd, pThis->lenRcvBuf, pThis->ptrRcvBuf);
 	RETiRet;
 }
 
diff --git a/tools/omfwd.c b/tools/omfwd.c
index fd326553..715457c9 100644
--- a/tools/omfwd.c
+++ b/tools/omfwd.c
@@ -555,7 +555,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	/* extract the host first (we do a trick - we replace the ';' or ':' with a '\0')
 	 * now skip to port and then template name. rgerhards 2005-07-06
 	 */
-	for(q = p ; *p && *p != ';' && *p != ':' ; ++p)
+	for(q = p ; *p && *p != ';' && *p != ':' && *p != '#' ; ++p)
 		/* JUST SKIP */;
 
 	pData->port = NULL;
@@ -579,30 +579,22 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	
 	/* now skip to template */
 	bErr = 0;
-	while(*p && *p != ';') {
-		if(*p && *p != ';' && !isspace((int) *p)) {
-			if(bErr == 0) { /* only 1 error msg! */
-				bErr = 1;
-				errno = 0;
-				errmsg.LogError(0, NO_ERRCODE, "invalid selector line (port), probably not doing "
-					 "what was intended");
-			}
-		}
-		++p;
-	}
+	while(*p && *p != ';'  && *p != '#' && !isspace((int) *p))
+		++p; /*JUST SKIP*/
 
 	/* TODO: make this if go away! */
-	if(*p == ';') {
+	if(*p == ';' || *p == '#' || isspace(*p)) {
+		uchar cTmp = *p;
 		*p = '\0'; /* trick to obtain hostname (later)! */
 		CHKmalloc(pData->f_hname = strdup((char*) q));
-		*p = ';';
+		*p = cTmp;
 	} else {
 		CHKmalloc(pData->f_hname = strdup((char*) q));
 	}
 
 	/* process template */
 	CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-			(pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName));
+		(pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName));
 
 	if(pData->protocol == FORW_TCP) {
 		/* create our tcpclt */
author	Rainer Gerhards <rgerhards@adiscon.com>	2008-07-01 15:09:07 +0200
committer	Rainer Gerhards <rgerhards@adiscon.com>	2008-07-01 15:09:07 +0200
commit	4fd98529572b57f494597a34f04ced96ef1031a2 (patch)
tree	a2abace73c6cfcfa9db49d4c04ed2a1e831cef74
parent	78543b7e31ea9559108d15fd645862db7dd63913 (diff)
parent	a16cbe655541421fefdcc852087219228bbc5c94 (diff)
download	rsyslog-4fd98529572b57f494597a34f04ced96ef1031a2.tar.gz rsyslog-4fd98529572b57f494597a34f04ced96ef1031a2.tar.xz rsyslog-4fd98529572b57f494597a34f04ced96ef1031a2.zip