added $Escape8BitCharactersOnReceive directive

Thanks to David Lang for suggesting this functionality.
author: Rainer Gerhards <rgerhards@adiscon.com> 2010-01-27 15:13:54 +0100
committer: Rainer Gerhards <rgerhards@adiscon.com> 2010-01-27 15:13:54 +0100
commit: 85045270f69e4dcb25c409c9661e96e3172d7f30 (patch)
tree: 6cc6c2ae9d93810398535c5ae4bbe9e3afb4c52a
parent: 4f00d1c91f60350026c4f6bc6b16d2918c729915 (diff)
download: rsyslog-85045270f69e4dcb25c409c9661e96e3172d7f30.tar.gz
rsyslog-85045270f69e4dcb25c409c9661e96e3172d7f30.tar.xz
rsyslog-85045270f69e4dcb25c409c9661e96e3172d7f30.zip
10 files changed, 83 insertions, 0 deletions
diff --git a/ChangeLog b/ChangeLog
index 9f8c4a25..7a5a6099 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.5.2  [DEVEL] (rgerhards), 2009-11-??
+- added $Escape8BitCharactersOnReceive directive
+  Thanks to David Lang for suggesting it.
 - worked around an issue where omfile failed to compile on32 bit platforms
   under some circumstances (this smells like a gcc problem, but a simple
   solution was available). Thanks to Kenneth Marshall for some advice.
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 2d451102..30f5771a 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -74,6 +74,7 @@ html_files = \
 	rsconf1_dropmsgswithmaliciousdnsptrrecords.html \
 	rsconf1_droptrailinglfonreception.html \
 	rsconf1_dynafilecachesize.html \
+	rsconf1_escape8bitcharactersonreceive.html \
 	rsconf1_escapecontrolcharactersonreceive.html \
 	rsconf1_failonchownfailure.html \
 	rsconf1_filecreatemode.html \
diff --git a/doc/rsconf1_escape8bitcharsonreceive.html b/doc/rsconf1_escape8bitcharsonreceive.html
new file mode 100644
index 00000000..408851c1
--- /dev/null
+++ b/doc/rsconf1_escape8bitcharsonreceive.html
@@ -0,0 +1,44 @@
+<html>
+<head>
+<title>rsyslog.conf file</title>
+</head>
+<body>
+<a href="rsyslog_conf_global.html">back</a>
+
+<h2>$Escape8BitCharactersOnReceive</h2>
+<p><b>Type:</b> global configuration directive</p>
+<p><b>Default:</b> off</p>
+<p><b>Available Since:</b> 5.5.2</p>
+<p><b>Description:</b></p>
+<p>This directive instructs rsyslogd to replace non US-ASCII characters (those that
+have the 8th bit set) during reception of the message.
+This may be useful for some systems.
+Please note that this escaping breaks Unicode and many other encodings. Most importantly,
+it can be assumed that Asian and European characters will be rendered hardly readable by 
+this settings. However, it may still be useful when the logs themself are primarily
+in English and only occasionally contain local script.
+If this option is turned on, all control-characters are converted to a 3-digit octal number and be prefixed with the $ControlCharacterEscapePrefix character (being '#' by default).
+<p><b>Warning:</b></p>
+<ul>
+	<li>turning on this option most probably destroys non-western character sets 
+	(like Japanese, Chinese and Korean) as well as European character sets.</li>
+	<li>turning on this option destroys digital signatures if such exists inside 
+	the message</li>
+	<li>if turned on, the drop-cc, space-cc and escape-cc
+	<a href="property_replacer.html">property replacer</a> options do not work 
+	as expected because control characters are already removed upon message 
+	reception. If you intend to use these property replacer options, you must 
+	turn off $Escape8BitCharactersOnReceive.</li>
+</ul>
+<p><b>Sample:</b></p>
+<p><code><b>$Escape8BitCharactersOnReceive on</b></code></p>
+
+<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual 
+index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
+<p><font size="2">This documentation is part of the
+<a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
+Copyright &copy; 2010 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL 
+version 3 or higher.</font></p>
+</body>
+</html>
diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html
index 8796de65..3e926db1 100644
--- a/doc/rsyslog_conf_global.html
+++ b/doc/rsyslog_conf_global.html
@@ -140,6 +140,7 @@ our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a
 <li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li>
 <li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li>
 <li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li>
+<li><a href="rsconf1_escape8bitcharsonreceive.html">$Escape8BitCharactersOnReceive</a></li>
 <li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li>
 <li><b>$EscapeControlCharactersOnReceive</b> [<b>on</b>|off] - escape USASCII HT character</li>
 <li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li>
diff --git a/runtime/parser.c b/runtime/parser.c
index b5245795..cbeb01e2 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -60,6 +60,7 @@ DEFobjCurrIf(ruleset)
 /* config data */
 static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */
 static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */
+static int bEscape8BitChars = 0; /* escape characters > 127 on reception: 0 - no, 1 - yes */
 static int bEscapeTab = 1;	/* escape tab control character when doing CC escapes: 0 - no, 1 - yes */
 static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */
 
@@ -353,6 +354,9 @@ SanitizeMsg(msg_t *pMsg)
 				bNeedSanitize = 1;
 				break;
 			}
+		} else if(pszMsg[iSrc] > 127 && bEscape8BitChars) {
+			bNeedSanitize = 1;
+			break;
 		}
 	}
 
@@ -387,6 +391,14 @@ SanitizeMsg(msg_t *pMsg)
 				pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3);
 				pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007));
 			}
+		} else if(pszMsg[iSrc] > 127 && bEscape8BitChars) {
+			/* In this case, we also do the conversion. Note that this most
+			 * probably breaks European languages. -- rgerhards, 2010-01-27
+			 */
+			pDst[iDst++] = cCCEscapeChar;
+			pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6);
+			pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3);
+			pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007));
 		} else {
 			pDst[iDst++] = pszMsg[iSrc];
 		}
@@ -625,6 +637,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus
 {
 	cCCEscapeChar = '#';
 	bEscapeCCOnRcv = 1; /* default is to escape control characters */
+	bEscape8BitChars = 0; /* default is to escape control characters */
 	bEscapeTab = 1; /* default is to escape control characters */
 	bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */
 
@@ -677,6 +690,7 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */
 	CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"escape8bitcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscape8BitChars, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactertab", 0, eCmdHdlrBinary, NULL, &bEscapeTab, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL));
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 1a7eba71..3a59f64d 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -97,12 +97,16 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   testsuites/master.tspgsql \
 	   testsuites/subsecond.conf \
 	   testsuites/master.subsecond \
+	   testsuites/parse_8bit_escape.conf \
+	   testsuites/8bit.parse_8bit_escape.conf \
 	   testsuites/parse1.conf \
 	   testsuites/field1.conf \
 	   testsuites/1.parse1 \
 	   testsuites/2.parse1 \
 	   testsuites/3.parse1 \
 	   testsuites/4.parse1 \
+	   testsuites/8bit.parse1 \
+	   testsuites/empty.parse1 \
 	   testsuites/oversizeTag-1.parse1 \
 	   testsuites/date1.parse1 \
 	   testsuites/date2.parse1 \
diff --git a/tests/parsertest.sh b/tests/parsertest.sh
index 470d9375..c7e3c67e 100755
--- a/tests/parsertest.sh
+++ b/tests/parsertest.sh
@@ -4,10 +4,14 @@ source $srcdir/diag.sh nettester parse1 udp
 source $srcdir/diag.sh nettester parse1 tcp
 source $srcdir/diag.sh nettester parse2 udp
 source $srcdir/diag.sh nettester parse2 tcp
+source $srcdir/diag.sh nettester parse_8bit_escape udp
+source $srcdir/diag.sh nettester parse_8bit_escape tcp
 
 echo \[parsertest.sh]: redoing tests in IPv4-only mode
 source $srcdir/diag.sh nettester parse1 udp -4
 source $srcdir/diag.sh nettester parse1 tcp -4
 source $srcdir/diag.sh nettester parse2 udp -4
 source $srcdir/diag.sh nettester parse2 tcp -4
+source $srcdir/diag.sh nettester parse_8bit_escape udp -4
+source $srcdir/diag.sh nettester parse_8bit_escape tcp -4
 source $srcdir/diag.sh init
diff --git a/tests/testsuites/8bit.parse1 b/tests/testsuites/8bit.parse1
new file mode 100644
index 00000000..90db6352
--- /dev/null
+++ b/tests/testsuites/8bit.parse1
@@ -0,0 +1,2 @@
+<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü
+6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: äöü
diff --git a/tests/testsuites/8bit.parse_8bit_escape b/tests/testsuites/8bit.parse_8bit_escape
new file mode 100644
index 00000000..b2f6335c
--- /dev/null
+++ b/tests/testsuites/8bit.parse_8bit_escape
@@ -0,0 +1,2 @@
+<6>AUG 10 22:18:24 host tag This msg contains 8-bit European chars: äöü
+6,kern,info,Aug 10 22:18:24,host,tag,tag, This msg contains 8-bit European chars: #303#244#303#266#303#274
diff --git a/tests/testsuites/parse_8bit_escape.conf b/tests/testsuites/parse_8bit_escape.conf
new file mode 100644
index 00000000..0598f33f
--- /dev/null
+++ b/tests/testsuites/parse_8bit_escape.conf
@@ -0,0 +1,9 @@
+$ModLoad ../plugins/omstdout/.libs/omstdout
+$IncludeConfig nettest.input.conf	# This picks the to be tested input from the test driver!
+
+$ErrorMessagesToStderr off
+$Escape8BitCharactersOnReceive on
+
+# use a special format that we can easily parse in expect
+$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n"
+*.* :omstdout:;expect
author	Rainer Gerhards <rgerhards@adiscon.com>	2010-01-27 15:13:54 +0100
committer	Rainer Gerhards <rgerhards@adiscon.com>	2010-01-27 15:13:54 +0100
commit	85045270f69e4dcb25c409c9661e96e3172d7f30 (patch)
tree	6cc6c2ae9d93810398535c5ae4bbe9e3afb4c52a
parent	4f00d1c91f60350026c4f6bc6b16d2918c729915 (diff)
download	rsyslog-85045270f69e4dcb25c409c9661e96e3172d7f30.tar.gz rsyslog-85045270f69e4dcb25c409c9661e96e3172d7f30.tar.xz rsyslog-85045270f69e4dcb25c409c9661e96e3172d7f30.zip