Merge branch 'lognorm' + some doc added

Conflicts:
	ChangeLog

29 files changed, 705 insertions, 38 deletions

diff --git a/ChangeLog b/ChangeLog
index 4a68a934..95caa839 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,24 @@
 ---------------------------------------------------------------------------
-Version 6.1.2  [DEVEL] (rgerhards), 2010-11-??
+Version 6.1.2  [DEVEL] (rgerhards), 2010-12-13
+- added experimental support for log normalizaton (via liblognorm)
+  support for normalizing log messages has been added in the form of
+  mmnormalize. The core engine (property replacer, filter engine) has
+  been enhanced to support properties from normalized events.
+  Note: this is EXPERIMENTAL code. It is currently know that
+  there are issues if the functionality is used with
+  - disk-based queues
+  - asynchronous action queues
+  You can not use the new functionality together with these features.
+  This limitation will be removed in later releases. However, we 
+  preferred to release early, so that one can experiment with the new
+  feature set and accepted the price that this means the full set of
+  functionality is not yet available. If not used together with
+  these features, log normalizing should be pretty stable.
 - enhanced testing tool tcpflood
   now supports sending via UDP and the capability to run multiple
   iterations and generate statistics data records
+- bugfix: potential abort when output modules with different parameter
+  passing modes were used in configured output modules
 ---------------------------------------------------------------------------
 Version 6.1.1  [DEVEL] (rgerhards), 2010-11-30
 - bugfix(important): problem in TLS handling could cause rsyslog to loop
diff --git a/Makefile.am b/Makefile.am
index 680a819e..bdbc2794 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -179,6 +179,10 @@ if ENABLE_RFC3195
 SUBDIRS += plugins/im3195
 endif
 
+if ENABLE_MMNORMALIZE
+SUBDIRS += plugins/mmnormalize
+endif
+
 if ENABLE_ORACLE
 SUBDIRS += plugins/omoracle
 endif
diff --git a/action.c b/action.c
index 8ad72975..999dc420 100644
--- a/action.c
+++ b/action.c
@@ -672,6 +672,8 @@ static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg, uchar **pp
 	for(i = 0 ; i < pAction->iNumTpls ; ++i) {
 		switch(pAction->eParamPassing) {
 			case ACT_STRING_PASSING:
+				if(ppMsgs[i] == NULL)
+					lenMsgs[i] = 0;
 				CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]), &lenMsgs[i]));
 				break;
 			case ACT_ARRAY_PASSING:
@@ -775,13 +777,20 @@ finalize_it:
 	 */
 	switch(pThis->eParamPassing) {
 	case ACT_STRING_PASSING:
-		/* nothing to do in that case */
+		/* nothing to do in this case */
+		/* TODO: find a better way of handling this situation, as it
+		 * costs performance to delete this array each time.
+		 */
+		for(i = 0 ; i < pThis->iNumTpls ; ++i) {
+			free(((uchar**)actParams)[i]);
+			((uchar**)actParams)[i] = NULL;
+		}
 		break;
 	case ACT_ARRAY_PASSING:
 		cleanupDoActionParams(pThis, actParams); /* iRet ignored! */
 		break;
 	case ACT_MSG_PASSING:
-		/* nothing to do in that case */
+		/* (almost) nothing to do in that case */
 		for(i = 0 ; i < pThis->iNumTpls ; ++i) {
 			((uchar**)actParams)[i] = NULL;
 		}
diff --git a/configure.ac b/configure.ac
index 10a98b0d..d5bb1d4b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -31,6 +31,10 @@ AC_CANONICAL_HOST
 
 PKG_PROG_PKG_CONFIG
 
+# modules we require
+PKG_CHECK_MODULES(LIBESTR, libestr >= 0.1.0)
+PKG_CHECK_MODULES(LIBEE, libee >= 0.1.0)
+
 case "${host}" in
   *-*-linux*)
     os_type="linux"
@@ -714,6 +718,24 @@ AC_ARG_ENABLE(imdiag,
 AM_CONDITIONAL(ENABLE_IMDIAG, test x$enable_imdiag = xyes)
 
 
+# mmnormalize
+AC_ARG_ENABLE(mmnormalize,
+        [AS_HELP_STRING([--enable-mmnormalize],[Enable building mmnormalize support @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_mmnormalize="yes" ;;
+          no) enable_mmnormalize="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-mmnormalize) ;;
+         esac],
+        [enable_mmnormalize=no]
+)
+if test "x$enable_mmnormalize" = "xyes"; then
+	PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 0.1.0)
+fi
+AM_CONDITIONAL(ENABLE_MMNORMALIZE, test x$enable_mmnormalize = xyes)
+AC_SUBST(LOGNORM_CFLAGS)
+AC_SUBST(LOGNORM_LIBS)
+
+
 # RELP support
 AC_ARG_ENABLE(relp,
         [AS_HELP_STRING([--enable-relp],[Enable RELP support @<:@default=no@:>@])],
@@ -1072,6 +1094,7 @@ AC_CONFIG_FILES([Makefile \
 		plugins/omsnmp/Makefile \
 		plugins/omoracle/Makefile \
 		plugins/omudpspoof/Makefile \
+		plugins/mmnormalize/Makefile \
 		plugins/cust1/Makefile \
 		java/Makefile \
 		tests/Makefile])
@@ -1114,6 +1137,9 @@ echo "---{ parser modules }---"
 echo "    pmrfc3164sd module will be compiled:      $enable_pmrfc3164sd"
 echo "    pmlastmsg module will be compiled:        $enable_pmlastmsg"
 echo
+echo "---{ message modification modules }---"
+echo "    mmnormalize module will be compiled:      $enable_mmnormalize"
+echo
 echo "---{ database support }---"
 echo "    MySql support enabled:                    $enable_mysql"
 echo "    libdbi support enabled:                   $enable_libdbi"
diff --git a/doc/rscript_abnf.html b/doc/rscript_abnf.html
index d60edb5c..9172d945 100644
--- a/doc/rscript_abnf.html
+++ b/doc/rscript_abnf.html
@@ -21,7 +21,58 @@ and many other languages).</p>
 <p>Below is the formal language definitionin ABNF (RFC 2234)
 format: <br>
 </p>
-<pre>; <span style="font-weight: bold;">all of this is a working document and may change!</span> -- rgerhards, 2008-02-24<br><br>script    := *stmt<br>stmt      := (if_stmt / block / vardef / run_s / load_s)<br>vardef    := "var" ["scope" = ("global" / "event")] <br>block     := "begin" stmt "end"<br>load_s    := "load" constraint ("module") modpath params ; load mod only if expr is true<br>run_s     := "run" constraint ("input") name<br>constraint:= "if" expr ; constrains some one-time commands<br>modpath   := expr<br>params    := ["params" *1param *("," param) "endparams"]<br>param     := paramname) "=" expr<br>paramname := [*(obqualifier ".") name]<br>modpath:= ; path to module<br>?line? := cfsysline / cfli<br>cfsysline:= BOL "$" *char EOL ; how to handle the first line? (no EOL in front!)<br>BOL := ; Begin of Line - implicitely set on file beginning and after each EOL<br>EOL := 0x0a ;LF<br>if_stmt := "if" expr "then"<br>old_filter:= BOL facility "." severity ; no whitespace allowed between BOL and facility!<br>facility := "*" / "auth" / "authpriv" / "cron" / "daemon" / "kern" / "lpr" / <br> "mail" / "mark" / "news" / "security" / "syslog" / "user" / "uucp" / <br> "local0" .. "local7" / "mark"<br>	; The keyword security should not be used anymore<br>	; mark is just internal<br>severity := TBD ; not really relevant in this context<br><br>; and now the actual expression<br>expr := e_and *("or" e_and)<br>e_and := e_cmp *("and" e_cmp)<br>e_cmp := val 0*1(cmp_op val)<br>val := term *(("+" / "-" / "&amp;") term)<br>term := factor *(("*" / "/" / "%") factor)<br>factor := ["not"] ["-"] terminal<br>terminal := var / constant / function / ( "(" expr ")" )<br>function := name "(" *("," expr) ")"<br>var := "$" varname<br>varname := msgvar / sysvar<br>msgvar := name<br>sysvar := "$" name<br>name := alpha *(alnum)<br>constant := string / number<br>string := simpstr / tplstr ; tplstr will be implemented in next phase<br>simpstr := "'" *char "'" ; use your imagination for char ;)<br>tplstr := '"' template '"' ; not initially implemented<br>number := ["-"] 1*digit ; 0nn = octal, 0xnn = hex, nn = decimal<br>cmp_op := "==" / "!=" / "&lt;&gt;" / "&lt;" / "&gt;" / "&lt;=" / "&gt;=" / "contains" / "contains_i" / "startswith" / "startswith_i"<br>digit := %x30-39<br>alpha := "a" ... "z" # all letters<br>alnum :* alpha / digit / "_" /"-" # "-" necessary to cover currently-existing message properties<br></pre>
+<pre>; <span style="font-weight: bold;">all of this is a working document and may change!</span> -- rgerhards, 2008-02-24<br>
+<br>
+script    := *stmt<br>
+stmt      := (if_stmt / block / vardef / run_s / load_s)<br>
+vardef    := "var" ["scope" = ("global" / "event")] <br>
+block     := "begin" stmt "end"<br>
+load_s    := "load" constraint ("module") modpath params ; load mod only if expr is true<br>
+run_s     := "run" constraint ("input") name<br>
+constraint:= "if" expr ; constrains some one-time commands<br>
+modpath   := expr<br>
+params    := ["params" *1param *("," param) "endparams"]<br>
+param     := paramname) "=" expr<br>
+paramname := [*(obqualifier ".") name]<br>
+modpath:= ; path to module<br>
+?line? := cfsysline / cfli<br>
+cfsysline:= BOL "$" *char EOL ; how to handle the first line? (no EOL in front!)<br>
+BOL := ; Begin of Line - implicitely set on file beginning and after each EOL<br>
+EOL := 0x0a ;LF<br>
+if_stmt := "if" expr "then"<br>
+old_filter:= BOL facility "." severity ; no whitespace allowed between BOL and facility!<br>
+facility := "*" / "auth" / "authpriv" / "cron" / "daemon" / "kern" / "lpr" / <br>
+"mail" / "mark" / "news" / "security" / "syslog" / "user" / "uucp" / <br>
+"local0" .. "local7" / "mark"<br>
+; The keyword security should not be used anymore<br>
+; mark is just internal<br>
+severity := TBD ; not really relevant in this context<br>
+<br>
+; and now the actual expression<br>
+expr := e_and *("or" e_and)<br>
+e_and := e_cmp *("and" e_cmp)<br>
+e_cmp := val 0*1(cmp_op val)<br>
+val := term *(("+" / "-" / "&amp;") term)<br>
+term := factor *(("*" / "/" / "%") factor)<br>
+factor := ["not"] ["-"] terminal<br>
+terminal := var / constant / function / ( "(" expr ")" )<br>
+function := name "(" *("," expr) ")"<br>
+var := "$" varname<br>
+varname := msgvar / sysvar / ceevar<br>
+msgvar := name<br>
+ceevar := "!" name<br>
+sysvar := "$" name<br>
+name := alpha *(alnum)<br>
+constant := string / number<br>
+string := simpstr / tplstr ; tplstr will be implemented in next phase<br>
+simpstr := "'" *char "'" ; use your imagination for char ;)<br>
+tplstr := '"' template '"' ; not initially implemented<br>
+number := ["-"] 1*digit ; 0nn = octal, 0xnn = hex, nn = decimal<br>
+cmp_op := "==" / "!=" / "&lt;&gt;" / "&lt;" / "&gt;" / "&lt;=" / "&gt;=" / "contains" / "contains_i" / "startswith" / "startswith_i"<br>
+digit := %x30-39<br>
+alpha := "a" ... "z" # all letters<br>
+alnum :* alpha / digit / "_" /"-" # "-" necessary to cover currently-existing message properties<br>
+</pre>
 <h2>Samples</h2>
 <p>Some samples of RainerScript:</p><p>define function IsLinux<br>begin<br>&nbsp; &nbsp; if $environ contains "linux" then return true else return false<br>end</p><p>load if IsLinux() 'imklog.so' params name='klog' endparams /* load klog under linux only */<br>run if IsLinux() input 'klog'<br>load 'ommysql.so'</p><p>if $message contains "error" then<br>&nbsp; action<br>&nbsp;&nbsp;&nbsp; type='ommysql.so', queue.mode='disk', queue.highwatermark = 300,<br>&nbsp; &nbsp; action.dbname='events', action.dbuser='uid',<br>&nbsp;
 &nbsp; [?action.template='templatename'?] or [?action.sql='insert into
diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html
index 63c29817..27f4a8a6 100644
--- a/doc/rsyslog_conf_filter.html
+++ b/doc/rsyslog_conf_filter.html
@@ -117,6 +117,13 @@ currently supported:</p>
 the property. There must be an exact match, wildcards are not supported.</td>
 </tr>
 <tr>
+<td>isempty</td>
+<td>Checks if the property is empty. The value is discarded. This is
+especially useful when working with normalized data, where some fields
+may be populated based on normalization result.
+Available since 6.6.2.
+</tr>
+<tr>
 <td>isequal</td>
 <td>Compares the "value" string provided and the property
 contents. These two values must be exactly equal to match. The
diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html
index 74aa319c..7623c114 100644
--- a/doc/rsyslog_conf_modules.html
+++ b/doc/rsyslog_conf_modules.html
@@ -93,12 +93,17 @@ repated n times" messages emitted by some syslogds.
 They can be implemented using either the output module or the parser module interface.
 From the rsyslog core's point of view, they actually are output or parser modules, it is their
 implementation that makes them special.
-<p>Currently, there do not exist any such modules, but could be written with
+<p>Currently, there exists only a limited set of such modules, but new ones could be written with
 the methods the engine provides. They could be used, for example, to:
 <ul>
 <li>anonymize message content
 <li>add dynamically computed content to message (fields)
 </ul>
+<p>Currently supported modules are:
+<ul>
+<li><a href="mmnormalize.html">mmnormalize</a> - used to normalize
+log messages.
+</ul>
 
 <a name="lm"></a><h2>String Generator Modules</h2>
 <p>String generator modules are used, as the name implies, to generate strings based
diff --git a/parse.c b/parse.c
index e335d423..a156b317 100644
--- a/parse.c
+++ b/parse.c
@@ -231,7 +231,8 @@ rsRetVal parsSkipWhitespace(rsParsObj *pThis)
 /* Parse string up to a delimiter.
  *
  * Input:
- * cDelim - the delimiter
+ * cDelim - the delimiter. Note that SP within a value always is a delimiter,
+ * so cDelim is actually an *additional* delimiter.
  *   The following two are for whitespace stripping,
  *   0 means "no", 1 "yes"
  *   - bTrimLeading
@@ -256,13 +257,13 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim
 
 	pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos;
 
-	while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != cDelim) {
+	while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != cDelim && *pC != ' ') {
 		CHKiRet(cstrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC));
 		++pThis->iCurrPos;
 		++pC;
 	}
 	
-	if(*pC == cDelim) {
+	if(pThis->iCurrPos < cstrLen(pThis->pCStr)) { //BUGFIX!!
 		++pThis->iCurrPos; /* eat delimiter */
 	}
 
diff --git a/plugins/mmnormalize/Makefile.am b/plugins/mmnormalize/Makefile.am
new file mode 100644
index 00000000..0a3b5ba5
--- /dev/null
+++ b/plugins/mmnormalize/Makefile.am
@@ -0,0 +1,8 @@
+pkglib_LTLIBRARIES = mmnormalize.la
+
+mmnormalize_la_SOURCES = mmnormalize.c
+mmnormalize_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBLOGNORM_CFLAGS) $(LIBEE_CFLAGS)
+mmnormalize_la_LDFLAGS = -module -avoid-version $(LIBLOGNORM_LIBS) $(LIBEE_LIBS)
+mmnormalize_la_LIBADD = 
+
+EXTRA_DIST = 
diff --git a/plugins/mmnormalize/mmnormalize.c b/plugins/mmnormalize/mmnormalize.c
new file mode 100644
index 00000000..49b143b6
--- /dev/null
+++ b/plugins/mmnormalize/mmnormalize.c
@@ -0,0 +1,273 @@
+/* mmnormalize.c
+ * This is a message modification module. It normalizes the input message with
+ * the help of liblognorm. The messages EE event structure is updated.
+ *
+ * NOTE: read comments in module-template.h for details on the calling interface!
+ *
+ * File begun on 2010-01-01 by RGerhards
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <signal.h>
+#include <errno.h>
+#include <unistd.h>
+#include <libestr.h>
+#include <libee/libee.h>
+#include <liblognorm.h>
+#include "conf.h"
+#include "syslogd-types.h"
+#include "template.h"
+#include "module-template.h"
+#include "errmsg.h"
+#include "cfsysline.h"
+#include "dirty.h"
+
+MODULE_TYPE_OUTPUT
+
+static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal);
+
+/* static data */
+DEFobjCurrIf(errmsg);
+
+/* internal structures
+ */
+DEF_OMOD_STATIC_DATA
+
+typedef struct _instanceData {
+	sbool bUseRawMsg;	/**< use %rawmsg% instead of %msg% */
+	ln_ctx ctxln;		/**< context to be used for liblognorm */
+	ee_ctx ctxee;		/**< context to be used for libee */
+} instanceData;
+
+typedef struct configSettings_s {
+	uchar *rulebase;		/**< name of sample db to use */
+	sbool bUseRawMsg;	/**< use %rawmsg% instead of %msg% */
+} configSettings_t;
+
+SCOPING_SUPPORT; /* must be set AFTER configSettings_t is defined */
+
+BEGINinitConfVars		/* (re)set config variables to default values */
+CODESTARTinitConfVars 
+	resetConfigVariables(NULL, NULL);
+ENDinitConfVars
+
+
+BEGINcreateInstance
+CODESTARTcreateInstance
+ENDcreateInstance
+
+
+BEGINisCompatibleWithFeature
+CODESTARTisCompatibleWithFeature
+ENDisCompatibleWithFeature
+
+
+BEGINfreeInstance
+CODESTARTfreeInstance
+	ee_exitCtx(pData->ctxee);
+	ln_exitCtx(pData->ctxln);
+ENDfreeInstance
+
+
+BEGINdbgPrintInstInfo
+CODESTARTdbgPrintInstInfo
+	dbgprintf("mmnormalize\n");
+ENDdbgPrintInstInfo
+
+
+BEGINtryResume
+CODESTARTtryResume
+ENDtryResume
+
+BEGINdoAction
+	msg_t *pMsg;
+	es_str_t *str;
+	uchar *buf;
+	int len;
+	int r;
+CODESTARTdoAction
+	pMsg = (msg_t*) ppString[0];
+	/* note that we can performance-optimize the interface, but this also
+	 * requires changes to the libraries. For now, we accept message
+	 * duplication. -- rgerhards, 2010-12-01
+	 */
+	if(pData->bUseRawMsg) {
+		getRawMsg(pMsg, &buf, &len);
+	} else {
+		buf = getMSG(pMsg);
+		len = getMSGLen(pMsg);
+	}
+	str = es_newStrFromCStr((char*)buf, len);
+	r = ln_normalize(pData->ctxln, str, &pMsg->event);
+	if(r != 0) {
+		DBGPRINTF("error %d during ln_normalize\n", r);
+	}
+	es_deleteStr(str);
+	/***DEBUG***/ // TODO: remove after initial testing - 2010-12-01
+			{
+			char *cstr;
+			ee_fmtEventToJSON(pMsg->event, &str);
+			cstr = es_str2cstr(str, NULL);
+			dbgprintf("mmnormalize generated: %s\n", cstr);
+			free(cstr);
+			es_deleteStr(str);
+			}
+	/***END DEBUG***/
+ENDdoAction
+
+
+BEGINparseSelectorAct
+CODESTARTparseSelectorAct
+CODE_STD_STRING_REQUESTparseSelectorAct(1)
+	/* first check if this config line is actually for us */
+	if(strncmp((char*) p, ":mmnormalize:", sizeof(":mmnormalize:") - 1)) {
+		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
+	}
+
+	if(cs.rulebase == NULL) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: no sample database was specified, use "
+				"$MMNormalizeSampleDB directive first!");
+		ABORT_FINALIZE(RS_RET_NO_RULESET);
+	}
+
+	/* ok, if we reach this point, we have something for us */
+	p += sizeof(":mmnormalize:") - 1; /* eat indicator sequence  (-1 because of '\0'!) */
+	CHKiRet(createInstance(&pData));
+
+	/* check if a non-standard template is to be applied */
+	if(*(p-1) == ';')
+		--p;
+	/* we call the function below because we need to call it via our interface definition. However,
+	 * the format specified (if any) is always ignored.
+	 */
+	CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_TPL_AS_MSG, (uchar*) "RSYSLOG_FileFormat"));
+
+	/* finally build the instance */
+	if((pData->ctxee = ee_initCtx()) == NULL) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize libee ctx, cannot "
+				"activate action");
+		ABORT_FINALIZE(RS_RET_ERR_LIBEE_INIT);
+	}
+
+	if((pData->ctxln = ln_initCtx()) == NULL) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize liblognorm ctx, cannot "
+				"activate action");
+		ee_exitCtx(pData->ctxee);
+		ABORT_FINALIZE(RS_RET_ERR_LIBLOGNORM_INIT);
+	}
+	ln_setEECtx(pData->ctxln, pData->ctxee);
+	if(ln_loadSamples(pData->ctxln, (char*) cs.rulebase) != 0) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: sample db '%s' could not be loaded "
+				"cannot activate action", cs.rulebase);
+		ee_exitCtx(pData->ctxee);
+		ln_exitCtx(pData->ctxln);
+		ABORT_FINALIZE(RS_RET_ERR_LIBLOGNORM_SAMPDB_LOAD);
+	}
+	pData->bUseRawMsg = cs.bUseRawMsg;
+
+	/* all config vars auto-reset! */
+	cs.bUseRawMsg = 0;
+	free(cs.rulebase);
+	cs.rulebase = NULL;
+CODE_STD_FINALIZERparseSelectorAct
+ENDparseSelectorAct
+
+
+BEGINmodExit
+CODESTARTmodExit
+	objRelease(errmsg, CORE_COMPONENT);
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_OMOD_QUERIES
+ENDqueryEtryPt
+
+
+
+/* Reset config variables for this module to default values.
+ */
+static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal)
+{
+	DEFiRet;
+	cs.rulebase = NULL;
+	cs.bUseRawMsg = 0;
+	RETiRet;
+}
+
+/* set the rulebase name */
+static rsRetVal
+setRuleBase(void __attribute__((unused)) *pVal, uchar *pszName)
+{
+	DEFiRet;
+	cs.rulebase = pszName;
+	pszName = NULL;
+	RETiRet;
+}
+
+BEGINmodInit()
+	rsRetVal localRet;
+	rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts);
+	unsigned long opts;
+	int bMsgPassingSupported;
+CODESTARTmodInit
+SCOPINGmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION;
+		/* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	/* check if the rsyslog core supports parameter passing code */
+	bMsgPassingSupported = 0;
+	localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts",
+			&pomsrGetSupportedTplOpts);
+	if(localRet == RS_RET_OK) {
+		/* found entry point, so let's see if core supports msg passing */
+		CHKiRet((*pomsrGetSupportedTplOpts)(&opts));
+		if(opts & OMSR_TPL_AS_MSG)
+			bMsgPassingSupported = 1;
+	} else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) {
+		ABORT_FINALIZE(localRet); /* Something else went wrong, not acceptable */
+	}
+	
+	if(!bMsgPassingSupported) {
+		DBGPRINTF("mmnormalize: msg-passing is not supported by rsyslog core, "
+			  "can not continue.\n");
+		ABORT_FINALIZE(RS_RET_NO_MSG_PASSING);
+	}
+
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmnormalizerulebase", 0, eCmdHdlrGetWord,
+				    setRuleBase, NULL, STD_LOADABLE_MODULE_ID, eConfObjAction));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmnormalizeuserawmsg", 0, eCmdHdlrInt,
+				    NULL, &cs.bUseRawMsg, STD_LOADABLE_MODULE_ID, eConfObjAction));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler,
+				    resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID, eConfObjAction));
+ENDmodInit
+
+/* vi:set ai:
+ */
diff --git a/runtime/Makefile.am b/runtime/Makefile.am
index 09cb6b41..c8e8ce2a 100644
--- a/runtime/Makefile.am
+++ b/runtime/Makefile.am
@@ -110,12 +110,12 @@ librsyslog_la_SOURCES = \
 # runtime or will no longer be needed. -- rgerhards, 2008-06-13
 
 if WITH_MODDIRS
-librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS)
+librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/:$(moddirs)\" $(PTHREADS_CFLAGS) $(LIBEE_CFLAGS)
 else
-librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS)
+librsyslog_la_CPPFLAGS = -DSD_EXPORT_SYMBOLS -D_PATH_MODDIR=\"$(pkglibdir)/\" -I$(top_srcdir) $(PTHREADS_CFLAGS) $(LIBEE_CFLAGS)
 endif
 #librsyslog_la_LDFLAGS = -module -avoid-version
-librsyslog_la_LIBADD =  $(DL_LIBS) $(RT_LIBS)
+librsyslog_la_LIBADD =  $(DL_LIBS) $(RT_LIBS) $(LIBEE_LIBS)
 
 #
 # regular expression support
diff --git a/runtime/conf.c b/runtime/conf.c
index 1f0badcb..e12cf82b 100644
--- a/runtime/conf.c
+++ b/runtime/conf.c
@@ -874,6 +874,14 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f)
 		rsParsDestruct(pPars);
 		return(iRet);
 	}
+	if(f->f_filterData.prop.propID == PROP_CEE) {
+		/* in CEE case, we need to preserve the actual property name */
+		if((f->f_filterData.prop.propName =
+		     es_newStrFromBuf((char*)cstrGetSzStrNoNULL(pCSPropName)+2, cstrLen(pCSPropName)-2)) == NULL) {
+			cstrDestruct(&pCSPropName);
+			return(RS_RET_ERR);
+		}
+	}
 	cstrDestruct(&pCSPropName);
 
 	/* read operation */
@@ -902,10 +910,13 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f)
 		iOffset = 0;
 	}
 
+dbgprintf("XXX: offset is %d, string '%s'\n", iOffset, rsCStrGetSzStrNoNULL(pCSCompOp));
 	if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "contains", 8)) {
 		f->f_filterData.prop.operation = FIOP_CONTAINS;
 	} else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "isequal", 7)) {
 		f->f_filterData.prop.operation = FIOP_ISEQUAL;
+	} else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "isempty", 7)) {
+		f->f_filterData.prop.operation = FIOP_ISEMPTY;
 	} else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "startswith", 10)) {
 		f->f_filterData.prop.operation = FIOP_STARTSWITH;
 	} else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "regex", 5)) {
@@ -918,12 +929,15 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f)
 	}
 	rsCStrDestruct(&pCSCompOp); /* no longer needed */
 
-	/* read compare value */
-	iRet = parsQuotedCStr(pPars, &f->f_filterData.prop.pCSCompValue);
-	if(iRet != RS_RET_OK) {
-		errmsg.LogError(0, iRet, "error %d compare value property - ignoring selector", iRet);
-		rsParsDestruct(pPars);
-		return(iRet);
+dbgprintf("XXX: fiop is %u\n", (unsigned) f->f_filterData.prop.operation);
+	if(f->f_filterData.prop.operation != FIOP_ISEMPTY) {
+		/* read compare value */
+		iRet = parsQuotedCStr(pPars, &f->f_filterData.prop.pCSCompValue);
+		if(iRet != RS_RET_OK) {
+			errmsg.LogError(0, iRet, "error %d compare value property - ignoring selector", iRet);
+			rsParsDestruct(pPars);
+			return(iRet);
+		}
 	}
 
 	/* skip to action part */
@@ -1065,7 +1079,6 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f)
 	 * and, if so, we copy them over. rgerhards, 2005-10-18
 	 */
 	if(pDfltProgNameCmp != NULL) {
-RUNLOG_STR("dflt ProgNameCmp != NULL, setting opCSProgNameComp");
 		CHKiRet(rsCStrConstructFromCStr(&(f->pCSProgNameComp), pDfltProgNameCmp));
 	}
 
diff --git a/runtime/ctok.c b/runtime/ctok.c
index 18ddaed2..6d97568e 100644
--- a/runtime/ctok.c
+++ b/runtime/ctok.c
@@ -277,6 +277,9 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken)
 	if(c == '$') { /* second dollar, we have a system variable */
 		pToken->tok = ctok_SYSVAR;
 		CHKiRet(ctokGetCharFromStream(pThis, &c)); /* "eat" it... */
+	} else if(c == '!') { /* cee variable indicator */
+		pToken->tok = ctok_CEEVAR;
+		CHKiRet(ctokGetCharFromStream(pThis, &c)); /* "eat" it... */
 	} else {
 		pToken->tok = ctok_MSGVAR;
 	}
diff --git a/runtime/ctok_token.h b/runtime/ctok_token.h
index d36689fa..1413c699 100644
--- a/runtime/ctok_token.h
+++ b/runtime/ctok_token.h
@@ -54,6 +54,7 @@ typedef struct {
 		ctok_FUNCTION = 17,
 		ctok_THEN = 18,
 		ctok_STRADD = 19,
+		ctok_CEEVAR = 20,
 		ctok_CMP_EQ = 100, /* all compare operations must be in a row */
 		ctok_CMP_NEQ = 101,
 		ctok_CMP_LT = 102,
diff --git a/runtime/expr.c b/runtime/expr.c
index e449d1c7..01431474 100644
--- a/runtime/expr.c
+++ b/runtime/expr.c
@@ -151,6 +151,11 @@ terminal(expr_t *pThis, ctok_t *tok)
 			CHKiRet(ctok_token.UnlinkVar(pToken, &pVar));
 			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHMSGVAR, pVar)); /* add to program */
 			break;
+		case ctok_CEEVAR:
+			dbgoprint((obj_t*) pThis, "SYSVAR\n");
+			CHKiRet(ctok_token.UnlinkVar(pToken, &pVar));
+			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHCEEVAR, pVar)); /* add to program */
+			break;
 		case ctok_SYSVAR:
 			dbgoprint((obj_t*) pThis, "SYSVAR\n");
 			CHKiRet(ctok_token.UnlinkVar(pToken, &pVar));
diff --git a/runtime/msg.c b/runtime/msg.c
index 82565f18..65ea101f 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -37,6 +37,7 @@
 #include <ctype.h>
 #include <sys/socket.h>
 #include <netdb.h>
+#include <libee/libee.h>
 #if HAVE_MALLOC_H
 #  include <malloc.h>
 #endif
@@ -444,6 +445,10 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID)
 		*pPropID = PROP_SYS_MINUTE;
 	} else if(!strcmp((char*) pName, "$myhostname")) {
 		*pPropID = PROP_SYS_MYHOSTNAME;
+	} else if(!strcmp((char*) pName, "$!all-json")) {
+		*pPropID = PROP_CEE_ALL_JSON;
+	} else if(!strncmp((char*) pName, "$!", 2)) {
+		*pPropID = PROP_CEE;
 	} else {
 		*pPropID = PROP_INVALID;
 		iRet = RS_RET_VAR_NOT_FOUND;
@@ -525,6 +530,10 @@ uchar *propIDToName(propid_t propID)
 			return UCHAR_CONSTANT("$MINUTE");
 		case PROP_SYS_MYHOSTNAME:
 			return UCHAR_CONSTANT("$MYHOSTNAME");
+		case PROP_CEE:
+			return UCHAR_CONSTANT("*CEE-based property*");
+		case PROP_CEE_ALL_JSON:
+			return UCHAR_CONSTANT("$!all-json");
 		default:
 			return UCHAR_CONSTANT("*invalid property id*");
 	}
@@ -707,6 +716,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 	pM->pRcvFromIP = NULL;
 	pM->rcvFrom.pRcvFrom = NULL;
 	pM->pRuleset = NULL;
+	pM->event = NULL;
 	memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt));
 	memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP));
 	pM->TAG.pszTAG = NULL;
@@ -833,6 +843,8 @@ CODESTARTobjDestruct(msg)
 			rsCStrDestruct(&pThis->pCSPROCID);
 		if(pThis->pCSMSGID != NULL)
 			rsCStrDestruct(&pThis->pCSMSGID);
+		if(pThis->event != NULL)
+			ee_deleteEvent(pThis->event);
 #	ifndef HAVE_ATOMIC_BUILTINS
 		MsgUnlock(pThis);
 # 	endif
@@ -1198,7 +1210,7 @@ char *getProtocolVersionString(msg_t *pM)
 }
 
 
-static inline void
+void
 getRawMsg(msg_t *pM, uchar **pBuf, int *piLen)
 {
 	if(pM == NULL) {
@@ -2219,6 +2231,41 @@ static uchar *getNOW(eNOWType eNow)
 #undef tmpBUFSIZE /* clean up */
 
 
+/* Get a CEE-Property from libee. This function probably should be
+ * placed somewhere else, but this smells like a big restructuring
+ * useful in any case. So for the time being, I'll simply leave the
+ * function here, as the context seems good enough. -- rgerhards, 2010-12-01
+ */
+static inline void
+getCEEPropVal(msg_t *pMsg, es_str_t *propName, uchar **pRes, int *buflen, unsigned short *pbMustBeFreed)
+{
+	struct ee_field *field;
+	es_str_t *str;
+
+	if(*pbMustBeFreed)
+		free(*pRes);
+	*pRes = NULL;
+
+	if(pMsg->event == NULL) goto finalize_it;
+	if((field = ee_getEventField(pMsg->event, propName)) == NULL)
+		goto finalize_it;
+	/* right now, we always extract data from the first field value. A reason for this
+	 * is that as of now (2010-12-01) liblognorm never populates more than one ;)
+	 */
+	if((str = ee_getFieldValueAsStr(field, 0)) == NULL) goto finalize_it;
+	*pRes = (unsigned char*) es_str2cstr(str, "#000");
+	es_deleteStr(str);
+	*buflen = (int) ustrlen(*pRes);
+	*pbMustBeFreed = 1;
+
+finalize_it:
+	if(*pRes == NULL) {
+		/* could not find any value, so set it to empty */
+		*pRes = (unsigned char*)"";
+		*pbMustBeFreed = 0;
+	}
+}
+
 /* This function returns a string-representation of the 
  * requested message property. This is a generic function used
  * to abstract properties so that these can be easier
@@ -2261,7 +2308,7 @@ static uchar *getNOW(eNOWType eNow)
 	*pPropLen = sizeof("**OUT OF MEMORY**") - 1; \
 	return(UCHAR_CONSTANT("**OUT OF MEMORY**"));}
 uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
-                 propid_t propID, size_t *pPropLen,
+                 propid_t propid, es_str_t *propName, size_t *pPropLen,
 		 unsigned short *pbMustBeFreed)
 {
 	uchar *pRes; /* result pointer */
@@ -2270,6 +2317,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 	uchar *pBuf;
 	int iLen;
 	short iOffs;
+	es_str_t *str; /* for CEE handling, temp. string */
 
 	BEGINfunc
 	assert(pMsg != NULL);
@@ -2283,7 +2331,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 
 	*pbMustBeFreed = 0;
 
-	switch(propID) {
+	switch(propid) {
 		case PROP_MSG:
 			pRes = getMSG(pMsg);
 			bufLen = getMSGLen(pMsg);
@@ -2416,11 +2464,20 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 		case PROP_SYS_MYHOSTNAME:
 			pRes = glbl.GetLocalHostName();
 			break;
+		case PROP_CEE_ALL_JSON:
+			ee_fmtEventToJSON(pMsg->event, &str);
+			pRes = (uchar*) es_str2cstr(str, "#000");
+			es_deleteStr(str);
+			*pbMustBeFreed = 1;	/* all of these functions allocate dyn. memory */
+			break;
+		case PROP_CEE:
+			getCEEPropVal(pMsg, propName, &pRes, &bufLen, pbMustBeFreed);
+			break;
 		default:
 			/* there is no point in continuing, we may even otherwise render the
 			 * error message unreadable. rgerhards, 2007-07-10
 			 */
-			dbgprintf("invalid property id: '%d'\n", propID);
+			dbgprintf("invalid property id: '%d'\n", propid);
 			*pbMustBeFreed = 0;
 			*pPropLen = sizeof("**INVALID PROPERTY NAME**") - 1;
 			return UCHAR_CONSTANT("**INVALID PROPERTY NAME**");
@@ -2428,6 +2485,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 
 	/* If we did not receive a template pointer, we are already done... */
 	if(pTpe == NULL) {
+		*pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen;
 		return pRes;
 	}
 	
@@ -3016,6 +3074,61 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 }
 
 
+/* The function returns a cee variable suitable for use with RainerScript. Most importantly, this means
+ * that the value is returned in a var_t object. The var_t is constructed inside this function and
+ * MUST be freed by the caller.
+ * Note that we need to do a lot of conversions between es_str_t and cstr -- this will go away once
+ * we have moved larger parts of rsyslog to es_str_t. Acceptable for the moment, especially as we intend
+ * to rewrite the script engine as well!
+ * rgerhards, 2010-12-03
+ */
+rsRetVal
+msgGetCEEVar(msg_t *pMsg, cstr_t *propName, var_t **ppVar)
+{
+	DEFiRet;
+	var_t *pVar;
+	cstr_t *pstrProp;
+	es_str_t *str = NULL;
+	es_str_t *epropName = NULL;
+	struct ee_field *field;
+
+	ISOBJ_TYPE_assert(pMsg, msg);
+	ASSERT(propName != NULL);
+	ASSERT(ppVar != NULL);
+
+	/* make sure we have a var_t instance */
+	CHKiRet(var.Construct(&pVar));
+	CHKiRet(var.ConstructFinalize(pVar));
+
+	epropName = es_newStrFromBuf((char*)propName->pBuf, propName->iStrLen);
+	if((field = ee_getEventField(pMsg->event, epropName)) != NULL) {
+		/* right now, we always extract data from the first field value. A reason for this
+		 * is that as of now (2010-12-01) liblognorm never populates more than one ;)
+		 */
+		str = ee_getFieldValueAsStr(field, 0);
+	}
+
+	if(str == NULL) {
+		CHKiRet(cstrConstruct(&pstrProp));
+		CHKiRet(cstrFinalize(pstrProp));
+	} else {
+		CHKiRet(cstrConstructFromESStr(&pstrProp, str));
+	}
+
+	/* now create a string object out of it and hand that over to the var */
+	CHKiRet(var.SetString(pVar, pstrProp));
+	es_deleteStr(str);
+
+	/* finally store var */
+	*ppVar = pVar;
+
+finalize_it:
+	if(epropName != NULL)
+		es_deleteStr(epropName);
+	RETiRet;
+}
+
+
 /* The returns a message variable suitable for use with RainerScript. Most importantly, this means
  * that the value is returned in a var_t object. The var_t is constructed inside this function and
  * MUST be freed by the caller.
@@ -3043,7 +3156,7 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar)
 	/* always call MsgGetProp() without a template specifier */
 	/* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */
 	propNameToID(pstrPropName, &propid);
-	pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, &propLen, &bMustBeFreed);
+	pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed);
 
 	/* now create a string object out of it and hand that over to the var */
 	CHKiRet(rsCStrConstructFromszStr(&pstrProp, pszProp));
@@ -3058,6 +3171,8 @@ finalize_it:
 
 	RETiRet;
 }
+
+
 /* This function can be used as a generic way to set properties.
  * We have to handle a lot of legacy, so our return value is not always
  * 100% correct (called functions do not always provide one, should
diff --git a/runtime/msg.h b/runtime/msg.h
index 4897959c..1fd95994 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -29,10 +29,12 @@
 #define	MSG_H_INCLUDED 1
 
 #include <pthread.h>
+#include <libestr.h>
 #include "obj.h"
 #include "syslogd-types.h"
 #include "template.h"
 #include "atomic.h"
+#include "libee/libee.h"
 
 
 /* rgerhards 2004-11-08: The following structure represents a
@@ -105,6 +107,7 @@ struct msg {
 				   it obviously is solved in way or another...). */
 	struct syslogTime tRcvdAt;/* time the message entered this program */
 	struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */
+	struct ee_event	*event;	/**< libee event */
 	/* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */
 	uchar szRawMsg[CONF_RAWMSG_BUFSIZE];	/* most messages are small, and these are stored here (without malloc/free!) */
 	uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE];
@@ -162,7 +165,8 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg);
 void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg);
 rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG);
 uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
-                 propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed);
+                  propid_t propid, es_str_t *propName,
+		  size_t *pPropLen, unsigned short *pbMustBeFreed);
 char *textpri(char *pRes, size_t pResLen, int pri);
 rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar);
 rsRetVal MsgEnableThreadSafety(void);
@@ -170,6 +174,8 @@ uchar *getRcvFrom(msg_t *pM);
 void getTAG(msg_t *pM, uchar **ppBuf, int *piLen);
 char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt);
 char *getPRI(msg_t *pMsg);
+void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen);
+rsRetVal msgGetCEEVar(msg_t *pThis, cstr_t *propName, var_t **ppVar);
 
 
 /* TODO: remove these five (so far used in action.c) */
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index c2f6463c..17b20de2 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -136,6 +136,8 @@ typedef uintTiny	propid_t;
 #define PROP_SYS_QHOUR			156
 #define PROP_SYS_MINUTE			157
 #define PROP_SYS_MYHOSTNAME		158
+#define PROP_CEE			200
+#define PROP_CEE_ALL_JSON		201
 
 
 /* The error codes below are orginally "borrowed" from
@@ -346,6 +348,9 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */
 
 	RS_RET_INVLD_CONF_OBJ= -2200,	/**< invalid config object (e.g. $Begin conf statement) */
+	RS_RET_ERR_LIBEE_INIT = -2201,	/**< cannot obtain libee ctx */
+	RS_RET_ERR_LIBLOGNORM_INIT = -2202,/**< cannot obtain liblognorm ctx */
+	RS_RET_ERR_LIBLOGNORM_SAMPDB_LOAD = -2203,/**< liblognorm sampledb load failed */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
diff --git a/runtime/rule.c b/runtime/rule.c
index 42773768..fc1ee17c 100644
--- a/runtime/rule.c
+++ b/runtime/rule.c
@@ -70,6 +70,12 @@ getFIOPName(unsigned iFIOP)
 		case FIOP_REGEX:
 			pRet = "regex";
 			break;
+		case FIOP_EREREGEX:
+			pRet = "ereregex";
+			break;
+		case FIOP_ISEMPTY:
+			pRet = "isempty";
+			break;
 		default:
 			pRet = "NOP";
 			break;
@@ -190,7 +196,8 @@ dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFac
 		bRet = (pResult->val.num) ? 1 : 0;
 	} else {
 		assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */
-		pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID, &propLen, &pbMustBeFreed);
+		pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID,
+					pRule->f_filterData.prop.propName, &propLen, &pbMustBeFreed);
 
 		/* Now do the compares (short list currently ;)) */
 		switch(pRule->f_filterData.prop.operation ) {
@@ -198,6 +205,10 @@ dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFac
 			if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1)
 				bRet = 1;
 			break;
+		case FIOP_ISEMPTY:
+			if(propLen == 0)
+				bRet = 1; /* process message! */
+			break;
 		case FIOP_ISEQUAL:
 			if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue,
 					  pszPropVal, ustrlen(pszPropVal)) == 0)
@@ -230,14 +241,28 @@ dbgprintf("testing filter, f_pmask %d\n", pRule->f_filterData.f_pmask[pMsg->iFac
 			bRet = (bRet == 1) ?  0 : 1;
 
 		if(Debug) {
-			dbgprintf("Filter: check for property '%s' (value '%s') ",
-			        propIDToName(pRule->f_filterData.prop.propID), pszPropVal);
+			char *cstr;
+			if(pRule->f_filterData.prop.propID == PROP_CEE) {
+				cstr = es_str2cstr(pRule->f_filterData.prop.propName, NULL);
+				dbgprintf("Filter: check for CEE property '%s' (value '%s') ",
+					cstr, pszPropVal);
+				free(cstr);
+			} else {
+				dbgprintf("Filter: check for property '%s' (value '%s') ",
+					propIDToName(pRule->f_filterData.prop.propID), pszPropVal);
+			}
 			if(pRule->f_filterData.prop.isNegated)
 				dbgprintf("NOT ");
-			dbgprintf("%s '%s': %s\n",
-			       getFIOPName(pRule->f_filterData.prop.operation),
-			       rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue),
-			       bRet ? "TRUE" : "FALSE");
+			if(pRule->f_filterData.prop.operation == FIOP_ISEMPTY) {
+				dbgprintf("%s : %s\n",
+				       getFIOPName(pRule->f_filterData.prop.operation),
+				       bRet ? "TRUE" : "FALSE");
+			} else {
+				dbgprintf("%s '%s': %s\n",
+				       getFIOPName(pRule->f_filterData.prop.operation),
+				       rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue),
+				       bRet ? "TRUE" : "FALSE");
+			}
 		}
 
 		/* cleanup */
@@ -324,6 +349,8 @@ CODESTARTobjDestruct(rule)
 			rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue);
 		if(pThis->f_filterData.prop.regex_cache != NULL)
 			rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache);
+		if(pThis->f_filterData.prop.propName != NULL)
+			es_deleteStr(pThis->f_filterData.prop.propName);
 	} else if(pThis->f_filter_type == FILTER_EXPR) {
 		if(pThis->f_filterData.f_expr != NULL)
 			expr.Destruct(&pThis->f_filterData.f_expr);
@@ -368,6 +395,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction)
 /* debugprint for the rule object */
 BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */
 	int i;
+	char *cstr;
 CODESTARTobjDebugPrint(rule)
 	dbgoprint((obj_t*) pThis, "rsyslog rule:\n");
 	if(pThis->pCSProgNameComp != NULL)
@@ -388,12 +416,19 @@ CODESTARTobjDebugPrint(rule)
 	} else {
 		dbgprintf("PROPERTY-BASED Filter:\n");
 		dbgprintf("\tProperty.: '%s'\n", propIDToName(pThis->f_filterData.prop.propID));
+		if(pThis->f_filterData.prop.propName != NULL) {
+			cstr = es_str2cstr(pThis->f_filterData.prop.propName, NULL);
+			dbgprintf("\tCEE-Prop.: '%s'\n", cstr);
+			free(cstr);
+		}
 		dbgprintf("\tOperation: ");
 		if(pThis->f_filterData.prop.isNegated)
 			dbgprintf("NOT ");
 		dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation));
-		dbgprintf("\tValue....: '%s'\n",
-		       rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue));
+		if(pThis->f_filterData.prop.pCSCompValue != NULL) {
+			dbgprintf("\tValue....: '%s'\n",
+			       rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue));
+		}
 		dbgprintf("\tAction...: ");
 	}
 
diff --git a/runtime/rule.h b/runtime/rule.h
index 309a2ed8..3b34e11a 100644
--- a/runtime/rule.h
+++ b/runtime/rule.h
@@ -25,6 +25,7 @@
 #ifndef INCLUDED_RULE_H
 #define INCLUDED_RULE_H
 
+#include "libestr.h"
 #include "linkedlist.h"
 #include "regexp.h"
 #include "expr.h"
@@ -49,6 +50,7 @@ struct rule_s {
 			cstr_t *pCSCompValue;		/* value to "compare" against */
 			sbool isNegated;	
 			propid_t propID;		/* ID of the requested property */
+			es_str_t *propName;		/* name of property for CEE-based filters */
 		} prop;
 		expr_t *f_expr;				/* expression object */
 	} f_filterData;
diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c
index ccf115c1..f4a9caae 100644
--- a/runtime/stringbuf.c
+++ b/runtime/stringbuf.c
@@ -35,6 +35,7 @@
 #include <string.h>
 #include <ctype.h>
 #include <sys/types.h>
+#include <libestr.h>
 #include "rsyslog.h"
 #include "stringbuf.h"
 #include "srUtils.h"
@@ -104,6 +105,34 @@ finalize_it:
 	RETiRet;
 }
 
+
+/* construct from es_str_t string
+ * rgerhards 2010-12-03
+ */
+rsRetVal cstrConstructFromESStr(cstr_t **ppThis, es_str_t *str)
+{
+	DEFiRet;
+	cstr_t *pThis;
+
+	assert(ppThis != NULL);
+
+	CHKiRet(rsCStrConstruct(&pThis));
+
+	pThis->iBufSize = pThis->iStrLen = es_strlen(str);
+	if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) {
+		RSFREEOBJ(pThis);
+		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+	}
+
+	/* we do NOT need to copy the \0! */
+	memcpy(pThis->pBuf, es_getBufAddr(str), pThis->iStrLen);
+
+	*ppThis = pThis;
+
+finalize_it:
+	RETiRet;
+}
+
 /* construct from CStr object. only the counted string is
  * copied, not the szString.
  * rgerhards 2005-10-18
diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h
index c5130238..df234012 100644
--- a/runtime/stringbuf.h
+++ b/runtime/stringbuf.h
@@ -36,6 +36,7 @@
 #define _STRINGBUF_H_INCLUDED__ 1
 
 #include <assert.h>
+#include <libestr.h>
 
 /** 
  * The dynamic string buffer object.
@@ -57,6 +58,7 @@ typedef struct cstr_s
  */
 rsRetVal cstrConstruct(cstr_t **ppThis);
 #define rsCStrConstruct(x) cstrConstruct((x))
+rsRetVal cstrConstructFromESStr(cstr_t **ppThis, es_str_t *str);
 rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz);
 rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom);
 
diff --git a/runtime/typedefs.h b/runtime/typedefs.h
index 38bc02d5..1f624f7a 100644
--- a/runtime/typedefs.h
+++ b/runtime/typedefs.h
@@ -127,7 +127,8 @@ typedef enum {
 	FIOP_ISEQUAL  = 2,	/* is (exactly) equal? */
 	FIOP_STARTSWITH = 3,	/* starts with a string? */
 	FIOP_REGEX = 4,		/* matches a (BRE) regular expression? */
-	FIOP_EREREGEX = 5	/* matches a ERE regular expression? */
+	FIOP_EREREGEX = 5,	/* matches a ERE regular expression? */
+	FIOP_ISEMPTY = 6	/* string empty <=> strlen(s) == 0 ?*/
 } fiop_t;
 
 /* types of configuration handlers
diff --git a/runtime/var.h b/runtime/var.h
index 6d890ec9..ae971bb5 100644
--- a/runtime/var.h
+++ b/runtime/var.h
@@ -40,6 +40,7 @@ typedef struct var_s {
 	varType_t varType;
 	union {
 		number_t num;
+		es_str_t *str;
 		cstr_t *pStr;
 		syslogTime_t vSyslogTime;
 
diff --git a/runtime/vm.c b/runtime/vm.c
index 0ed174d1..c5521c31 100644
--- a/runtime/vm.c
+++ b/runtime/vm.c
@@ -448,6 +448,7 @@ BEGINop(PUSHMSGVAR) /* remember to set the instruction also in the ENDop macro!
 	var_t *pVal; /* the value to push */
 	cstr_t *pstrVal;
 CODESTARTop(PUSHMSGVAR)
+dbgprintf("XXX: pushMSGVAR, var '%s'\n", rsCStrGetSzStr(pOp->operand.pVar->val.pStr));
 	if(pThis->pMsg == NULL) {
 		/* TODO: flag an error message! As a work-around, we permit
 		 * execution to continue here with an empty string
@@ -468,6 +469,31 @@ finalize_it:
 ENDop(PUSHMSGVAR)
 
 
+BEGINop(PUSHCEEVAR) /* remember to set the instruction also in the ENDop macro! */
+	var_t *pVal; /* the value to push */
+	cstr_t *pstrVal;
+CODESTARTop(PUSHCEEVAR)
+dbgprintf("XXX: pushCEEVAR, var '%s'\n", rsCStrGetSzStr(pOp->operand.pVar->val.pStr));
+	if(pThis->pMsg == NULL) {
+		/* TODO: flag an error message! As a work-around, we permit
+		 * execution to continue here with an empty string
+		 */
+		CHKiRet(var.Construct(&pVal));
+		CHKiRet(var.ConstructFinalize(pVal));
+		CHKiRet(rsCStrConstructFromszStr(&pstrVal, (uchar*)""));
+		CHKiRet(var.SetString(pVal, pstrVal));
+	} else {
+		/* we have a message, so pull value from there */
+		CHKiRet(msgGetCEEVar(pThis->pMsg, pOp->operand.pVar->val.pStr, &pVal));
+	}
+
+	/* if we reach this point, we have a valid pVal and can push it */
+	vmstk.Push(pThis->pStk, pVal);
+dbgprintf("XXX: pushCEEVAR, result '%s'\n", rsCStrGetSzStr(pVal->val.pStr));
+finalize_it:
+ENDop(PUSHCEEVAR)
+
+
 BEGINop(PUSHSYSVAR) /* remember to set the instruction also in the ENDop macro! */
 	var_t *pVal; /* the value to push */
 CODESTARTop(PUSHSYSVAR)
@@ -685,6 +711,7 @@ execProg(vm_t *pThis, vmprg_t *pProg)
 			doOP(NOT);
 			doOP(PUSHCONSTANT);
 			doOP(PUSHMSGVAR);
+			doOP(PUSHCEEVAR);
 			doOP(PUSHSYSVAR);
 			doOP(STRADD);
 			doOP(PLUS);
diff --git a/runtime/vmop.h b/runtime/vmop.h
index 67048c26..c085a940 100644
--- a/runtime/vmop.h
+++ b/runtime/vmop.h
@@ -59,6 +59,7 @@ typedef enum {	 /* do NOT start at 0 to detect uninitialized types after calloc(
 	opcode_PUSHSYSVAR      = 1001,	 /* requires var operand */
 	opcode_PUSHMSGVAR      = 1002,	 /* requires var operand */
 	opcode_PUSHCONSTANT    = 1003,	 /* requires var operand */
+	opcode_PUSHCEEVAR      = 1004,	 /* requires var operand */
 	opcode_UNARY_MINUS     = 1010,
 	opcode_FUNC_CALL       = 1012,
 	opcode_END_PROG        = 2000
diff --git a/template.c b/template.c
index 06949e45..1ea2575a 100644
--- a/template.c
+++ b/template.c
@@ -113,7 +113,8 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *
 			iLenVal = pTpe->data.constant.iLenConstant;
 			bMustBeFreed = 0;
 		} else 	if(pTpe->eEntryType == FIELD) {
-			pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, &iLenVal, &bMustBeFreed);
+			pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid,
+						   pTpe->data.field.propName,  &iLenVal, &bMustBeFreed);
 			/* we now need to check if we should use SQL option. In this case,
 			 * we must go over the generated string and escape '\'' characters.
 			 * rgerhards, 2005-09-22: the option values below look somewhat misplaced,
@@ -192,7 +193,8 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr)
 		if(pTpe->eEntryType == CONSTANT) {
 			CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant));
 		} else 	if(pTpe->eEntryType == FIELD) {
-			pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, &propLen, &bMustBeFreed);
+			pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid,
+						   pTpe->data.field.propName,  &propLen, &bMustBeFreed);
 			if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */
 				pArr[iArr] = pVal; /* ... so we can use it! */
 			} else {
@@ -586,7 +588,14 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 		cstrDestruct(&pStrB);
 		return 1;
 	}
-	cstrDestruct(&pStrB); /* no longer needed, now use ID */
+	if(pTpe->data.field.propid == PROP_CEE) {
+		/* in CEE case, we need to preserve the actual property name */
+		if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrB)+2, cstrLen(pStrB)-2)) == NULL) {
+			cstrDestruct(&pStrB);
+			return 1;
+		}
+	}
+	cstrDestruct(&pStrB);
 
 	/* Check frompos, if it has an R, then topos should be a regex */
 	if(*p == ':') {
@@ -1073,6 +1082,8 @@ void tplDeleteAll(void)
 						regexp.regfree(&(pTpeDel->data.field.re));
 					}
 				}
+				if(pTpeDel->data.field.propName != NULL)
+					es_deleteStr(pTpeDel->data.field.propName);
 				break;
 			}
 			/*dbgprintf("\n");*/
@@ -1128,6 +1139,8 @@ void tplDeleteNew(void)
 						regexp.regfree(&(pTpeDel->data.field.re));
 					}
 				}
+				if(pTpeDel->data.field.propName != NULL)
+					es_deleteStr(pTpeDel->data.field.propName);
 				break;
 			}
 			/*dbgprintf("\n");*/
@@ -1177,6 +1190,11 @@ void tplPrintList(void)
 				break;
 			case FIELD:
 				dbgprintf("(FIELD), value: '%d' ", pTpe->data.field.propid);
+				if(pTpe->data.field.propid == PROP_CEE) {
+					char *cstr = es_str2cstr(pTpe->data.field.propName, NULL);
+					dbgprintf("[EE-Property: '%s'] ", cstr);
+					free(cstr);
+				}
 				switch(pTpe->data.field.eDateFormat) {
 				case tplFmtDefault:
 					break;
diff --git a/template.h b/template.h
index f7ac2e08..2fb8e1ae 100644
--- a/template.h
+++ b/template.h
@@ -25,6 +25,7 @@
 #ifndef	TEMPLATE_H_INCLUDED
 #define	TEMPLATE_H_INCLUDED 1
 
+#include <libestr.h>
 #include "regexp.h"
 #include "stringbuf.h"
 
@@ -88,6 +89,8 @@ struct templateEntry {
 			unsigned char field_delim; /* support for field-counting: field delemiter char */
 			int field_expand;	/* use multiple instances of the field delimiter as a single one? */
 
+			es_str_t *propName;	/**< property name (currently being used for CEE only) */
+
 			enum tplFormatTypes eDateFormat;
 			enum tplFormatCaseConvTypes eCaseConv;
 			struct { 		/* bit fields! */
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 96657ad4..5c3f7a40 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -36,7 +36,7 @@ rsyslogd_SOURCES = \
 	\
 	../dirty.h
 rsyslogd_CPPFLAGS =  $(PTHREADS_CFLAGS) $(RSRT_CFLAGS)
-rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS)
+rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBEE_LIBS) $(LIBLOGNORM_LIBS)
 rsyslogd_LDFLAGS = -export-dynamic
 
 if ENABLE_DIAGTOOLS