milestone: added mmnormalize (compiles, but untested)

7 files changed, 306 insertions, 1 deletions

diff --git a/Makefile.am b/Makefile.am
index 680a819e..bdbc2794 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -179,6 +179,10 @@ if ENABLE_RFC3195
 SUBDIRS += plugins/im3195
 endif
 
+if ENABLE_MMNORMALIZE
+SUBDIRS += plugins/mmnormalize
+endif
+
 if ENABLE_ORACLE
 SUBDIRS += plugins/omoracle
 endif
diff --git a/configure.ac b/configure.ac
index 10a98b0d..d5bb1d4b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -31,6 +31,10 @@ AC_CANONICAL_HOST
 
 PKG_PROG_PKG_CONFIG
 
+# modules we require
+PKG_CHECK_MODULES(LIBESTR, libestr >= 0.1.0)
+PKG_CHECK_MODULES(LIBEE, libee >= 0.1.0)
+
 case "${host}" in
   *-*-linux*)
     os_type="linux"
@@ -714,6 +718,24 @@ AC_ARG_ENABLE(imdiag,
 AM_CONDITIONAL(ENABLE_IMDIAG, test x$enable_imdiag = xyes)
 
 
+# mmnormalize
+AC_ARG_ENABLE(mmnormalize,
+        [AS_HELP_STRING([--enable-mmnormalize],[Enable building mmnormalize support @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_mmnormalize="yes" ;;
+          no) enable_mmnormalize="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-mmnormalize) ;;
+         esac],
+        [enable_mmnormalize=no]
+)
+if test "x$enable_mmnormalize" = "xyes"; then
+	PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 0.1.0)
+fi
+AM_CONDITIONAL(ENABLE_MMNORMALIZE, test x$enable_mmnormalize = xyes)
+AC_SUBST(LOGNORM_CFLAGS)
+AC_SUBST(LOGNORM_LIBS)
+
+
 # RELP support
 AC_ARG_ENABLE(relp,
         [AS_HELP_STRING([--enable-relp],[Enable RELP support @<:@default=no@:>@])],
@@ -1072,6 +1094,7 @@ AC_CONFIG_FILES([Makefile \
 		plugins/omsnmp/Makefile \
 		plugins/omoracle/Makefile \
 		plugins/omudpspoof/Makefile \
+		plugins/mmnormalize/Makefile \
 		plugins/cust1/Makefile \
 		java/Makefile \
 		tests/Makefile])
@@ -1114,6 +1137,9 @@ echo "---{ parser modules }---"
 echo "    pmrfc3164sd module will be compiled:      $enable_pmrfc3164sd"
 echo "    pmlastmsg module will be compiled:        $enable_pmlastmsg"
 echo
+echo "---{ message modification modules }---"
+echo "    mmnormalize module will be compiled:      $enable_mmnormalize"
+echo
 echo "---{ database support }---"
 echo "    MySql support enabled:                    $enable_mysql"
 echo "    libdbi support enabled:                   $enable_libdbi"
diff --git a/plugins/mmnormalize/Makefile.am b/plugins/mmnormalize/Makefile.am
new file mode 100644
index 00000000..cda11e2d
--- /dev/null
+++ b/plugins/mmnormalize/Makefile.am
@@ -0,0 +1,8 @@
+pkglib_LTLIBRARIES = mmnormalize.la
+
+mmnormalize_la_SOURCES = mmnormalize.c
+mmnormalize_la_CPPFLAGS =  $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(LIBLOGNORM_CFLAGS)
+mmnormalize_la_LDFLAGS = -module -avoid-version
+mmnormalize_la_LIBADD = 
+
+EXTRA_DIST = 
diff --git a/plugins/mmnormalize/mmnormalize.c b/plugins/mmnormalize/mmnormalize.c
new file mode 100644
index 00000000..6c0293cf
--- /dev/null
+++ b/plugins/mmnormalize/mmnormalize.c
@@ -0,0 +1,261 @@
+/* mmnormalize.c
+ * This is a message modification module. It normalizes the input message with
+ * the help of liblognorm. The messages EE event structure is updated.
+ *
+ * NOTE: read comments in module-template.h for details on the calling interface!
+ *
+ * File begun on 2010-01-01 by RGerhards
+ *
+ * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <signal.h>
+#include <errno.h>
+#include <unistd.h>
+#include <libestr.h>
+#include <libee/libee.h>
+#include <liblognorm.h>
+#include "conf.h"
+#include "syslogd-types.h"
+#include "template.h"
+#include "module-template.h"
+#include "errmsg.h"
+#include "cfsysline.h"
+#include "dirty.h"
+
+MODULE_TYPE_OUTPUT
+
+static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal);
+
+/* static data */
+DEFobjCurrIf(errmsg);
+
+/* internal structures
+ */
+DEF_OMOD_STATIC_DATA
+
+typedef struct _instanceData {
+	sbool bUseRawMsg;	/**< use %rawmsg% instead of %msg% */
+	ln_ctx ctxln;		/**< context to be used for liblognorm */
+	ee_ctx ctxee;		/**< context to be used for libee */
+} instanceData;
+
+typedef struct configSettings_s {
+	uchar *sampdb;		/**< name of sample db to use */
+	sbool bUseRawMsg;	/**< use %rawmsg% instead of %msg% */
+} configSettings_t;
+
+SCOPING_SUPPORT; /* must be set AFTER configSettings_t is defined */
+
+BEGINinitConfVars		/* (re)set config variables to default values */
+CODESTARTinitConfVars 
+	resetConfigVariables(NULL, NULL);
+ENDinitConfVars
+
+
+BEGINcreateInstance
+CODESTARTcreateInstance
+ENDcreateInstance
+
+
+BEGINisCompatibleWithFeature
+CODESTARTisCompatibleWithFeature
+ENDisCompatibleWithFeature
+
+
+BEGINfreeInstance
+CODESTARTfreeInstance
+	ee_exitCtx(pData->ctxee);
+	ln_exitCtx(pData->ctxln);
+ENDfreeInstance
+
+
+BEGINdbgPrintInstInfo
+CODESTARTdbgPrintInstInfo
+	dbgprintf("mmnormalize\n");
+ENDdbgPrintInstInfo
+
+
+BEGINtryResume
+CODESTARTtryResume
+ENDtryResume
+
+BEGINdoAction
+	msg_t *pMsg;
+	es_str_t *str;
+	uchar *buf;
+	int len;
+	int r;
+CODESTARTdoAction
+	pMsg = (msg_t*) ppString[0];
+	/* note that we can performance-optimize the interface, but this also
+	 * requires changes to the libraries. For now, we accept message
+	 * duplication. -- rgerhards, 2010-12-01
+	 */
+	if(pData->bUseRawMsg) {
+		getRawMsg(pMsg, &buf, &len);
+	} else {
+		buf = getMSG(pMsg);
+		len = getMSGLen(pMsg);
+	}
+	str = es_newStrFromCStr((char*)buf, len);
+	r = ln_normalize(pData->ctxln, str, &pMsg->event);
+	if(r != 0) {
+		DBGPRINTF("error %d during ln_normalize\n", r);
+	}
+	es_deleteStr(str);
+ENDdoAction
+
+
+BEGINparseSelectorAct
+CODESTARTparseSelectorAct
+CODE_STD_STRING_REQUESTparseSelectorAct(1)
+	/* first check if this config line is actually for us */
+	if(strncmp((char*) p, ":mmnormalize:", sizeof(":mmnormalize:") - 1)) {
+		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
+	}
+
+	if(cs.sampdb == NULL) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: no sample database was specified, use "
+				"$MMNormalizeSampleDB directive first!");
+		ABORT_FINALIZE(RS_RET_NO_RULESET);
+	}
+
+	/* ok, if we reach this point, we have something for us */
+	p += sizeof(":mmnormalize:") - 1; /* eat indicator sequence  (-1 because of '\0'!) */
+	CHKiRet(createInstance(&pData));
+
+	/* check if a non-standard template is to be applied */
+	if(*(p-1) == ';')
+		--p;
+	/* we call the function below because we need to call it via our interface definition. However,
+	 * the format specified (if any) is always ignored.
+	 */
+	CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_TPL_AS_MSG, (uchar*) "RSYSLOG_FileFormat"));
+
+	/* finally build the instance */
+	if((pData->ctxee = ee_initCtx()) == NULL) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize libee ctx, cannot "
+				"activate action");
+		ABORT_FINALIZE(RS_RET_ERR_LIBEE_INIT);
+	}
+
+	if((pData->ctxln = ln_initCtx()) == NULL) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize liblognorm ctx, cannot "
+				"activate action");
+		ee_exitCtx(pData->ctxee);
+		ABORT_FINALIZE(RS_RET_ERR_LIBLOGNORM_INIT);
+	}
+	if(ln_loadSamples(pData->ctxln, (char*) cs.sampdb) != 0) {
+		errmsg.LogError(0, RS_RET_NO_RULESET, "error: sample db '%s' could not be loaded "
+				"cannot activate action", cs.sampdb);
+		ee_exitCtx(pData->ctxee);
+		ln_exitCtx(pData->ctxln);
+		ABORT_FINALIZE(RS_RET_ERR_LIBLOGNORM_SAMPDB_LOAD);
+	}
+	pData->bUseRawMsg = cs.bUseRawMsg;
+
+	/* all config vars auto-reset! */
+	cs.bUseRawMsg = 0;
+	cs.sampdb = NULL;
+CODE_STD_FINALIZERparseSelectorAct
+ENDparseSelectorAct
+
+
+BEGINmodExit
+CODESTARTmodExit
+	objRelease(errmsg, CORE_COMPONENT);
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_OMOD_QUERIES
+ENDqueryEtryPt
+
+
+
+/* Reset config variables for this module to default values.
+ */
+static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal)
+{
+	DEFiRet;
+	cs.sampdb = NULL;
+	cs.bUseRawMsg = 0;
+	RETiRet;
+}
+
+/* set the sampdb name */
+static rsRetVal
+setSampDB(void __attribute__((unused)) *pVal, uchar *pszName)
+{
+	DEFiRet;
+	cs.sampdb = pszName;
+	pszName = NULL;
+	RETiRet;
+}
+
+BEGINmodInit()
+	rsRetVal localRet;
+	rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts);
+	unsigned long opts;
+	int bMsgPassingSupported;
+CODESTARTmodInit
+SCOPINGmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION;
+		/* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	/* check if the rsyslog core supports parameter passing code */
+	bMsgPassingSupported = 0;
+	localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts",
+			&pomsrGetSupportedTplOpts);
+	if(localRet == RS_RET_OK) {
+		/* found entry point, so let's see if core supports msg passing */
+		CHKiRet((*pomsrGetSupportedTplOpts)(&opts));
+		if(opts & OMSR_TPL_AS_MSG)
+			bMsgPassingSupported = 1;
+	} else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) {
+		ABORT_FINALIZE(localRet); /* Something else went wrong, not acceptable */
+	}
+	
+	if(!bMsgPassingSupported) {
+		DBGPRINTF("mmnormalize: msg-passing is not supported by rsyslog core, "
+			  "can not continue.\n");
+		ABORT_FINALIZE(RS_RET_NO_MSG_PASSING);
+	}
+
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmnormalizesampledb", 0, eCmdHdlrGetWord,
+				    setSampDB, NULL, STD_LOADABLE_MODULE_ID, eConfObjAction));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"mmnormalizeuserawmsg", 0, eCmdHdlrInt,
+				    NULL, &cs.bUseRawMsg, STD_LOADABLE_MODULE_ID, eConfObjAction));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler,
+				    resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID, eConfObjAction));
+ENDmodInit
+
+/* vi:set ai:
+ */
diff --git a/runtime/msg.c b/runtime/msg.c
index 82565f18..346389fc 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -1198,7 +1198,7 @@ char *getProtocolVersionString(msg_t *pM)
 }
 
 
-static inline void
+void
 getRawMsg(msg_t *pM, uchar **pBuf, int *piLen)
 {
 	if(pM == NULL) {
diff --git a/runtime/msg.h b/runtime/msg.h
index 4897959c..b699366e 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -33,6 +33,7 @@
 #include "syslogd-types.h"
 #include "template.h"
 #include "atomic.h"
+#include "libee/libee.h"
 
 
 /* rgerhards 2004-11-08: The following structure represents a
@@ -105,6 +106,7 @@ struct msg {
 				   it obviously is solved in way or another...). */
 	struct syslogTime tRcvdAt;/* time the message entered this program */
 	struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */
+	struct ee_event	*event;	/**< libee event */
 	/* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */
 	uchar szRawMsg[CONF_RAWMSG_BUFSIZE];	/* most messages are small, and these are stored here (without malloc/free!) */
 	uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE];
@@ -170,6 +172,7 @@ uchar *getRcvFrom(msg_t *pM);
 void getTAG(msg_t *pM, uchar **ppBuf, int *piLen);
 char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt);
 char *getPRI(msg_t *pMsg);
+void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen);
 
 
 /* TODO: remove these five (so far used in action.c) */
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index c2f6463c..686b9ba1 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -346,6 +346,9 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_FILE_NOT_SPECIFIED = -2180, /**< file name not configured where this was required */
 
 	RS_RET_INVLD_CONF_OBJ= -2200,	/**< invalid config object (e.g. $Begin conf statement) */
+	RS_RET_ERR_LIBEE_INIT = -2201,	/**< cannot obtain libee ctx */
+	RS_RET_ERR_LIBLOGNORM_INIT = -2202,/**< cannot obtain liblognorm ctx */
+	RS_RET_ERR_LIBLOGNORM_SAMPDB_LOAD = -2203,/**< liblognorm sampledb load failed */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */