moved rfc3164/5424 code to new parser modules

another milestone commit: the program works, the new interface
is used, some more cleanup is needed and the per-ruleset config
options are still missing. But we are getting closer...

12 files changed, 594 insertions, 420 deletions

diff --git a/action.c b/action.c
index e58d2e85..06d1095c 100644
--- a/action.c
+++ b/action.c
@@ -882,7 +882,6 @@ submitBatch(action_t *pAction, batch_t *pBatch, int nElem, int *pbShutdownImmedi
 
 	bDone = 0;
 	do {
-dbgprintf("XXX: submitBatch in loop, batch size %d\n", nElem);
 		localRet = tryDoAction(pAction, pBatch, &nElem, pbShutdownImmediate);
 		if(localRet == RS_RET_FORCE_TERM)
 			FINALIZE;
diff --git a/runtime/parser.c b/runtime/parser.c
index f59d1974..2c764739 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -40,6 +40,7 @@
 #include "datetime.h"
 #include "errmsg.h"
 #include "parser.h"
+#include "ruleset.h"
 #include "unicode-helper.h"
 #include "dirty.h"
 #include "cfsysline.h"
@@ -52,21 +53,25 @@ DEFobjStaticHelpers
 DEFobjCurrIf(glbl)
 DEFobjCurrIf(errmsg)
 DEFobjCurrIf(datetime)
+DEFobjCurrIf(ruleset)
 
 /* static data */
-static int bParseHOSTNAMEandTAG;	/* cache for the equally-named global param - performance enhancement */
 
 /* config data */
 static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */
 static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */
 static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */
 
-/* we create a small helper list of parsers so that we can obtain their
- * handles if the config needs one. This is also used to unload all modules on
- * shutdown.
+/* This is the list of all parsers known to us.
+ * This is also used to unload all modules on shutdown.
  */
 parserList_t *pParsLstRoot = NULL;
 
+/* this is the list of the default parsers, to be used if no others
+ * are specified.
+ */
+parserList_t *pDfltParsLst = NULL;
+
 
 /* intialize (but NOT allocate) a parser list. Primarily meant as a hook
  * which can be used to extend the list in the future. So far, just sets
@@ -139,6 +144,27 @@ finalize_it:
 /* --- END helper functions for parser list handling --- */
 
 
+/* Add a an already existing parser to the default list. As usual, order
+ * of calls is important (most importantly, that means the legacy parser,
+ * which can process everything, MUST be added last!).
+ * rgerhards, 2009-11-04
+ */
+static rsRetVal
+AddDfltParser(uchar *pName)
+{
+	parser_t *pParser;
+	DEFiRet;
+
+	CHKiRet(FindParser(&pParser, pName));
+	CHKiRet(AddParserToList(&pDfltParsLst, pParser));
+	dbgprintf("Parser '%s' added to default parser set.\n", pName);
+	
+finalize_it:
+	RETiRet;
+}
+
+
+
 BEGINobjConstruct(parser) /* be sure to specify the object type also in END macro! */
 ENDobjConstruct(parser)
 
@@ -152,7 +178,7 @@ rsRetVal parserConstructFinalize(parser_t *pThis)
 
 	ISOBJ_TYPE_assert(pThis, parser);
 	CHKiRet(AddParserToList(&pParsLstRoot, pThis));
-	DBGPRINTF("parser '%s' added to list of available parser\n", pThis->pName);
+	DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName);
 
 finalize_it:
 	RETiRet;
@@ -164,383 +190,6 @@ CODESTARTobjDestruct(parser)
 	//TODO: free module!
 ENDobjDestruct(parser)
 
-/***************************RFC 5425 PARSER ******************************************************/
-
-
-/* Helper to parseRFCSyslogMsg. This function parses a field up to
- * (and including) the SP character after it. The field contents is
- * returned in a caller-provided buffer. The parsepointer is advanced
- * to after the terminating SP. The caller must ensure that the 
- * provided buffer is large enough to hold the to be extracted value.
- * Returns 0 if everything is fine or 1 if either the field is not
- * SP-terminated or any other error occurs. -- rger, 2005-11-24
- * The function now receives the size of the string and makes sure
- * that it does not process more than that. The *pLenStr counter is
- * updated on exit. -- rgerhards, 2009-09-23
- */
-static int parseRFCField(uchar **pp2parse, uchar *pResult, int *pLenStr)
-{
-	uchar *p2parse;
-	int iRet = 0;
-
-	assert(pp2parse != NULL);
-	assert(*pp2parse != NULL);
-	assert(pResult != NULL);
-
-	p2parse = *pp2parse;
-
-	/* this is the actual parsing loop */
-	while(*pLenStr > 0  && *p2parse != ' ') {
-		*pResult++ = *p2parse++;
-		--(*pLenStr);
-	}
-
-	if(*pLenStr > 0 && *p2parse == ' ') {
-		++p2parse; /* eat SP, but only if not at end of string */
-		--(*pLenStr);
-	} else {
-		iRet = 1; /* there MUST be an SP! */
-	}
-	*pResult = '\0';
-
-	/* set the new parse pointer */
-	*pp2parse = p2parse;
-	return 0;
-}
-
-
-/* Helper to parseRFCSyslogMsg. This function parses the structured
- * data field of a message. It does NOT parse inside structured data,
- * just gets the field as whole. Parsing the single entities is left
- * to other functions. The parsepointer is advanced
- * to after the terminating SP. The caller must ensure that the 
- * provided buffer is large enough to hold the to be extracted value.
- * Returns 0 if everything is fine or 1 if either the field is not
- * SP-terminated or any other error occurs. -- rger, 2005-11-24
- * The function now receives the size of the string and makes sure
- * that it does not process more than that. The *pLenStr counter is
- * updated on exit. -- rgerhards, 2009-09-23
- */
-static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr)
-{
-	uchar *p2parse;
-	int bCont = 1;
-	int iRet = 0;
-	int lenStr;
-
-	assert(pp2parse != NULL);
-	assert(*pp2parse != NULL);
-	assert(pResult != NULL);
-
-	p2parse = *pp2parse;
-	lenStr = *pLenStr;
-
-	/* this is the actual parsing loop
-	 * Remeber: structured data starts with [ and includes any characters
-	 * until the first ] followed by a SP. There may be spaces inside
-	 * structured data. There may also be \] inside the structured data, which
-	 * do NOT terminate an element.
-	 */
-	if(lenStr == 0 || *p2parse != '[')
-		return 1; /* this is NOT structured data! */
-
-	if(*p2parse == '-') { /* empty structured data? */
-		*pResult++ = '-';
-		++p2parse;
-		--lenStr;
-	} else {
-		while(bCont) {
-			if(lenStr < 2) {
-				/* we now need to check if we have only structured data */
-				if(lenStr > 0 && *p2parse == ']') {
-					*pResult++ = *p2parse;
-					p2parse++;
-					lenStr--;
-					bCont = 0;
-				} else {
-					iRet = 1; /* this is not valid! */
-					bCont = 0;
-				}
-			} else if(*p2parse == '\\' && *(p2parse+1) == ']') {
-				/* this is escaped, need to copy both */
-				*pResult++ = *p2parse++;
-				*pResult++ = *p2parse++;
-				lenStr -= 2;
-			} else if(*p2parse == ']' && *(p2parse+1) == ' ') {
-				/* found end, just need to copy the ] and eat the SP */
-				*pResult++ = *p2parse;
-				p2parse += 2;
-				lenStr -= 2;
-				bCont = 0;
-			} else {
-				*pResult++ = *p2parse++;
-				--lenStr;
-			}
-		}
-	}
-
-	if(lenStr > 0 && *p2parse == ' ') {
-		++p2parse; /* eat SP, but only if not at end of string */
-		--lenStr;
-	} else {
-		iRet = 1; /* there MUST be an SP! */
-	}
-	*pResult = '\0';
-
-	/* set the new parse pointer */
-	*pp2parse = p2parse;
-	*pLenStr = lenStr;
-	return 0;
-}
-
-/* parse a RFC5424-formatted syslog message. This function returns
- * 0 if processing of the message shall continue and 1 if something
- * went wrong and this messe should be ignored. This function has been
- * implemented in the effort to support syslog-protocol. Please note that
- * the name (parse *RFC*) stems from the hope that syslog-protocol will
- * some time become an RFC. Do not confuse this with informational
- * RFC 3164 (which is legacy syslog).
- *
- * currently supported format:
- *
- * <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP [SD-ID]s SP MSG
- *
- * <PRI> is already stripped when this function is entered. VERSION already
- * has been confirmed to be "1", but has NOT been stripped from the message.
- *
- * rger, 2005-11-24
- */
-static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
-{
-	uchar *p2parse;
-	uchar *pBuf;
-	int lenMsg;
-	int bContParse = 1;
-
-	BEGINfunc
-	assert(pMsg != NULL);
-	assert(pMsg->pszRawMsg != NULL);
-	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
-	lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI;
-
-	/* do a sanity check on the version and eat it (the caller checked this already) */
-	assert(p2parse[0] == '1' && p2parse[1] == ' ');
-	p2parse += 2;
-	lenMsg -= 2;
-
-	/* Now get us some memory we can use as a work buffer while parsing.
-	 * We simply allocated a buffer sufficiently large to hold all of the
-	 * message, so we can not run into any troubles. I think this is
-	 * more wise then to use individual buffers.
-	 */
-	if((pBuf = MALLOC(sizeof(uchar) * (lenMsg + 1))) == NULL)
-		return 1;
-		
-	/* IMPORTANT NOTE:
-	 * Validation is not actually done below nor are any errors handled. I have
-	 * NOT included this for the current proof of concept. However, it is strongly
-	 * advisable to add it when this code actually goes into production.
-	 * rgerhards, 2005-11-24
-	 */
-
-	/* TIMESTAMP */
-	if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP),  &p2parse, &lenMsg) == RS_RET_OK) {
-		if(flags & IGNDATE) {
-			/* we need to ignore the msg data, so simply copy over reception date */
-			memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
-		}
-	} else {
-		DBGPRINTF("no TIMESTAMP detected!\n");
-		bContParse = 0;
-	}
-
-	/* HOSTNAME */
-	if(bContParse) {
-		parseRFCField(&p2parse, pBuf, &lenMsg);
-		MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf));
-	}
-
-	/* APP-NAME */
-	if(bContParse) {
-		parseRFCField(&p2parse, pBuf, &lenMsg);
-		MsgSetAPPNAME(pMsg, (char*)pBuf);
-	}
-
-	/* PROCID */
-	if(bContParse) {
-		parseRFCField(&p2parse, pBuf, &lenMsg);
-		MsgSetPROCID(pMsg, (char*)pBuf);
-	}
-
-	/* MSGID */
-	if(bContParse) {
-		parseRFCField(&p2parse, pBuf, &lenMsg);
-		MsgSetMSGID(pMsg, (char*)pBuf);
-	}
-
-	/* STRUCTURED-DATA */
-	if(bContParse) {
-		parseRFCStructuredData(&p2parse, pBuf, &lenMsg);
-		MsgSetStructuredData(pMsg, (char*)pBuf);
-	}
-
-	/* MSG */
-	MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg);
-
-	free(pBuf);
-	ENDfunc
-	return 0; /* all ok */
-}
-
-
-/*********************** END RFC5425 PARSER ******************************************/
-
-/***************************RFC 5425 PARSER ******************************************************/
-
-
-/* parse a legay-formatted syslog message. This function returns
- * 0 if processing of the message shall continue and 1 if something
- * went wrong and this messe should be ignored. This function has been
- * implemented in the effort to support syslog-protocol.
- * rger, 2005-11-24
- * As of 2006-01-10, I am removing the logic to continue parsing only
- * when a valid TIMESTAMP is detected. Validity of other fields already
- * is ignored. This is due to the fact that the parser has grown smarter
- * and is now more able to understand different dialects of the syslog
- * message format. I do not expect any bad side effects of this change,
- * but I thought I log it in this comment.
- * rgerhards, 2006-01-10
- */
-static int parseLegacySyslogMsg(msg_t *pMsg, int flags)
-{
-	uchar *p2parse;
-	int lenMsg;
-	int bTAGCharDetected;
-	int i;	/* general index for parsing */
-	uchar bufParseTAG[CONF_TAG_MAXSIZE];
-	uchar bufParseHOSTNAME[CONF_TAG_HOSTNAME];
-	BEGINfunc
-
-	assert(pMsg != NULL);
-	assert(pMsg->pszRawMsg != NULL);
-	lenMsg = pMsg->iLenRawMsg - (pMsg->offAfterPRI + 1);
-	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
-
-	/* Check to see if msg contains a timestamp. We start by assuming
-	 * that the message timestamp is the time of reception (which we 
-	 * generated ourselfs and then try to actually find one inside the
-	 * message. There we go from high-to low precison and are done
-	 * when we find a matching one. -- rgerhards, 2008-09-16
-	 */
-	if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) {
-		/* we are done - parse pointer is moved by ParseTIMESTAMP3339 */;
-	} else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) {
-		/* we are done - parse pointer is moved by ParseTIMESTAMP3164 */;
-	} else if(*p2parse == ' ' && lenMsg > 1) { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */
-		++p2parse;	/* move over space */
-		--lenMsg;
-		if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) {
-			/* indeed, we got it! */
-			/* we are done - parse pointer is moved by ParseTIMESTAMP3164 */;
-		} else {/* parse pointer needs to be restored, as we moved it off-by-one
-			 * for this try.
-			 */
-			--p2parse;
-			++lenMsg;
-		}
-	}
-
-	if(flags & IGNDATE) {
-		/* we need to ignore the msg data, so simply copy over reception date */
-		memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
-	}
-
-	/* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we 
-	 * do this only when the user has not forbidden this. I now introduce some
-	 * code that allows a user to configure rsyslogd to treat the rest of the
-	 * message as MSG part completely. In this case, the hostname will be the
-	 * machine that we received the message from and the tag will be empty. This
-	 * is meant to be an interim solution, but for now it is in the code.
-	 */
-	if(bParseHOSTNAMEandTAG && !(flags & INTERNAL_MSG)) {
-		/* parse HOSTNAME - but only if this is network-received!
-		 * rger, 2005-11-14: we still have a problem with BSD messages. These messages
-		 * do NOT include a host name. In most cases, this leads to the TAG to be treated
-		 * as hostname and the first word of the message as the TAG. Clearly, this is not
-		 * of advantage ;) I think I have now found a way to handle this situation: there
-		 * are certain characters which are frequently used in TAG (e.g. ':'), which are
-		 * *invalid* in host names. So while parsing the hostname, I check for these characters.
-		 * If I find them, I set a simple flag but continue. After parsing, I check the flag.
-		 * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change
-		 * the fields. I think this logic shall work with any type of syslog message.
-		 * rgerhards, 2009-06-23: and I now have extended this logic to every character
-		 * that is not a valid hostname.
-		 */
-		bTAGCharDetected = 0;
-		if(lenMsg > 0 && flags & PARSE_HOSTNAME) {
-			i = 0;
-			while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.'
-				|| p2parse[i] == '_' || p2parse[i] == '-') && i < CONF_TAG_MAXSIZE) {
-				bufParseHOSTNAME[i] = p2parse[i];
-				++i;
-			}
-
-			if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) {
-				/* we got a hostname! */
-				p2parse += i + 1; /* "eat" it (including SP delimiter) */
-				lenMsg -= i + 1;
-				bufParseHOSTNAME[i] = '\0';
-				MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i);
-			}
-		}
-
-		/* now parse TAG - that should be present in message from all sources.
-		 * This code is somewhat not compliant with RFC 3164. As of 3164,
-		 * the TAG field is ended by any non-alphanumeric character. In
-		 * practice, however, the TAG often contains dashes and other things,
-		 * which would end the TAG. So it is not desirable. As such, we only
-		 * accept colon and SP to be terminators. Even there is a slight difference:
-		 * a colon is PART of the TAG, while a SP is NOT part of the tag
-		 * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character
-		 * size limit (from RFC3164) on the tag. This had bad effects on existing
-		 * envrionments, as sysklogd didn't obey it either (probably another bug
-		 * in RFC3164...). We now receive the full size, but will modify the
-		 * outputs so that only 32 characters max are used by default.
-		 */
-		i = 0;
-		while(lenMsg > 0 && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) {
-			bufParseTAG[i++] = *p2parse++;
-			--lenMsg;
-		}
-		if(lenMsg > 0 && *p2parse == ':') {
-			++p2parse; 
-			--lenMsg;
-			bufParseTAG[i++] = ':';
-		}
-
-		/* no TAG can only be detected if the message immediatly ends, in which case an empty TAG
-		 * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23
-		 */
-		bufParseTAG[i] = '\0';	/* terminate string */
-		MsgSetTAG(pMsg, bufParseTAG, i);
-	} else {/* we enter this code area when the user has instructed rsyslog NOT
-		 * to parse HOSTNAME and TAG - rgerhards, 2006-03-13
-		 */
-		if(!(flags & INTERNAL_MSG)) {
-			DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n");
-		}
-	}
-
-	/* The rest is the actual MSG */
-	MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg);
-
-	ENDfunc
-	return 0; /* all ok */
-}
-
-
-/***************************END RFC 5425 PARSER ******************************************************/
-
 
 /* uncompress a received message if it is compressed.
  * pMsg->pszRawMsg buffer is updated.
@@ -645,10 +294,6 @@ SanitizeMsg(msg_t *pMsg)
 	assert(pMsg != NULL);
 	assert(pMsg->iLenRawMsg > 0);
 
-#	ifdef USE_NETZIP
-	CHKiRet(uncompressMessage(pMsg));
-#	endif
-
 	pszMsg = pMsg->pszRawMsg;
 	lenMsg = pMsg->iLenRawMsg;
 
@@ -740,15 +385,22 @@ finalize_it:
 static rsRetVal
 ParseMsg(msg_t *pMsg)
 {
-	DEFiRet;
 	uchar *msg;
 	int pri;
 	int lenMsg;
 	int iPriText;
+	rsRetVal localRet;
+	parserList_t *pParserList;
+	static int iErrMsgRateLimiter = 0;
+	DEFiRet;
 
 	if(pMsg->iLenRawMsg == 0)
 		ABORT_FINALIZE(RS_RET_EMPTY_MSG);
 
+#	ifdef USE_NETZIP
+	CHKiRet(uncompressMessage(pMsg));
+#	endif
+
 	CHKiRet(SanitizeMsg(pMsg));
 
 	/* we needed to sanitize first, because we otherwise do not have a C-string we can print... */
@@ -777,25 +429,35 @@ ParseMsg(msg_t *pMsg)
 	pMsg->iSeverity = LOG_PRI(pri);
 	MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg);
 
-	/* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have
-	 * a traditional syslog message or one formatted according to syslog-protocol.
-	 * We need to apply different parsers depending on that. We use the
-	 * -protocol VERSION field for the detection.
+	/* we now need to go through our list of parsers and see which one is capable of
+	 * parsing the message.
 	 */
-	if(msg[0] == '1' && msg[1] == ' ') {
-		dbgprintf("Message has syslog-protocol format.\n");
-		setProtocolVersion(pMsg, 1);
-		if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) {
-			msgDestruct(&pMsg);
-			ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases!
-		}
-	} else { /* we have legacy syslog */
-		dbgprintf("Message has legacy syslog format.\n");
-		setProtocolVersion(pMsg, 0);
-		if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) {
-			msgDestruct(&pMsg);
-			ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases!
+	pParserList = ruleset.GetParserList(pMsg);
+	if(pParserList == NULL)
+		pParserList = pDfltParsLst;
+	DBGPRINTF("Using parser list %p%s.\n", pParserList,
+		  (pParserList == pDfltParsLst) ? " (the default list)" : "");
+
+	while(pParserList != NULL) {
+		localRet = pParserList->pParser->pModule->mod.pm.parse(pMsg);
+		if(localRet != RS_RET_COULD_NOT_PARSE)
+			break;
+		pParserList = pParserList->pNext;
+	}
+
+	/* We need to log a warning message and drop the message if we did not find a parser.
+	 * Note that we log at most the first 1000 message, as this may very well be a problem
+	 * that causes a message generation loop. We do not synchronize that counter, it doesn't
+	 * matter if we log a handful messages more than we should...
+	 */
+	if(localRet != RS_RET_OK) {
+		if(++iErrMsgRateLimiter > 1000) {
+			errmsg.LogError(0, localRet, "Error: one message could not be processed by "
+			 	"any parser, message is being discarded (start of raw msg: '%50s')", 
+				pMsg->pszRawMsg);
 		}
+		DBGPRINTF("No parser could process the message (state %d), we need to discard it.\n", localRet);
+		ABORT_FINALIZE(localRet);
 	}
 
 	/* finalize message object */
@@ -877,6 +539,7 @@ CODESTARTobjQueryInterface(parser)
 	pIf->SanitizeMsg = SanitizeMsg;
 	pIf->InitParserList = InitParserList;
 	pIf->AddParserToList = AddParserToList;
+	pIf->AddDfltParser = AddDfltParser;
 	pIf->FindParser = FindParser;
 finalize_it:
 ENDobjQueryInterface(parser)
@@ -902,13 +565,11 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus
  * rgerhards, 2009-11-02
  */
 BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */
-//BEGINAbstractObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */
 	/* request objects we use */
 	CHKiRet(objUse(glbl, CORE_COMPONENT));
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 	CHKiRet(objUse(datetime, CORE_COMPONENT));
-
-	bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */
+	CHKiRet(objUse(ruleset, CORE_COMPONENT));
 
 	CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL));
@@ -916,5 +577,6 @@ BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */
 	CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL));
 
 	InitParserList(&pParsLstRoot);
+	InitParserList(&pDfltParsLst);
 ENDObjClassInit(parser)
 
diff --git a/runtime/parser.h b/runtime/parser.h
index 47521f63..d600fb19 100644
--- a/runtime/parser.h
+++ b/runtime/parser.h
@@ -56,6 +56,7 @@ BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */
 	/* static functions */
 	rsRetVal (*ParseMsg)(msg_t *pMsg);
 	rsRetVal (*SanitizeMsg)(msg_t *pMsg);
+	rsRetVal (*AddDfltParser)(uchar *);
 ENDinterface(parser)
 #define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */
 
diff --git a/runtime/queue.c b/runtime/queue.c
index ed4ba83e..1c885925 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1350,16 +1350,12 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch)
 	ISOBJ_TYPE_assert(pThis, qqueue);
 	assert(pBatch != NULL);
 
-dbgprintf("XXX: deleteProcessedBatch total entries %d with state[0] %d\n", pBatch->nElem, pBatch->pElem[0].state);
 	for(i = 0 ; i < pBatch->nElem ; ++i) {
-dbgprintf("XXX: deleteProcessedBatch delete entry %d, ptr %p, refcnt %d with state %d\n",
-i, pBatch->pElem[i].pUsrp, ((msg_t*)pBatch->pElem[i].pUsrp)->iRefCount, pBatch->pElem[i].state);
 		pUsr = pBatch->pElem[i].pUsrp;
 		if(   pBatch->pElem[i].state == BATCH_STATE_RDY
 		   || pBatch->pElem[i].state == BATCH_STATE_SUB) {
 			localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY,
 				       (obj_t*)MsgAddRef((msg_t*) pUsr));
-dbgprintf("we need to requeue the entry, refcnt now %d\n", ((msg_t*) pUsr)->iRefCount);
 			++nEnqueued;
 			if(localRet != RS_RET_OK) {
 				DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet);
@@ -1368,7 +1364,7 @@ dbgprintf("we need to requeue the entry, refcnt now %d\n", ((msg_t*) pUsr)->iRef
 		objDestruct(pUsr);
 	}
 
-dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued);
+	dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued);
 
 	if(nEnqueued > 0)
 		qqueueChkPersist(pThis, nEnqueued);
@@ -1656,7 +1652,8 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti)
 	d_pthread_mutex_lock(pThis->mut);
 
 finalize_it:
-dbgprintf("XXX: regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
+	dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet,
+	          getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
 	RETiRet;
 }
 
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 4d029c08..a5e8cf5d 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -408,6 +408,7 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_RULESET_NOT_FOUND = -2157,/**< a required ruleset could not be found */
 	RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */
 	RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */
+	RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
diff --git a/runtime/ruleset.c b/runtime/ruleset.c
index 975ed3ed..5e1bfced 100644
--- a/runtime/ruleset.c
+++ b/runtime/ruleset.c
@@ -167,6 +167,17 @@ dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet);
 }
 
 
+/* return the ruleset-assigned parser list. NULL means use the default
+ * parser list.
+ * rgerhards, 2009-11-04
+ */
+static parserList_t*
+GetParserList(msg_t *pMsg)
+{
+	return (pMsg->pRuleset == NULL) ? NULL : pMsg->pRuleset->pParserLst;
+}
+
+
 /* Add a new rule to the end of the current rule set. We do a number
  * of checks and ignore the rule if it does not pass them.
  */
@@ -467,6 +478,7 @@ CODESTARTobjQueryInterface(ruleset)
 	pIf->SetDefaultRuleset = SetDefaultRuleset;
 	pIf->SetCurrRuleset = SetCurrRuleset;
 	pIf->GetRulesetQueue = GetRulesetQueue;
+	pIf->GetParserList = GetParserList;
 finalize_it:
 ENDobjQueryInterface(ruleset)
 
diff --git a/runtime/ruleset.h b/runtime/ruleset.h
index 6c8f64b5..222d773e 100644
--- a/runtime/ruleset.h
+++ b/runtime/ruleset.h
@@ -54,8 +54,10 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */
 	rsRetVal (*SetCurrRuleset)(uchar*);
 	ruleset_t* (*GetCurrent)(void);
 	qqueue_t* (*GetRulesetQueue)(ruleset_t*);
+	/* v3, 2009-11-04 */
+	parserList_t* (*GetParserList)(msg_t *);
 ENDinterface(ruleset)
-#define rulesetCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */
+#define rulesetCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */
 
 
 /* prototypes */
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 1dff4b45..32d6843b 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -17,6 +17,8 @@ rsyslogd_SOURCES = \
 	omdiscard.h \
 	pmrfc5424.c \
 	pmrfc5424.h \
+	pmrfc3164.c \
+	pmrfc3164.h \
 	iminternal.c \
 	iminternal.h \
 	pidfile.c \
diff --git a/tools/pmrfc3164.c b/tools/pmrfc3164.c
new file mode 100644
index 00000000..b106d3d0
--- /dev/null
+++ b/tools/pmrfc3164.c
@@ -0,0 +1,228 @@
+/* pmrfc3164.c
+ * This is a parser module for RFC3164(legacy syslog)-formatted messages.
+ *
+ * NOTE: read comments in module-template.h to understand how this file
+ *       works!
+ *
+ * File begun on 2009-11-04 by RGerhards
+ *
+ * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <ctype.h>
+#include "syslogd.h"
+#include "conf.h"
+#include "syslogd-types.h"
+#include "template.h"
+#include "msg.h"
+#include "module-template.h"
+#include "glbl.h"
+#include "errmsg.h"
+#include "parser.h"
+#include "datetime.h"
+#include "unicode-helper.h"
+
+MODULE_TYPE_PARSER
+PARSER_NAME("rsyslog.rfc3164")
+
+/* internal structures
+ */
+DEF_PMOD_STATIC_DATA
+DEFobjCurrIf(errmsg)
+DEFobjCurrIf(glbl)
+DEFobjCurrIf(parser)
+DEFobjCurrIf(datetime)
+
+
+/* static data */
+static int bParseHOSTNAMEandTAG;	/* cache for the equally-named global param - performance enhancement */
+
+
+BEGINisCompatibleWithFeature
+CODESTARTisCompatibleWithFeature
+ENDisCompatibleWithFeature
+
+
+/* parse a legay-formatted syslog message.
+ */
+BEGINparse
+	uchar *p2parse;
+	int lenMsg;
+	int bTAGCharDetected;
+	int i;	/* general index for parsing */
+	uchar bufParseTAG[CONF_TAG_MAXSIZE];
+	uchar bufParseHOSTNAME[CONF_TAG_HOSTNAME];
+CODESTARTparse
+	dbgprintf("Message will now be parsed by the legacy syslog parser (one size fits all... ;)).\n");
+	assert(pMsg != NULL);
+	assert(pMsg->pszRawMsg != NULL);
+	lenMsg = pMsg->iLenRawMsg - (pMsg->offAfterPRI + 1);
+	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
+	setProtocolVersion(pMsg, 0);
+
+	/* Check to see if msg contains a timestamp. We start by assuming
+	 * that the message timestamp is the time of reception (which we 
+	 * generated ourselfs and then try to actually find one inside the
+	 * message. There we go from high-to low precison and are done
+	 * when we find a matching one. -- rgerhards, 2008-09-16
+	 */
+	if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) {
+		/* we are done - parse pointer is moved by ParseTIMESTAMP3339 */;
+	} else if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) {
+		/* we are done - parse pointer is moved by ParseTIMESTAMP3164 */;
+	} else if(*p2parse == ' ' && lenMsg > 1) { /* try to see if it is slighly malformed - HP procurve seems to do that sometimes */
+		++p2parse;	/* move over space */
+		--lenMsg;
+		if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse, &lenMsg) == RS_RET_OK) {
+			/* indeed, we got it! */
+			/* we are done - parse pointer is moved by ParseTIMESTAMP3164 */;
+		} else {/* parse pointer needs to be restored, as we moved it off-by-one
+			 * for this try.
+			 */
+			--p2parse;
+			++lenMsg;
+		}
+	}
+
+	if(pMsg->msgFlags & IGNDATE) {
+		/* we need to ignore the msg data, so simply copy over reception date */
+		memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
+	}
+
+	/* rgerhards, 2006-03-13: next, we parse the hostname and tag. But we 
+	 * do this only when the user has not forbidden this. I now introduce some
+	 * code that allows a user to configure rsyslogd to treat the rest of the
+	 * message as MSG part completely. In this case, the hostname will be the
+	 * machine that we received the message from and the tag will be empty. This
+	 * is meant to be an interim solution, but for now it is in the code.
+	 */
+	if(bParseHOSTNAMEandTAG && !(pMsg->msgFlags & INTERNAL_MSG)) {
+		/* parse HOSTNAME - but only if this is network-received!
+		 * rger, 2005-11-14: we still have a problem with BSD messages. These messages
+		 * do NOT include a host name. In most cases, this leads to the TAG to be treated
+		 * as hostname and the first word of the message as the TAG. Clearly, this is not
+		 * of advantage ;) I think I have now found a way to handle this situation: there
+		 * are certain characters which are frequently used in TAG (e.g. ':'), which are
+		 * *invalid* in host names. So while parsing the hostname, I check for these characters.
+		 * If I find them, I set a simple flag but continue. After parsing, I check the flag.
+		 * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change
+		 * the fields. I think this logic shall work with any type of syslog message.
+		 * rgerhards, 2009-06-23: and I now have extended this logic to every character
+		 * that is not a valid hostname.
+		 */
+		bTAGCharDetected = 0;
+		if(lenMsg > 0 && pMsg->msgFlags & PARSE_HOSTNAME) {
+			i = 0;
+			while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.'
+				|| p2parse[i] == '_' || p2parse[i] == '-') && i < CONF_TAG_MAXSIZE) {
+				bufParseHOSTNAME[i] = p2parse[i];
+				++i;
+			}
+
+			if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) {
+				/* we got a hostname! */
+				p2parse += i + 1; /* "eat" it (including SP delimiter) */
+				lenMsg -= i + 1;
+				bufParseHOSTNAME[i] = '\0';
+				MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i);
+			}
+		}
+
+		/* now parse TAG - that should be present in message from all sources.
+		 * This code is somewhat not compliant with RFC 3164. As of 3164,
+		 * the TAG field is ended by any non-alphanumeric character. In
+		 * practice, however, the TAG often contains dashes and other things,
+		 * which would end the TAG. So it is not desirable. As such, we only
+		 * accept colon and SP to be terminators. Even there is a slight difference:
+		 * a colon is PART of the TAG, while a SP is NOT part of the tag
+		 * (it is CONTENT). Starting 2008-04-04, we have removed the 32 character
+		 * size limit (from RFC3164) on the tag. This had bad effects on existing
+		 * envrionments, as sysklogd didn't obey it either (probably another bug
+		 * in RFC3164...). We now receive the full size, but will modify the
+		 * outputs so that only 32 characters max are used by default.
+		 */
+		i = 0;
+		while(lenMsg > 0 && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) {
+			bufParseTAG[i++] = *p2parse++;
+			--lenMsg;
+		}
+		if(lenMsg > 0 && *p2parse == ':') {
+			++p2parse; 
+			--lenMsg;
+			bufParseTAG[i++] = ':';
+		}
+
+		/* no TAG can only be detected if the message immediatly ends, in which case an empty TAG
+		 * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23
+		 */
+		bufParseTAG[i] = '\0';	/* terminate string */
+		MsgSetTAG(pMsg, bufParseTAG, i);
+	} else {/* we enter this code area when the user has instructed rsyslog NOT
+		 * to parse HOSTNAME and TAG - rgerhards, 2006-03-13
+		 */
+		if(!(pMsg->msgFlags & INTERNAL_MSG)) {
+			DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n");
+		}
+	}
+
+	/* The rest is the actual MSG */
+	MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg);
+ENDparse
+
+
+BEGINmodExit
+CODESTARTmodExit
+	/* release what we no longer need */
+	objRelease(errmsg, CORE_COMPONENT);
+	objRelease(glbl, CORE_COMPONENT);
+	objRelease(parser, CORE_COMPONENT);
+	objRelease(datetime, CORE_COMPONENT);
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_PMOD_QUERIES
+CODEqueryEtryPt_IsCompatibleWithFeature_IF_OMOD_QUERIES
+ENDqueryEtryPt
+
+
+BEGINmodInit(pmrfc3164)
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(glbl, CORE_COMPONENT));
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	CHKiRet(objUse(parser, CORE_COMPONENT));
+	CHKiRet(objUse(datetime, CORE_COMPONENT));
+
+	dbgprintf("rfc3164 parser init called\n");
+ 	bParseHOSTNAMEandTAG = glbl.GetParseHOSTNAMEandTAG(); /* cache value, is set only during rsyslogd option processing */
+
+
+ENDmodInit
+
+/* vim:set ai:
+ */
diff --git a/tools/pmrfc3164.h b/tools/pmrfc3164.h
new file mode 100644
index 00000000..24304094
--- /dev/null
+++ b/tools/pmrfc3164.h
@@ -0,0 +1,33 @@
+/* pmrfc3164.h
+ * These are the definitions for the RFC3164 parser module.
+ *
+ * File begun on 2009-11-04 by RGerhards
+ *
+ * Copyright 2009 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#ifndef	PMRFC3164_H_INCLUDED
+#define	PMRFC3164_H_INCLUDED 1
+
+/* prototypes */
+rsRetVal modInitpmrfc3164(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*);
+
+#endif /* #ifndef PMRFC3164_H_INCLUDED */
+/* vi:set ai:
+ */
diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c
index acc21817..2fa2c981 100644
--- a/tools/pmrfc5424.c
+++ b/tools/pmrfc5424.c
@@ -27,6 +27,7 @@
  */
 #include "config.h"
 #include "rsyslog.h"
+#include <stdlib.h>
 #include <string.h>
 #include <assert.h>
 #include <errno.h>
@@ -39,6 +40,7 @@
 #include "glbl.h"
 #include "errmsg.h"
 #include "parser.h"
+#include "datetime.h"
 #include "unicode-helper.h"
 
 MODULE_TYPE_PARSER
@@ -50,6 +52,7 @@ DEF_PMOD_STATIC_DATA
 DEFobjCurrIf(errmsg)
 DEFobjCurrIf(glbl)
 DEFobjCurrIf(parser)
+DEFobjCurrIf(datetime)
 
 
 /* config data */
@@ -60,8 +63,231 @@ CODESTARTisCompatibleWithFeature
 ENDisCompatibleWithFeature
 
 
+/* Helper to parseRFCSyslogMsg. This function parses a field up to
+ * (and including) the SP character after it. The field contents is
+ * returned in a caller-provided buffer. The parsepointer is advanced
+ * to after the terminating SP. The caller must ensure that the 
+ * provided buffer is large enough to hold the to be extracted value.
+ * Returns 0 if everything is fine or 1 if either the field is not
+ * SP-terminated or any other error occurs. -- rger, 2005-11-24
+ * The function now receives the size of the string and makes sure
+ * that it does not process more than that. The *pLenStr counter is
+ * updated on exit. -- rgerhards, 2009-09-23
+ */
+static int parseRFCField(uchar **pp2parse, uchar *pResult, int *pLenStr)
+{
+	uchar *p2parse;
+	int iRet = 0;
+
+	assert(pp2parse != NULL);
+	assert(*pp2parse != NULL);
+	assert(pResult != NULL);
+
+	p2parse = *pp2parse;
+
+	/* this is the actual parsing loop */
+	while(*pLenStr > 0  && *p2parse != ' ') {
+		*pResult++ = *p2parse++;
+		--(*pLenStr);
+	}
+
+	if(*pLenStr > 0 && *p2parse == ' ') {
+		++p2parse; /* eat SP, but only if not at end of string */
+		--(*pLenStr);
+	} else {
+		iRet = 1; /* there MUST be an SP! */
+	}
+	*pResult = '\0';
+
+	/* set the new parse pointer */
+	*pp2parse = p2parse;
+	return 0;
+}
+
+
+/* Helper to parseRFCSyslogMsg. This function parses the structured
+ * data field of a message. It does NOT parse inside structured data,
+ * just gets the field as whole. Parsing the single entities is left
+ * to other functions. The parsepointer is advanced
+ * to after the terminating SP. The caller must ensure that the 
+ * provided buffer is large enough to hold the to be extracted value.
+ * Returns 0 if everything is fine or 1 if either the field is not
+ * SP-terminated or any other error occurs. -- rger, 2005-11-24
+ * The function now receives the size of the string and makes sure
+ * that it does not process more than that. The *pLenStr counter is
+ * updated on exit. -- rgerhards, 2009-09-23
+ */
+static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr)
+{
+	uchar *p2parse;
+	int bCont = 1;
+	int iRet = 0;
+	int lenStr;
+
+	assert(pp2parse != NULL);
+	assert(*pp2parse != NULL);
+	assert(pResult != NULL);
+
+	p2parse = *pp2parse;
+	lenStr = *pLenStr;
+
+	/* this is the actual parsing loop
+	 * Remeber: structured data starts with [ and includes any characters
+	 * until the first ] followed by a SP. There may be spaces inside
+	 * structured data. There may also be \] inside the structured data, which
+	 * do NOT terminate an element.
+	 */
+	if(lenStr == 0 || *p2parse != '[')
+		return 1; /* this is NOT structured data! */
+
+	if(*p2parse == '-') { /* empty structured data? */
+		*pResult++ = '-';
+		++p2parse;
+		--lenStr;
+	} else {
+		while(bCont) {
+			if(lenStr < 2) {
+				/* we now need to check if we have only structured data */
+				if(lenStr > 0 && *p2parse == ']') {
+					*pResult++ = *p2parse;
+					p2parse++;
+					lenStr--;
+					bCont = 0;
+				} else {
+					iRet = 1; /* this is not valid! */
+					bCont = 0;
+				}
+			} else if(*p2parse == '\\' && *(p2parse+1) == ']') {
+				/* this is escaped, need to copy both */
+				*pResult++ = *p2parse++;
+				*pResult++ = *p2parse++;
+				lenStr -= 2;
+			} else if(*p2parse == ']' && *(p2parse+1) == ' ') {
+				/* found end, just need to copy the ] and eat the SP */
+				*pResult++ = *p2parse;
+				p2parse += 2;
+				lenStr -= 2;
+				bCont = 0;
+			} else {
+				*pResult++ = *p2parse++;
+				--lenStr;
+			}
+		}
+	}
+
+	if(lenStr > 0 && *p2parse == ' ') {
+		++p2parse; /* eat SP, but only if not at end of string */
+		--lenStr;
+	} else {
+		iRet = 1; /* there MUST be an SP! */
+	}
+	*pResult = '\0';
+
+	/* set the new parse pointer */
+	*pp2parse = p2parse;
+	*pLenStr = lenStr;
+	return 0;
+}
+
+/* parse a RFC5424-formatted syslog message. This function returns
+ * 0 if processing of the message shall continue and 1 if something
+ * went wrong and this messe should be ignored. This function has been
+ * implemented in the effort to support syslog-protocol. Please note that
+ * the name (parse *RFC*) stems from the hope that syslog-protocol will
+ * some time become an RFC. Do not confuse this with informational
+ * RFC 3164 (which is legacy syslog).
+ *
+ * currently supported format:
+ *
+ * <PRI>VERSION SP TIMESTAMP SP HOSTNAME SP APP-NAME SP PROCID SP MSGID SP [SD-ID]s SP MSG
+ *
+ * <PRI> is already stripped when this function is entered. VERSION already
+ * has been confirmed to be "1", but has NOT been stripped from the message.
+ *
+ * rger, 2005-11-24
+ */
+//static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 BEGINparse
+	uchar *p2parse;
+	uchar *pBuf = NULL;
+	int lenMsg;
+	int bContParse = 1;
 CODESTARTparse
+	assert(pMsg != NULL);
+	assert(pMsg->pszRawMsg != NULL);
+	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
+	lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI;
+
+	/* check if we are the right parser */
+	if(lenMsg < 2 || p2parse[0] != '1' || p2parse[1] != ' ') {
+		ABORT_FINALIZE(RS_RET_COULD_NOT_PARSE);
+	}
+	DBGPRINTF("Message has RFC5424/syslog-protocol format.\n");
+	setProtocolVersion(pMsg, 1);
+	p2parse += 2;
+	lenMsg -= 2;
+
+	/* Now get us some memory we can use as a work buffer while parsing.
+	 * We simply allocated a buffer sufficiently large to hold all of the
+	 * message, so we can not run into any troubles. I think this is
+	 * wiser than to use individual buffers.
+	 */
+	CHKmalloc(pBuf = MALLOC(sizeof(uchar) * (lenMsg + 1)));
+		
+	/* IMPORTANT NOTE:
+	 * Validation is not actually done below nor are any errors handled. I have
+	 * NOT included this for the current proof of concept. However, it is strongly
+	 * advisable to add it when this code actually goes into production.
+	 * rgerhards, 2005-11-24
+	 */
+
+	/* TIMESTAMP */
+	if(datetime.ParseTIMESTAMP3339(&(pMsg->tTIMESTAMP),  &p2parse, &lenMsg) == RS_RET_OK) {
+		if(pMsg->msgFlags & IGNDATE) {
+			/* we need to ignore the msg data, so simply copy over reception date */
+			memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
+		}
+	} else {
+		DBGPRINTF("no TIMESTAMP detected!\n");
+		bContParse = 0;
+	}
+
+	/* HOSTNAME */
+	if(bContParse) {
+		parseRFCField(&p2parse, pBuf, &lenMsg);
+		MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf));
+	}
+
+	/* APP-NAME */
+	if(bContParse) {
+		parseRFCField(&p2parse, pBuf, &lenMsg);
+		MsgSetAPPNAME(pMsg, (char*)pBuf);
+	}
+
+	/* PROCID */
+	if(bContParse) {
+		parseRFCField(&p2parse, pBuf, &lenMsg);
+		MsgSetPROCID(pMsg, (char*)pBuf);
+	}
+
+	/* MSGID */
+	if(bContParse) {
+		parseRFCField(&p2parse, pBuf, &lenMsg);
+		MsgSetMSGID(pMsg, (char*)pBuf);
+	}
+
+	/* STRUCTURED-DATA */
+	if(bContParse) {
+		parseRFCStructuredData(&p2parse, pBuf, &lenMsg);
+		MsgSetStructuredData(pMsg, (char*)pBuf);
+	}
+
+	/* MSG */
+	MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg);
+
+finalize_it:
+	if(pBuf != NULL)
+		free(pBuf);
 ENDparse
 
 
@@ -71,6 +297,7 @@ CODESTARTmodExit
 	objRelease(errmsg, CORE_COMPONENT);
 	objRelease(glbl, CORE_COMPONENT);
 	objRelease(parser, CORE_COMPONENT);
+	objRelease(datetime, CORE_COMPONENT);
 ENDmodExit
 
 
@@ -88,6 +315,7 @@ CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(glbl, CORE_COMPONENT));
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 	CHKiRet(objUse(parser, CORE_COMPONENT));
+	CHKiRet(objUse(datetime, CORE_COMPONENT));
 
 	dbgprintf("rfc5424 parser init called\n");
 	dbgprintf("GetParserName addr %p\n", GetParserName);
diff --git a/tools/syslogd.c b/tools/syslogd.c
index aeab292c..5b7f1e65 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -33,7 +33,7 @@
  *       to the database).
  *
  * rsyslog - An Enhanced syslogd Replacement.
- * Copyright 2003-2008 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2003-2009 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -126,6 +126,7 @@
 #include "omfile.h"
 #include "omdiscard.h"
 #include "pmrfc5424.h"
+#include "pmrfc3164.h"
 #include "threads.h"
 #include "wti.h"
 #include "queue.h"
@@ -346,7 +347,6 @@ static char **crunch_list(char *list);
 static void reapchild();
 static void debug_switch();
 static void sighup_handler();
-static void processImInternal(void);
 
 
 static int usage(void)
@@ -600,6 +600,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShu
 {
 	int i;
 	msg_t *pMsg;
+	rsRetVal localRet;
 	DEFiRet;
 
 	assert(pBatch != NULL);
@@ -608,9 +609,12 @@ msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch, int *pbShu
 		pMsg = (msg_t*) pBatch->pElem[i].pUsrp;
 		DBGPRINTF("msgConsumer processes msg %d/%d\n", i, pBatch->nElem);
 		if((pMsg->msgFlags & NEEDS_PARSING) != 0) {
-			parser.ParseMsg(pMsg);
+			localRet = parser.ParseMsg(pMsg);
+			if(localRet == RS_RET_OK)
+				ruleset.ProcessMsg(pMsg);
+		} else {
+			ruleset.ProcessMsg(pMsg);
 		}
-		ruleset.ProcessMsg(pMsg);
 		/* if we reach this point, the message is considered committed (by definition!) */
 		pBatch->pElem[i].state = BATCH_STATE_COMM;
 	}
@@ -1738,7 +1742,7 @@ void sigttin_handler()
  * really help us. TODO: add error messages?
  * rgerhards, 2007-08-03
  */
-static void processImInternal(void)
+static inline void processImInternal(void)
 {
 	int iPri;
 	msg_t *pMsg;
@@ -1888,6 +1892,11 @@ static rsRetVal loadBuildInModules(void)
 
 	/* load build-in parser modules */
 	CHKiRet(module.doModInit(modInitpmrfc5424, UCHAR_CONSTANT("builtin-pmrfc5424"), NULL));
+	CHKiRet(module.doModInit(modInitpmrfc3164, UCHAR_CONSTANT("builtin-pmrfc3164"), NULL));
+
+	/* and set default parser modules (order is *very* important, legacy (3164) parse needs to go last! */
+	CHKiRet(parser.AddDfltParser(UCHAR_CONSTANT("rsyslog.rfc5424")));
+	CHKiRet(parser.AddDfltParser(UCHAR_CONSTANT("rsyslog.rfc3164")));
 
 	/* ok, initialization of the command handler probably does not 100% belong right in
 	 * this space here. However, with the current design, this is actually quite a good