Merge branch 'omfile'

Conflicts:
	ChangeLog

8 files changed, 642 insertions, 1050 deletions

diff --git a/tools/Makefile.am b/tools/Makefile.am
index e523b854..f0f9afab 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -26,8 +26,10 @@ rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS)
 rsyslogd_LDFLAGS = -export-dynamic
 
 if ENABLE_DIAGTOOLS
-sbin_PROGRAMS += rsyslog_diag_hostname msggen
+sbin_PROGRAMS += rsyslog_diag_hostname msggen zpipe
 rsyslog_diag_hostname_SOURCES = gethostn.c
+zpipe_SOURCES = zpipe.c
+zpipe_LDADD = -lz
 msggen_SOURCES = msggen.c
 endif
 
diff --git a/tools/omfile.c b/tools/omfile.c
index 3e845a73..b62d9c57 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -12,7 +12,13 @@
  * of the "old" message code without any modifications. However, it
  * helps to have things at the right place one we go to the meat of it.
  *
- * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH.
+ * A large re-write of this file was done in June, 2009. The focus was
+ * to introduce many more features (like zipped writing), clean up the code
+ * and make it more reliable. In short, that rewrite tries to provide a new
+ * solid basis for the next three to five years to come. During it, bugs
+ * may have been introduced ;) -- rgerhards, 2009-06-04
+ *
+ * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -41,6 +47,7 @@
 #include <assert.h>
 #include <errno.h>
 #include <ctype.h>
+#include <libgen.h>
 #include <unistd.h>
 #include <sys/file.h>
 
@@ -48,7 +55,7 @@
 #	include <fcntl.h>
 #endif
 
-#include "syslogd.h"
+#include "conf.h"
 #include "syslogd-types.h"
 #include "srUtils.h"
 #include "template.h"
@@ -57,6 +64,9 @@
 #include "cfsysline.h"
 #include "module-template.h"
 #include "errmsg.h"
+#include "stream.h"
+#include "zlibw.h"
+#include "unicode-helper.h"
 
 MODULE_TYPE_OUTPUT
 
@@ -64,17 +74,22 @@ MODULE_TYPE_OUTPUT
  */
 DEF_OMOD_STATIC_DATA
 DEFobjCurrIf(errmsg)
+DEFobjCurrIf(zlibw)
+DEFobjCurrIf(strm)
 
 /* The following structure is a dynafile name cache entry.
  */
 struct s_dynaFileCacheEntry {
-	uchar *pName;	/* name currently open, if dynamic name */
-	short	fd;		/* name associated with file name in cache */
-	time_t	lastUsed;	/* for LRU - last access */
+	uchar *pName;		/* name currently open, if dynamic name */
+	strm_t	*pStrm;		/* our output stream */
+	time_t	lastUsed;	/* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) 
 };
 typedef struct s_dynaFileCacheEntry dynaFileCacheEntry;
 
 
+#define IOBUF_DFLT_SIZE 1024	/* default size for io buffers */
+#define FLUSH_INTRVL_DFLT 1 	/* default buffer flush interval (in seconds) */
+
 /* globals for default values */
 static int iDynaFileCacheSize = 10; /* max cache for dynamic files */
 static int fCreateMode = 0644; /* mode to use when creating files */
@@ -86,19 +101,17 @@ static uid_t	dirUID;		/* UID to be used for newly created directories */
 static uid_t	dirGID;		/* GID to be used for newly created directories */
 static int	bCreateDirs;	/* auto-create directories for dynaFiles: 0 - no, 1 - yes */
 static int	bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */
+static int	iZipLevel = 0;	/* zip compression mode (0..9 as usual) */
+static bool	bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */
+static int	iIOBufSize = IOBUF_DFLT_SIZE;	/* size of an io buffer */
+static int	iFlushInterval = FLUSH_INTRVL_DFLT; 	/* how often flush the output buffer on inactivity? */
 static uchar	*pszTplName = NULL; /* name of the default template to use */
 /* end globals for default values */
 
 
 typedef struct _instanceData {
 	uchar	f_fname[MAXFNAME];/* file or template name (display only) */
-	short	fd;		  /* file descriptor for (current) file */
-	enum {
-		eTypeFILE,
-		eTypeTTY,
-		eTypeCONSOLE,
-		eTypePIPE
-	} fileType;	
+	strm_t	*pStrm;		/* our output stream */
 	char	bDynamicName;	/* 0 - static name, 1 - dynamic name (with properties) */
 	int	fCreateMode;	/* file creation mode for open() */
 	int	fDirCreateMode;	/* creation mode for mkdir() */
@@ -117,8 +130,12 @@ typedef struct _instanceData {
 	 * pointer points to the overall structure.
 	 */
 	dynaFileCacheEntry **dynCache;
-	off_t	f_sizeLimit;		/* file size limit, 0 = no limit */
-	char	*f_sizeLimitCmd;	/* command to carry out when size limit is reached */
+	off_t	iSizeLimit;		/* file size limit, 0 = no limit */
+	uchar	*pszSizeLimitCmd;	/* command to carry out when size limit is reached */
+	int 	iZipLevel;		/* zip mode to use for this selector */
+	int	iIOBufSize;		/* size of associated io buffer */
+	int	iFlushInterval;		/* how fast flush buffer on inactivity? */
+	bool	bFlushOnTXEnd;		/* flush write buffers when transaction has ended? */
 } instanceData;
 
 
@@ -147,7 +164,7 @@ CODESTARTdbgPrintInstInfo
 			);
 	} else { /* regular file */
 		dbgprintf("%s", pData->f_fname);
-		if (pData->fd == -1)
+		if (pData->pStrm == NULL)
 			dbgprintf(" (unused)");
 	}
 ENDdbgPrintInstInfo
@@ -168,13 +185,13 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal)
 		errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg);
 		iRet = RS_RET_VAL_OUT_OF_RANGE;
 		iNewVal = 1;
-	} else if(iNewVal > 10000) {
+	} else if(iNewVal > 1000) {
 		snprintf((char*) errMsg, sizeof(errMsg)/sizeof(uchar),
-		         "DynaFileCacheSize maximum is 10,000 (%d given), changed to 10,000.", iNewVal);
+		         "DynaFileCacheSize maximum is 1,000 (%d given), changed to 1,000.", iNewVal);
 		errno = 0;
 		errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg);
 		iRet = RS_RET_VAL_OUT_OF_RANGE;
-		iNewVal = 10000;
+		iNewVal = 1000;
 	}
 
 	iDynaFileCacheSize = iNewVal;
@@ -199,14 +216,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR
 	struct outchannel *pOch;
 	char szBuf[128];	/* should be more than sufficient */
 
-	/* this must always be a file, because we can not set a size limit
-	 * on a pipe...
-	 * rgerhards 2005-06-21: later, this will be a separate type, but let's
-	 * emulate things for the time being. When everything runs, we can
-	 * extend it...
-	 */
-	pData->fileType = eTypeFILE;
-
 	++p; /* skip '$' */
 	i = 0;
 	/* get outchannel name */
@@ -241,12 +250,12 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR
 	}
 
 	/* OK, we finally got a correct template. So let's use it... */
-	strncpy((char*) pData->f_fname, (char*) pOch->pszFileTemplate, MAXFNAME);
-	pData->f_sizeLimit = pOch->uSizeLimit;
+	ustrncpy(pData->f_fname, pOch->pszFileTemplate, MAXFNAME);
+	pData->iSizeLimit = pOch->uSizeLimit;
 	/* WARNING: It is dangerous "just" to pass the pointer. As we
 	 * never rebuild the output channel description, this is acceptable here.
 	 */
-	pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit;
+	pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit;
 
 	iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts,
 				       (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName);
@@ -256,64 +265,6 @@ finalize_it:
 }
 
 
-/* rgerhards 2005-06-21: Try to resolve a size limit
- * situation. This first runs the command, and then
- * checks if we are still above the treshold.
- * returns 0 if ok, 1 otherwise
- * TODO: consider moving the initial check in here, too
- */
-int resolveFileSizeLimit(instanceData *pData)
-{
-	uchar *pParams;
-	uchar *pCmd;
-	uchar *p;
-	off_t actualFileSize;
-	ASSERT(pData != NULL);
-
-	if(pData->f_sizeLimitCmd == NULL)
-		return 1; /* nothing we can do in this case... */
-	
-	/* the execProg() below is probably not great, but at least is is
-	 * fairly secure now. Once we change the way file size limits are
-	 * handled, we should also revisit how this command is run (and
-	 * with which parameters).   rgerhards, 2007-07-20
-	 */
-	/* we first check if we have command line parameters. We assume this, 
-	 * when we have a space in the program name. If we find it, everything after
-	 * the space is treated as a single argument.
-	 */
-	if((pCmd = (uchar*)strdup((char*)pData->f_sizeLimitCmd)) == NULL) {
-		/* there is not much we can do - we make syslogd close the file in this case */
-		return 1;
-		}
-
-	for(p = pCmd ; *p && *p != ' ' ; ++p) {
-		/* JUST SKIP */
-	}
-
-	if(*p == ' ') {
-		*p = '\0'; /* pretend string-end */
-		pParams = p+1;
-	} else
-		pParams = NULL;
-
-	execProg(pCmd, 1, pParams);
-
-	free(pCmd);
-
-	pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
-			pData->fCreateMode);
-
-	actualFileSize = lseek(pData->fd, 0, SEEK_END);
-	if(actualFileSize >= pData->f_sizeLimit) {
-		/* OK, it didn't work out... */
-		return 1;
-		}
-
-	return 0;
-}
-
-
 /* This function deletes an entry from the dynamic file name
  * cache. A pointer to the cache must be passed in as well
  * as the index of the to-be-deleted entry. This index may
@@ -321,23 +272,24 @@ int resolveFileSizeLimit(instanceData *pData)
  * function immediately returns. Parameter bFreeEntry is 1
  * if the entry should be d_free()ed and 0 if not.
  */
-static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry)
+static rsRetVal
+dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry)
 {
+	DEFiRet;
 	ASSERT(pCache != NULL);
 
-	BEGINfunc;
-
 	if(pCache[iEntry] == NULL)
 		FINALIZE;
 
 	DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry,
-		pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName);
+		pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName);
 	/* if the name is NULL, this is an improperly initilized entry which
 	 * needs to be discarded. In this case, neither the file is to be closed
 	 * not the name to be freed.
 	 */
 	if(pCache[iEntry]->pName != NULL) {
-		close(pCache[iEntry]->fd);
+		if(pCache[iEntry]->pStrm != NULL)
+			strm.Destruct(&pCache[iEntry]->pStrm);
 		d_free(pCache[iEntry]->pName);
 		pCache[iEntry]->pName = NULL;
 	}
@@ -348,7 +300,7 @@ static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int b
 	}
 
 finalize_it:
-	ENDfunc;
+	RETiRet;
 }
 
 
@@ -356,7 +308,8 @@ finalize_it:
  * relevant files. Part of Shutdown and HUP processing.
  * rgerhards, 2008-10-23
  */
-static inline void dynaFileFreeCacheEntries(instanceData *pData)
+static inline void
+dynaFileFreeCacheEntries(instanceData *pData)
 {
 	register int i;
 	ASSERT(pData != NULL);
@@ -392,25 +345,18 @@ static void dynaFileFreeCache(instanceData *pData)
 static rsRetVal
 prepareFile(instanceData *pData, uchar *newFileName)
 {
+	int fd;
 	DEFiRet;
-	if(pData->fileType == eTypePIPE) {
-		pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC);
-		FINALIZE; /* we are done in this case */
-	}
 
-	if(access((char*)newFileName, F_OK) == 0) {
-		/* file already exists */
-		pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
-				pData->fCreateMode);
-	} else {
-		pData->fd = -1;
+	if(access((char*)newFileName, F_OK) != 0) {
 		/* file does not exist, create it (and eventually parent directories */
+		fd = -1;
 		if(pData->bCreateDirs) {
 			/* We first need to create parent dirs if they are missing.
 			 * We do not report any errors here ourselfs but let the code
 			 * fall through to error handler below.
 			 */
-			if(makeFileParentDirs(newFileName, strlen((char*)newFileName),
+			if(makeFileParentDirs(newFileName, ustrlen(newFileName),
 			     pData->fDirCreateMode, pData->dirUID,
 			     pData->dirGID, pData->bFailOnChown) != 0) {
 			     	ABORT_FINALIZE(RS_RET_ERR); /* we give up */
@@ -419,17 +365,17 @@ prepareFile(instanceData *pData, uchar *newFileName)
 		/* no matter if we needed to create directories or not, we now try to create
 		 * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater)
 		 */
-		pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
+		fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
 				pData->fCreateMode);
-		if(pData->fd != -1) {
+		if(fd != -1) {
 			/* check and set uid/gid */
 			if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) {
 				/* we need to set owner/group */
-				if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) {
+				if(fchown(fd, pData->fileUID, pData->fileGID) != 0) {
 					if(pData->bFailOnChown) {
 						int eSave = errno;
-						close(pData->fd);
-						pData->fd = -1;
+						close(fd);
+						fd = -1;
 						errno = eSave;
 					}
 					/* we will silently ignore the chown() failure
@@ -437,19 +383,37 @@ prepareFile(instanceData *pData, uchar *newFileName)
 					 */
 				}
 			}
+			close(fd); /* close again, as we need a stream further on */
 		}
 	}
-finalize_it:
-	/* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean
-	 * non-open file descriptor. Anyhow, I leave this comment for the time being to that if
-	 * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19
-	 */
-	if(pData->fd != -1 && isatty(pData->fd)) {
-		DBGPRINTF("file %d is a tty file\n", pData->fd);
-		pData->fileType = eTypeTTY;
-		untty();
-	}
 
+	/* the copies below are clumpsy, but there is no way around given the
+	 * anomalies in dirname() and basename() [they MODIFY the provided buffer...]
+	 */
+	uchar szNameBuf[MAXFNAME];
+	uchar szDirName[MAXFNAME];
+	uchar szBaseName[MAXFNAME];
+	ustrncpy(szNameBuf, newFileName, MAXFNAME);
+	ustrncpy(szDirName, (uchar*)dirname((char*)szNameBuf), MAXFNAME);
+	ustrncpy(szNameBuf, newFileName, MAXFNAME);
+	ustrncpy(szBaseName, (uchar*)basename((char*)szNameBuf), MAXFNAME);
+
+	CHKiRet(strm.Construct(&pData->pStrm));
+	CHKiRet(strm.SetFName(pData->pStrm, szBaseName, ustrlen(szBaseName)));
+	CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName)));
+	CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel));
+	CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize));
+	CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval));
+	CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND));
+	CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode));
+	CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile));
+	CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE));
+	CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit));
+	if(pData->pszSizeLimitCmd != NULL)
+		CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd)));
+	CHKiRet(strm.ConstructFinalize(pData->pStrm));
+	
+finalize_it:
 	RETiRet;
 }
 
@@ -462,15 +426,16 @@ finalize_it:
  * be written.
  * This is a helper to writeFile(). rgerhards, 2007-07-03
  */
-static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
+static inline rsRetVal
+prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
 {
 	time_t ttOldest; /* timestamp of oldest element */
 	int iOldest;
 	int i;
 	int iFirstFree;
+	rsRetVal localRet;
 	dynaFileCacheEntry **pCache;
-	
-	BEGINfunc
+	DEFiRet;
 
 	ASSERT(pData != NULL);
 	ASSERT(newFileName != NULL);
@@ -481,15 +446,17 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 	 * I *hope* this will be a performance enhancement.
 	 */
 	if(   (pData->iCurrElt != -1)
-	   && !strcmp((char*) newFileName, (char*) pCache[pData->iCurrElt]->pName)) {
+	   && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) {
 	   	/* great, we are all set */
-		pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */
-		return 0;
+		pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call!
+		// LRU needs only a strictly monotonically increasing counter, so such a one could do
+		FINALIZE;
 	}
 
 	/* ok, no luck. Now let's search the table if we find a matching spot.
 	 * While doing so, we also prepare for creation of a new one.
 	 */
+	pData->iCurrElt = -1;	/* invalid current element pointer */
 	iFirstFree = -1; /* not yet found */
 	iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */
 	ttOldest = time(NULL) + 1; /* there must always be an older one */
@@ -498,12 +465,12 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 			if(iFirstFree == -1)
 				iFirstFree = i;
 		} else { /* got an element, let's see if it matches */
-			if(!strcmp((char*) newFileName, (char*) pCache[i]->pName)) {
+			if(!ustrcmp(newFileName, pCache[i]->pName)) {
 				/* we found our element! */
-				pData->fd = pCache[i]->fd;
+				pData->pStrm = pCache[i]->pStrm;
 				pData->iCurrElt = i;
 				pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */
-				return 0;
+				FINALIZE;
 			}
 			/* did not find it - so lets keep track of the counters for LRU */
 			if(pCache[i]->lastUsed < ttOldest) {
@@ -524,18 +491,14 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 		iFirstFree = iOldest; /* this one *is* now free ;) */
 	} else {
 		/* we need to allocate memory for the cache structure */
-		pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry));
-		if(pCache[iFirstFree] == NULL) {
-			DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n");
-			return -1;
-		}
+		CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)));
 	}
 
 	/* Ok, we finally can open the file */
-	prepareFile(pData, newFileName); /* ignore exact error, we check fd below */
+	localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */
 
 	/* file is either open now or an error state set */
-	if(pData->fd == -1) {
+	if(pData->pStrm == NULL) {
 		/* do not report anything if the message is an internally-generated
 		 * message. Otherwise, we could run into a never-ending loop. The bad
 		 * news is that we also lose errors on startup messages, but so it is.
@@ -543,22 +506,43 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 		if(iMsgOpts & INTERNAL_MSG) {
 			DBGPRINTF("Could not open dynaFile, discarding message\n");
 		} else {
-			errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName);
+			errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName);
 		}
 		dynaFileDelCacheEntry(pCache, iFirstFree, 1);
-		pData->iCurrElt = -1;
-		return -1;
+		ABORT_FINALIZE(localRet);
 	}
 
-	pCache[iFirstFree]->fd = pData->fd;
-	pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */
-	pCache[iFirstFree]->lastUsed = time(NULL);
+	CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName));
+	pCache[iFirstFree]->pStrm = pData->pStrm;
+	pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance
 	pData->iCurrElt = iFirstFree;
 	DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName);
 
-	ENDfunc
+finalize_it:
+	RETiRet;
+}
+
+
+/* do the actual write process. This function is to be called once we are ready for writing.
+ * It will do buffered writes and persist data only when the buffer is full. Note that we must
+ * be careful to detect when the file handle changed.
+ * rgerhards, 2009-06-03
+ */
+static  rsRetVal
+doWrite(instanceData *pData, uchar *pszBuf, int lenBuf)
+{
+	DEFiRet;
+	ASSERT(pData != NULL);
+	ASSERT(pszBuf != NULL);
+
+dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf);
+	if(pData->pStrm != NULL){
+		CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf));
+		FINALIZE;
+	}
 
-	return 0;
+finalize_it:
+	RETiRet;
 }
 
 
@@ -566,10 +550,9 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
  * will be called for all outputs using file semantics,
  * for example also for pipes.
  */
-static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData)
+static rsRetVal
+writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData)
 {
-	off_t actualFileSize;
-	int iLenWritten;
 	DEFiRet;
 
 	ASSERT(pData != NULL);
@@ -578,106 +561,27 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa
 	 * check if it still is ok or a new file needs to be created
 	 */
 	if(pData->bDynamicName) {
-		if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0)
-			ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */
-	}
-	
-	if(pData->fd == -1) {
-		rsRetVal iRetLocal;
-		iRetLocal = prepareFile(pData, pData->f_fname);
-		if((iRetLocal != RS_RET_OK) || (pData->fd == -1))
-			ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */
-	}
-
-	/* create the message based on format specified */
-again:
-	/* check if we have a file size limit and, if so,
-	 * obey to it.
-	 */
-	if(pData->f_sizeLimit != 0) {
-		actualFileSize = lseek(pData->fd, 0, SEEK_END);
-		if(actualFileSize >= pData->f_sizeLimit) {
-			char errMsg[256];
-			/* for now, we simply disable a file once it is
-			 * beyond the maximum size. This is better than having
-			 * us aborted by the OS... rgerhards 2005-06-21
-			 */
-			(void) close(pData->fd);
-			/* try to resolve the situation */
-			if(resolveFileSizeLimit(pData) != 0) {
-				/* didn't work out, so disable... */
-				snprintf(errMsg, sizeof(errMsg),
-					 "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation",
-					 pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize);
-				errno = 0;
-				errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg);
-				ABORT_FINALIZE(RS_RET_DISABLE_ACTION);
-			} else {
-				snprintf(errMsg, sizeof(errMsg),
-					 "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation",
-					 pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize);
-				errno = 0;
-				errmsg.LogError(0, NO_ERRCODE, "%s", errMsg);
-			}
+		CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts));
+	} else { /* "regular", non-dynafile */
+		if(pData->pStrm == NULL) {
+			CHKiRet(prepareFile(pData, pData->f_fname));
 		}
 	}
 
-	iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0]));
-//dbgprintf("lenwritten: %d\n", iLenWritten);
-	if(iLenWritten < 0) {
-		int e = errno;
-		char errStr[1024];
-		rs_strerror_r(errno, errStr, sizeof(errStr));
-		DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr);
-
-		/* If a named pipe is full, we suspend this action for a while */
-		if(pData->fileType == eTypePIPE && e == EAGAIN)
-			ABORT_FINALIZE(RS_RET_SUSPENDED);
-
-		close(pData->fd);
-		pData->fd = -1; /* tell that fd is no longer open! */
-		if(pData->bDynamicName && pData->iCurrElt != -1) {
-			/* in this case, we need to invalidate the name in the cache, too
-			 * otherwise, an invalid fd may show up if we had a file name change.
-			 * rgerhards, 2009-03-19
-			 */
-			pData->dynCache[pData->iCurrElt]->fd = -1;
-		}
-		/* Check for EBADF on TTY's due to vhangup()
-		 * Linux uses EIO instead (mrn 12 May 96)
-		 */
-		if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE)
-#ifdef linux
-			&& e == EIO
-#else
-			&& e == EBADF
-#endif
-			) {
-			pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC);
-			if (pData->fd < 0) {
-				iRet = RS_RET_SUSPENDED;
-				errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname);
-			} else {
-				untty();
-				goto again;
-			}
-		} else {
-			iRet = RS_RET_SUSPENDED;
-			errno = e;
-			errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname);
-		}
-	} else if (pData->bSyncFile) {
-		fsync(pData->fd);
-	}
+	CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0]))));
 
 finalize_it:
+	if(iRet != RS_RET_OK) {
+		/* in v5, we shall return different states for message-cause failur (but only there!) */
+		iRet = RS_RET_SUSPENDED;
+	}
 	RETiRet;
 }
 
 
 BEGINcreateInstance
 CODESTARTcreateInstance
-	pData->fd = -1;
+	pData->pStrm = NULL;
 ENDcreateInstance
 
 
@@ -685,8 +589,8 @@ BEGINfreeInstance
 CODESTARTfreeInstance
 	if(pData->bDynamicName) {
 		dynaFileFreeCache(pData);
-	} else if(pData->fd != -1)
-		close(pData->fd);
+	} else if(pData->pStrm != NULL)
+		strm.Destruct(&pData->pStrm);
 ENDfreeInstance
 
 
@@ -696,39 +600,32 @@ ENDtryResume
 
 BEGINdoAction
 CODESTARTdoAction
-	DBGPRINTF(" (%s)\n", pData->f_fname);
-	iRet = writeFile(ppString, iMsgOpts, pData);
+	DBGPRINTF("file to log to: %s\n", pData->f_fname);
+	CHKiRet(writeFile(ppString, iMsgOpts, pData));
+	if(pData->bFlushOnTXEnd) {
+		/* TODO v5: do this in endTransaction only! */
+		CHKiRet(strm.Flush(pData->pStrm));
+	}
+finalize_it:
 ENDdoAction
 
 
 BEGINparseSelectorAct
 CODESTARTparseSelectorAct
-	/* yes, the if below is redundant, but I need it now. Will go away as
-	 * the code further changes.  -- rgerhards, 2007-07-25
-	 */
-	if(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-') {
-		if((iRet = createInstance(&pData)) != RS_RET_OK) {
-			ENDfunc
-			return iRet; /* this can not use RET_iRet! */
-		}
-	} else {
-		/* this is not clean, but we need it for the time being
-		 * TODO: remove when cleaning up modularization 
-		 */
-		ENDfunc
-		return RS_RET_CONFLINE_UNPROCESSED;
-	}
+	if(!(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-'))
+		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
+
+	CHKiRet(createInstance(&pData));
 
 	if(*p == '-') {
 		pData->bSyncFile = 0;
 		p++;
 	} else {
-		pData->bSyncFile = bEnableSync ? 1 : 0;
+		pData->bSyncFile = bEnableSync;
 	}
-	pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */
+	pData->iSizeLimit = 0; /* default value, use outchannels to configure! */
 
-	switch (*p)
-	{
+	switch(*p) {
         case '$':
 		CODE_STD_STRING_REQUESTparseSelectorAct(1)
 		/* rgerhards 2005-06-21: this is a special setting for output-channel
@@ -738,13 +635,8 @@ CODESTARTparseSelectorAct
 		 * rgerhards, 2007-07-24: output-channels will go away. We keep them
 		 * for compatibility reasons, but seems to have been a bad idea.
 		 */
-		if((iRet = cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)) == RS_RET_OK) {
-			pData->bDynamicName = 0;
-			pData->fCreateMode = fCreateMode; /* preserve current setting */
-			pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */
-			pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
-					 pData->fCreateMode);
-		}
+		CHKiRet(cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS));
+		pData->bDynamicName = 0;
 		break;
 
 	case '?': /* This is much like a regular file handle, but we need to obtain
@@ -752,84 +644,56 @@ CODESTARTparseSelectorAct
 		   */
 		CODE_STD_STRING_REQUESTparseSelectorAct(2)
 		++p; /* eat '?' */
-		if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName))
-		   != RS_RET_OK)
-			break;
+		CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
+				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName));
 		/* "filename" is actually a template name, we need this as string 1. So let's add it
 		 * to the pOMSR. -- rgerhards, 2007-07-27
 		 */
-		if((iRet = OMSRsetEntry(*ppOMSR, 1, (uchar*)strdup((char*) pData->f_fname), OMSR_NO_RQD_TPL_OPTS)) != RS_RET_OK)
-			break;
+		CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pData->f_fname), OMSR_NO_RQD_TPL_OPTS));
 
 		pData->bDynamicName = 1;
 		pData->iCurrElt = -1;		  /* no current element */
-		pData->fCreateMode = fCreateMode; /* freeze current setting */
-		pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */
-		pData->bCreateDirs = bCreateDirs;
-		pData->bFailOnChown = bFailOnChown;
-		pData->fileUID = fileUID;
-		pData->fileGID = fileGID;
-		pData->dirUID = dirUID;
-		pData->dirGID = dirGID;
-		pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */
-		/* we now allocate the cache table. We use calloc() intentionally, as we 
-		 * need all pointers to be initialized to NULL pointers.
-		 */
-		if((pData->dynCache = (dynaFileCacheEntry**)
-		    calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) {
-			iRet = RS_RET_OUT_OF_MEMORY;
-			DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n");
-		}
+		/* we now allocate the cache table */
+		CHKmalloc(pData->dynCache = (dynaFileCacheEntry**)
+				calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*)));
 		break;
 
         case '|':
 	case '/':
 		CODE_STD_STRING_REQUESTparseSelectorAct(1)
-		/* rgerhards, 2007-0726: first check if file or pipe */
-		if(*p == '|') {
-			pData->fileType = eTypePIPE;
-			++p;
-		} else {
-			pData->fileType = eTypeFILE;
-		}
-		/* rgerhards 2004-11-17: from now, we need to have different
-		 * processing, because after the first comma, the template name
-		 * to use is specified. So we need to scan for the first coma first
-		 * and then look at the rest of the line.
-		 */
-		if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName))
-		   != RS_RET_OK)
-			break;
-
+		CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
+				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName));
 		pData->bDynamicName = 0;
-		pData->fCreateMode = fCreateMode; /* preserve current setting */
-		pData->fDirCreateMode = fDirCreateMode;
-		pData->bCreateDirs = bCreateDirs;
-		pData->bFailOnChown = bFailOnChown;
-		pData->fileUID = fileUID;
-		pData->fileGID = fileGID;
-		pData->dirUID = dirUID;
-		pData->dirGID = dirGID;
-
-		/* at this stage, we ignore the return value of prepareFile, this is taken
-		 * care of in later steps. -- rgerhards, 2009-03-19
+		break;
+	default:
+		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
+	}
+
+	/* freeze current paremeters for this action */
+	pData->iDynaFileCacheSize = iDynaFileCacheSize;
+	pData->fCreateMode = fCreateMode;
+	pData->fDirCreateMode = fDirCreateMode;
+	pData->bCreateDirs = bCreateDirs;
+	pData->bFailOnChown = bFailOnChown;
+	pData->fileUID = fileUID;
+	pData->fileGID = fileGID;
+	pData->dirUID = dirUID;
+	pData->dirGID = dirGID;
+	pData->iZipLevel = iZipLevel;
+	pData->bFlushOnTXEnd = bFlushOnTXEnd;
+	pData->iIOBufSize = iIOBufSize;
+	pData->iFlushInterval = iFlushInterval;
+
+	if(pData->bDynamicName == 0) {
+		/* try open and emit error message if not possible. At this stage, we ignore the
+		 * return value of prepareFile, this is taken care of in later steps.
 		 */
 		prepareFile(pData, pData->f_fname);
 		        
-	  	if(pData->fd < 0 ) {
-			pData->fd = -1;
+	  	if(pData->pStrm == NULL) {
 			DBGPRINTF("Error opening log file: %s\n", pData->f_fname);
 			errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname);
-			break;
 		}
-		if(strcmp((char*) p, _PATH_CONSOLE) == 0)
-			pData->fileType = eTypeCONSOLE;
-		break;
-	default:
-		iRet = RS_RET_CONFLINE_UNPROCESSED;
-		break;
 	}
 CODE_STD_FINALIZERparseSelectorAct
 ENDparseSelectorAct
@@ -850,6 +714,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	fDirCreateMode = 0700;
 	bCreateDirs = 1;
 	bEnableSync = 0;
+	iZipLevel = 0;
+	bFlushOnTXEnd = 1;
+	iIOBufSize = IOBUF_DFLT_SIZE;
+	iFlushInterval = FLUSH_INTRVL_DFLT;
 	if(pszTplName != NULL) {
 		free(pszTplName);
 		pszTplName = NULL;
@@ -865,9 +733,9 @@ CODESTARTdoHUP
 		dynaFileFreeCacheEntries(pData);
 		pData->iCurrElt = -1; /* invalidate current element */
 	} else {
-		if(pData->fd != -1) {
-			close(pData->fd);
-			pData->fd = -1;
+		if(pData->pStrm != NULL) {
+			strm.Destruct(&pData->pStrm);
+			pData->pStrm = NULL;
 		}
 	}
 ENDdoHUP
@@ -875,8 +743,10 @@ ENDdoHUP
 
 BEGINmodExit
 CODESTARTmodExit
-	if(pszTplName != NULL)
-		free(pszTplName);
+	objRelease(errmsg, CORE_COMPONENT);
+	objRelease(strm, CORE_COMPONENT);
+	objRelease(zlibw, LM_ZLIBW_FILENAME);
+	free(pszTplName);
 ENDmodExit
 
 
@@ -892,7 +762,13 @@ CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME));
+	CHKiRet(objUse(strm, CORE_COMPONENT));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID));
diff --git a/tools/omfwd.c b/tools/omfwd.c
index c8fedfc9..2966a5e4 100644
--- a/tools/omfwd.c
+++ b/tools/omfwd.c
@@ -48,6 +48,7 @@
 #endif
 #include <pthread.h>
 #include "syslogd.h"
+#include "conf.h"
 #include "syslogd-types.h"
 #include "srUtils.h"
 #include "net.h"
diff --git a/tools/omshell.c b/tools/omshell.c
index 7b815869..f8a68527 100644
--- a/tools/omshell.c
+++ b/tools/omshell.c
@@ -38,7 +38,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
-#include "syslogd.h"
+#include "conf.h"
 #include "syslogd-types.h"
 #include "srUtils.h"
 #include "omshell.h"
diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c
index 830bbc87..499a11dd 100644
--- a/tools/omusrmsg.c
+++ b/tools/omusrmsg.c
@@ -66,7 +66,7 @@
 #include "srUtils.h"
 #include "stringbuf.h"
 #include "syslogd-types.h"
-#include "syslogd.h"
+#include "conf.h"
 #include "omusrmsg.h"
 #include "module-template.h"
 #include "errmsg.h"
diff --git a/tools/syslogd.c b/tools/syslogd.c
index f686a769..6f264e5e 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -132,12 +132,14 @@
 #include "queue.h"
 #include "stream.h"
 #include "conf.h"
-#include "vm.h"
 #include "errmsg.h"
 #include "datetime.h"
 #include "parser.h"
-#include "sysvar.h"
 #include "unicode-helper.h"
+#include "ruleset.h"
+#include "rule.h"
+#include "net.h"
+#include "vm.h"
 
 /* definitions for objects we access */
 DEFobjCurrIf(obj)
@@ -145,10 +147,10 @@ DEFobjCurrIf(glbl)
 DEFobjCurrIf(datetime)
 DEFobjCurrIf(conf)
 DEFobjCurrIf(expr)
-DEFobjCurrIf(vm)
-DEFobjCurrIf(var)
 DEFobjCurrIf(module)
 DEFobjCurrIf(errmsg)
+DEFobjCurrIf(rule)
+DEFobjCurrIf(ruleset)
 DEFobjCurrIf(net) /* TODO: make go away! */
 
 
@@ -220,7 +222,7 @@ static rsRetVal GlobalClassExit(void);
 #endif
 
 #ifndef _PATH_TTY
-#define _PATH_TTY	"/dev/tty"
+#	define _PATH_TTY	"/dev/tty"
 #endif
 
 static uchar	*ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */
@@ -247,8 +249,6 @@ int	repeatinterval[2] = { 30, 60 };	/* # of secs before flush */
 
 #define LIST_DELIMITER	':'		/* delimiter between two hosts */
 
-struct	filed *Files = NULL; /* read-only after init() (but beware of sigusr1!) */
-
 static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */
 
 typedef struct legacyOptsLL_s {
@@ -298,6 +298,7 @@ static queueType_t MainMsgQueType = QUEUETYPE_FIXED_ARRAY;	/* type of the main m
 static uchar *pszMainMsgQFName = NULL;				/* prefix for the main message queue file */
 static int64 iMainMsgQueMaxFileSize = 1024*1024;
 static int iMainMsgQPersistUpdCnt = 0;				/* persist queue info every n updates */
+static int bMainMsgQSyncQeueFiles = 0;				/* sync queue files on every write? */
 static int iMainMsgQtoQShutdown = 0;				/* queue shutdown */ 
 static int iMainMsgQtoActShutdown = 1000;			/* action shutdown (in phase 2) */ 
 static int iMainMsgQtoEnq = 2000;				/* timeout for queue enque */ 
@@ -313,7 +314,8 @@ static int iMainMsgQueueDeqtWinToHr = 25;			/* hour begin of time frame when que
 /* support for simple textual representation of FIOP names
  * rgerhards, 2005-09-27
  */
-static char* getFIOPName(unsigned iFIOP)
+char*
+getFIOPName(unsigned iFIOP)
 {
 	char *pRet;
 	switch(iFIOP) {
@@ -360,6 +362,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	iMainMsgQueMaxFileSize = 1024 * 1024;
 	iMainMsgQueueNumWorkers = 1;
 	iMainMsgQPersistUpdCnt = 0;
+	bMainMsgQSyncQeueFiles = 0;
 	iMainMsgQtoQShutdown = 0;
 	iMainMsgQtoActShutdown = 1000;
 	iMainMsgQtoEnq = 2000;
@@ -395,7 +398,6 @@ static char **crunch_list(char *list);
 static void reapchild();
 static void debug_switch();
 static void sighup_handler();
-static void freeSelectors(void);
 static void processImInternal(void);
 
 
@@ -430,67 +432,6 @@ diagGetMainMsgQSize(int *piSize)
 /* ------------------------------ end support functions for imdiag  ------------------------------ */
 
 
-/* function to destruct a selector_t object
- * rgerhards, 2007-08-01
- */
-rsRetVal
-selectorDestruct(void *pVal)
-{
-	selector_t *pThis = (selector_t *) pVal;
-
-	assert(pThis != NULL);
-
-	if(pThis->pCSHostnameComp != NULL)
-		rsCStrDestruct(&pThis->pCSHostnameComp);
-	if(pThis->pCSProgNameComp != NULL)
-		rsCStrDestruct(&pThis->pCSProgNameComp);
-
-	if(pThis->f_filter_type == FILTER_PROP) {
-		if(pThis->f_filterData.prop.pCSPropName != NULL)
-			rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName);
-		if(pThis->f_filterData.prop.pCSCompValue != NULL)
-			rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue);
-		if(pThis->f_filterData.prop.regex_cache != NULL)
-			rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache);
-	} else if(pThis->f_filter_type == FILTER_EXPR) {
-		if(pThis->f_filterData.f_expr != NULL)
-			expr.Destruct(&pThis->f_filterData.f_expr);
-	}
-
-	llDestroy(&pThis->llActList);
-	free(pThis);
-	
-	return RS_RET_OK;
-}
-
-
-/* function to construct a selector_t object
- * rgerhards, 2007-08-01
- */
-rsRetVal
-selectorConstruct(selector_t **ppThis)
-{
-	DEFiRet;
-	selector_t *pThis;
-
-	assert(ppThis != NULL);
-	
-	if((pThis = (selector_t*) calloc(1, sizeof(selector_t))) == NULL) {
-		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
-	}
-	CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL));
-
-finalize_it:
-	if(iRet != RS_RET_OK) {
-		if(pThis != NULL) {
-			selectorDestruct(pThis);
-		}
-	}
-	*ppThis = pThis;
-	RETiRet;
-}
-
-
 /* rgerhards, 2005-10-24: crunch_list is called only during option processing. So
  * it is never called once rsyslogd is running (not even when HUPed). This code
  * contains some exits, but they are considered safe because they only happen
@@ -562,7 +503,7 @@ static char **crunch_list(char *list)
 void untty(void)
 #ifdef HAVE_SETSID
 {
-	if ( !Debug ) {
+	if(!Debug) {
 		setsid();
 	}
 	return;
@@ -571,18 +512,18 @@ void untty(void)
 {
 	int i;
 
-	if ( !Debug ) {
+	if(!Debug) {
 		i = open(_PATH_TTY, O_RDWR|O_CLOEXEC);
 		if (i >= 0) {
 #			if !defined(__hpux)
-				(void) ioctl(i, (int) TIOCNOTTY, (char *)0);
+				(void) ioctl(i, (int) TIOCNOTTY, NULL);
 #			else
 				/* TODO: we need to implement something for HP UX! -- rgerhards, 2008-03-04 */
 				/* actually, HP UX should have setsid, so the code directly above should
 				 * trigger. So the actual question is why it doesn't do that...
 				 */
 #			endif
-			(void) close(i);
+			close(i);
 		}
 	}
 }
@@ -641,7 +582,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f
 	if(pszInputName != NULL)
 		MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName));
 	MsgSetFlowControlType(pMsg, flowCtlType);
-	MsgSetRawMsg(pMsg, (char*)msg);
+	MsgSetRawMsgWOSize(pMsg, (char*)msg);
 	
 	/* test for special codes */
 	pri = DEFUPRI;
@@ -668,7 +609,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f
 	 * being the local host).  rgerhards 2004-11-16
 	 */
 	if((pMsg->msgFlags & PARSE_HOSTNAME) == 0)
-		MsgSetHOSTNAME(pMsg, hname);
+		MsgSetHOSTNAME(pMsg, hname, ustrlen(hname));
 	MsgSetRcvFrom(pMsg, hname);
 	MsgSetAfterPRIOffs(pMsg, p - msg);
 	CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP));
@@ -941,19 +882,19 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags)
 
 	CHKiRet(msgConstruct(&pMsg));
 	MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1);
-	MsgSetRawMsg(pMsg, (char*)msg);
-	MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName());
+	MsgSetRawMsgWOSize(pMsg, (char*)msg);
+	MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName()));
 	MsgSetRcvFrom(pMsg, glbl.GetLocalHostName());
 	MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1"));
 	/* check if we have an error code associated and, if so,
 	 * adjust the tag. -- rgerhards, 2008-06-27
 	 */
 	if(iErr == NO_ERRCODE) {
-		MsgSetTAG(pMsg, "rsyslogd:");
+		MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd:"), sizeof("rsyslogd:") - 1);
 	} else {
-		snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr);
+		size_t len = snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr);
 		pszTag[32] = '\0'; /* just to make sure... */
-		MsgSetTAG(pMsg, (char*)pszTag);
+		MsgSetTAG(pMsg, pszTag, len);
 	}
 	pMsg->iFacility = LOG_FAC(pri);
 	pMsg->iSeverity = LOG_PRI(pri);
@@ -985,233 +926,6 @@ finalize_it:
 	RETiRet;
 }
 
-/* This functions looks at the given message and checks if it matches the
- * provided filter condition. If so, it returns true, else it returns
- * false. This is a helper to logmsg() and meant to drive the decision
- * process if a message is to be processed or not. As I expect this
- * decision code to grow more complex over time AND logmsg() is already
- * a very lengthy function, I thought a separate function is more appropriate.
- * 2005-09-19 rgerhards
- * 2008-02-25 rgerhards: changed interface, now utilizes iRet, bProcessMsg
- * returns is message should be procesed.
- */
-static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProcessMsg)
-{
-	DEFiRet;
-	unsigned short pbMustBeFreed;
-	char *pszPropVal;
-	int bRet = 0;
-	vm_t *pVM = NULL;
-	var_t *pResult = NULL;
-
-	assert(f != NULL);
-	assert(pMsg != NULL);
-
-	/* we first have a look at the global, BSD-style block filters (for tag
-	 * and host). Only if they match, we evaluate the actual filter.
-	 * rgerhards, 2005-10-18
-	 */
-	if(f->eHostnameCmpMode == HN_NO_COMP) {
-		/* EMPTY BY INTENSION - we check this value first, because
-		 * it is the one most often used, so this saves us time!
-		 */
-	} else if(f->eHostnameCmpMode == HN_COMP_MATCH) {
-		if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) {
-			/* not equal, so we are already done... */
-			DBGPRINTF("hostname filter '+%s' does not match '%s'\n", 
-				rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg));
-			FINALIZE;
-		}
-	} else { /* must be -hostname */
-		if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) {
-			/* not equal, so we are already done... */
-			DBGPRINTF("hostname filter '-%s' does not match '%s'\n", 
-				rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg));
-			FINALIZE;
-		}
-	}
-	
-	if(f->pCSProgNameComp != NULL) {
-		int bInv = 0, bEqv = 0, offset = 0;
-		if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp)) == '-') {
-			if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp) + 1) == '-')
-				offset = 1;
-			else {
-				bInv = 1;
-				offset = 1;
-			}
-		}
-		if(!rsCStrOffsetSzStrCmp(f->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg)))
-			bEqv = 1;
-
-		if((!bEqv && !bInv) || (bEqv && bInv)) {
-			/* not equal or inverted selection, so we are already done... */
-			DBGPRINTF("programname filter '%s' does not match '%s'\n", 
-				rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg));
-			FINALIZE;
-		}
-	}
-	
-	/* done with the BSD-style block filters */
-
-	if(f->f_filter_type == FILTER_PRI) {
-		/* skip messages that are incorrect priority */
-		if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) ||
-		    ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) )
-			bRet = 0;
-		else
-			bRet = 1;
-	} else if(f->f_filter_type == FILTER_EXPR) {
-		CHKiRet(vm.Construct(&pVM));
-		CHKiRet(vm.ConstructFinalize(pVM));
-		CHKiRet(vm.SetMsg(pVM, pMsg));
-		CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg));
-		CHKiRet(vm.PopBoolFromStack(pVM, &pResult));
-		DBGPRINTF("result of expression evaluation: %lld\n", pResult->val.num);
-		/* VM is destructed on function exit */
-		bRet = (pResult->val.num) ? 1 : 0;
-	} else {
-		assert(f->f_filter_type == FILTER_PROP); /* assert() just in case... */
-		pszPropVal = MsgGetProp(pMsg, NULL, f->f_filterData.prop.pCSPropName, &pbMustBeFreed);
-
-		/* Now do the compares (short list currently ;)) */
-		switch(f->f_filterData.prop.operation ) {
-		case FIOP_CONTAINS:
-			if(rsCStrLocateInSzStr(f->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1)
-				bRet = 1;
-			break;
-		case FIOP_ISEQUAL:
-			if(rsCStrSzStrCmp(f->f_filterData.prop.pCSCompValue,
-					  (uchar*) pszPropVal, strlen(pszPropVal)) == 0)
-				bRet = 1; /* process message! */
-			break;
-		case FIOP_STARTSWITH:
-			if(rsCStrSzStrStartsWithCStr(f->f_filterData.prop.pCSCompValue,
-					  (uchar*) pszPropVal, strlen(pszPropVal)) == 0)
-				bRet = 1; /* process message! */
-			break;
-		case FIOP_REGEX:
-			if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue,
-					(unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK)
-				bRet = 1;
-			break;
-		case FIOP_EREREGEX:
-			if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue,
-					  (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK)
-				bRet = 1;
-			break;
-		default:
-			/* here, it handles NOP (for performance reasons) */
-			assert(f->f_filterData.prop.operation == FIOP_NOP);
-			bRet = 1; /* as good as any other default ;) */
-			break;
-		}
-
-		/* now check if the value must be negated */
-		if(f->f_filterData.prop.isNegated)
-			bRet = (bRet == 1) ?  0 : 1;
-
-		if(Debug) {
-			dbgprintf("Filter: check for property '%s' (value '%s') ",
-			        rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName),
-			        pszPropVal);
-			if(f->f_filterData.prop.isNegated)
-				dbgprintf("NOT ");
-			dbgprintf("%s '%s': %s\n",
-			       getFIOPName(f->f_filterData.prop.operation),
-			       rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue),
-			       bRet ? "TRUE" : "FALSE");
-		}
-
-		/* cleanup */
-		if(pbMustBeFreed)
-			free(pszPropVal);
-	}
-
-finalize_it:
-	/* destruct in any case, not just on error, but it makes error handling much easier */
-	if(pVM != NULL)
-		vm.Destruct(&pVM);
-
-	if(pResult != NULL)
-		var.Destruct(&pResult);
-
-	*bProcessMsg = bRet;
-	RETiRet;
-}
-
-
-/* helper to processMsg(), used to call the configured actions. It is
- * executed from within llExecFunc() of the action list.
- * rgerhards, 2007-08-02
- */
-typedef struct processMsgDoActions_s {
-	int bPrevWasSuspended; /* was the previous action suspended? */
-	msg_t *pMsg;
-} processMsgDoActions_t;
-DEFFUNC_llExecFunc(processMsgDoActions)
-{
-	DEFiRet;
-	rsRetVal iRetMod;	/* return value of module - we do not always pass that back */
-	action_t *pAction = (action_t*) pData;
-	processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam;
-
-	assert(pAction != NULL);
-
-	if((pAction->bExecWhenPrevSusp  == 1) && (pDoActData->bPrevWasSuspended == 0)) {
-		DBGPRINTF("not calling action because the previous one is not suspended\n");
-		ABORT_FINALIZE(RS_RET_OK);
-	}
-
-	iRetMod = actionCallAction(pAction, pDoActData->pMsg);
-	if(iRetMod == RS_RET_DISCARDMSG) {
-		ABORT_FINALIZE(RS_RET_DISCARDMSG);
-	} else if(iRetMod == RS_RET_SUSPENDED) {
-		/* indicate suspension for next module to be called */
-		pDoActData->bPrevWasSuspended = 1;
-	} else {
-		pDoActData->bPrevWasSuspended = 0;
-	}
-
-finalize_it:
-	RETiRet;
-}
-
-
-/* Process (consume) a received message. Calls the actions configured.
- * rgerhards, 2005-10-13
- */
-static void
-processMsg(msg_t *pMsg)
-{
-	selector_t *f;
-	int bContinue;
-	int bProcessMsg;
-	processMsgDoActions_t DoActData;
-	rsRetVal iRet;
-
-	BEGINfunc
-	assert(pMsg != NULL);
-
-	/* log the message to the particular outputs */
-
-	bContinue = 1;
-	for (f = Files; f != NULL && bContinue ; f = f->f_next) {
-		/* first check the filters... */
-		iRet = shouldProcessThisMessage(f, pMsg, &bProcessMsg);
-		if(!bProcessMsg) {
-			continue;
-		}
-
-		/* ok -- from here, we have action-specific code, nothing really selector-specific -- rger 2007-08-01 */
-		DoActData.pMsg = pMsg;
-		DoActData.bPrevWasSuspended = 0;
-		if(llExecFunc(&f->llActList, processMsgDoActions, (void*)&DoActData) == RS_RET_DISCARDMSG)
-			bContinue = 0;
-	}
-	ENDfunc
-}
-
 
 /* The consumer of dequeued messages. This function is called by the
  * queue engine on dequeueing of a message. It runs on a SEPARATE
@@ -1229,7 +943,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr)
 	if((pMsg->msgFlags & NEEDS_PARSING) != 0) {
 		parseMsg(pMsg);
 	}
-	processMsg(pMsg);
+	ruleset.ProcessMsg(pMsg);
 	msgDestruct(&pMsg);
 
 	RETiRet;
@@ -1338,7 +1052,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult)
 	return 0;
 }
 
-/* parse a RFC-formatted syslog message. This function returns
+/* parse a RFC5424-formatted syslog message. This function returns
  * 0 if processing of the message shall continue and 1 if something
  * went wrong and this messe should be ignored. This function has been
  * implemented in the effort to support syslog-protocol. Please note that
@@ -1399,12 +1113,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 	/* HOSTNAME */
 	if(bContParse) {
 		parseRFCField(&p2parse, pBuf);
-		MsgSetHOSTNAME(pMsg, pBuf);
-	} else {
-		/* we can not parse, so we get the system we
-		 * received the data from.
-		 */
-		MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
+		MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf));
 	}
 
 	/* APP-NAME */
@@ -1432,7 +1141,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 	}
 
 	/* MSG */
-	MsgSetMSG(pMsg, (char*)p2parse);
+	MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg);
 
 	free(pBuf);
 	ENDfunc
@@ -1456,11 +1165,10 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 {
 	uchar *p2parse;
-	char *pBuf;
-	char *pWork;
-	cstr_t *pStrB;
-	int iCnt;
 	int bTAGCharDetected;
+	int i;	/* general index for parsing */
+	uchar bufParseTAG[CONF_TAG_MAXSIZE];
+	uchar bufParseHOSTNAME[CONF_TAG_HOSTNAME];
 	BEGINfunc
 
 	assert(pMsg != NULL);
@@ -1482,8 +1190,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 		if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse) == RS_RET_OK) {
 			/* indeed, we got it! */
 			/* we are done - parse pointer is moved by ParseTIMESTAMP3164 */;
-		} else {
-			/* parse pointer needs to be restored, as we moved it off-by-one
+		} else {/* parse pointer needs to be restored, as we moved it off-by-one
 			 * for this try.
 			 */
 			--p2parse;
@@ -1513,50 +1220,24 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 		 * If I find them, I set a simple flag but continue. After parsing, I check the flag.
 		 * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change
 		 * the fields. I think this logic shall work with any type of syslog message.
+		 * rgerhards, 2009-06-23: and I now have extended this logic to every character
+		 * that is not a valid hostname.
 		 */
 		bTAGCharDetected = 0;
 		if(flags & PARSE_HOSTNAME) {
-			/* TODO: quick and dirty memory allocation */
-			/* the memory allocated is far too much in most cases. But on the plus side,
-			 * it is quite fast... - rgerhards, 2007-09-20
-			 */
-			if((pBuf = malloc(sizeof(char)* (ustrlen(p2parse) +1))) == NULL)
-				return 1;
-			pWork = pBuf;
-			/* this is the actual parsing loop */
-			while(*p2parse && *p2parse != ' ' && *p2parse != ':') {
-				if(*p2parse == '[' || *p2parse == ']' || *p2parse == '/')
-					bTAGCharDetected = 1;
-				*pWork++ = *p2parse++;
+			i = 0;
+			while((isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.'
+				|| p2parse[i] == '_') && i < CONF_TAG_MAXSIZE) {
+				bufParseHOSTNAME[i] = p2parse[i];
+				++i;
+			}
+
+			if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) {
+				/* we got a hostname! */
+				p2parse += i + 1; /* "eat" it (including SP delimiter) */
+				bufParseHOSTNAME[i] = '\0';
+				MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i);
 			}
-			/* we need to handle ':' seperately, because it terminates the
-			 * TAG - so we also need to terminate the parser here!
-			 * rgerhards, 2007-09-10 *p2parse points to a valid address here in 
-			 * any case. We can reach this point only if we are at end of string,
-			 * or we have a ':' or ' '. What the if below does is check if we are
-			 * not at end of string and, if so, advance the parse pointer. If we 
-			 * are already at end of string, *p2parse is equal to '\0', neither if
-			 * will be true and the parse pointer remain as is. This is perfectly
-			 * well.
-			 */
-			if(*p2parse == ':') {
-				bTAGCharDetected = 1;
-				/* We will move hostname to tag, so preserve ':' (otherwise we 
-				 * will needlessly change the message format) */
-				*pWork++ = *p2parse++; 
-			} else if(*p2parse == ' ')
-				++p2parse;
-			*pWork = '\0';
-			MsgAssignHOSTNAME(pMsg, pBuf);
-		}
-		/* check if we seem to have a TAG */
-		if(bTAGCharDetected) {
-			/* indeed, this smells like a TAG, so lets use it for this. We take
-			 * the HOSTNAME from the sender system instead.
-			 */
-			DBGPRINTF("HOSTNAME contains invalid characters, assuming it to be a TAG.\n");
-			moveHOSTNAMEtoTAG(pMsg);
-			MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 		}
 
 		/* now parse TAG - that should be present in message from all sources.
@@ -1572,68 +1253,37 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 		 * in RFC3164...). We now receive the full size, but will modify the
 		 * outputs so that only 32 characters max are used by default.
 		 */
-		/* The following code in general is quick & dirty - I need to get
-		 * it going for a test, rgerhards 2004-11-16 */
-		/* lol.. we tried to solve it, just to remind ourselfs that 32 octets
-		 * is the max size ;) we need to shuffle the code again... Just for 
-		 * the records: the code is currently clean, but we could optimize it! */
-		if(!bTAGCharDetected) {
-			uchar *pszTAG;
-			if(cstrConstruct(&pStrB) != RS_RET_OK) 
-				return 1;
-			rsCStrSetAllocIncrement(pStrB, 33);
-			pWork = pBuf;
-			iCnt = 0;
-			while(*p2parse && *p2parse != ':' && *p2parse != ' ') {
-				cstrAppendChar(pStrB, *p2parse++);
-				++iCnt;
-			}
-			if(*p2parse == ':') {
-				++p2parse; 
-				cstrAppendChar(pStrB, ':');
-			}
-			cstrFinalize(pStrB);
-			cstrConvSzStrAndDestruct(pStrB, &pszTAG, 1);
-			if(pszTAG == NULL)
-			{	/* rger, 2005-11-10: no TAG found - this implies that what
-				 * we have considered to be the HOSTNAME is most probably the
-				 * TAG. We consider it so probable, that we now adjust it
-				 * that way. So we pick up the previously set hostname, assign
-				 * it to tag and use the sender system (from IP stack) as
-				 * the hostname. This situation is the standard case with
-				 * stock BSD syslogd.
-				 */
-				DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n");
-				moveHOSTNAMEtoTAG(pMsg);
-				MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
-			} else { /* we have a TAG, so we can happily set it ;) */
-				MsgAssignTAG(pMsg, pszTAG);
-			}
-		} else {
-			/* we have no TAG, so we ... */
-			/*DO NOTHING*/;
+		i = 0;
+		while(*p2parse && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) {
+			bufParseTAG[i++] = *p2parse++;
 		}
-	} else {
-		/* we enter this code area when the user has instructed rsyslog NOT
+		if(*p2parse == ':') {
+			++p2parse; 
+			bufParseTAG[i++] = ':';
+		}
+
+		/* no TAG can only be detected if the message immediatly ends, in which case an empty TAG
+		 * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23
+		 */
+		bufParseTAG[i] = '\0';	/* terminate string */
+		MsgSetTAG(pMsg, bufParseTAG, i);
+	} else {/* we enter this code area when the user has instructed rsyslog NOT
 		 * to parse HOSTNAME and TAG - rgerhards, 2006-03-13
 		 */
-		if(!(flags & INTERNAL_MSG))
-		{
+		if(!(flags & INTERNAL_MSG)) {
 			DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n");
-			MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 		}
 	}
 
 	/* The rest is the actual MSG */
-	MsgSetMSG(pMsg, (char*)p2parse);
+	MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg);
 
 	ENDfunc
 	return 0; /* all ok */
 }
 
 
-/* submit a fully created message to the main message queue. The message is
- * fully processed and parsed, so no parsing at all happens. This is primarily
+/* submit a message to the main message queue.   This is primarily
  * a hook to prevent the need for callers to know about the main message queue
  * (which may change in the future as we will probably have multiple rule
  * sets and thus queues...).
@@ -1653,6 +1303,28 @@ submitMsg(msg_t *pMsg)
 }
 
 
+/* submit multiple messages at once, very similar to submitMsg, just
+ * for multi_submit_t.
+ * rgerhards, 2009-06-16
+ */
+rsRetVal
+multiSubmitMsg(multi_submit_t *pMultiSub)
+{
+	int i;
+	DEFiRet;
+	assert(pMultiSub != NULL);
+
+	for(i = 0 ; i < pMultiSub->nElem ; ++i) {
+		MsgPrepareEnqueue(pMultiSub->ppMsgs[i]);
+	}
+
+	iRet = qqueueMultiEnqObj(pMsgQueue, pMultiSub);
+	pMultiSub->nElem = 0;
+
+	RETiRet;
+}
+
+
 /* Log a message to the appropriate log files, users, etc. based on
  * the priority.
  * rgerhards 2004-11-08: actually, this also decodes all but the PRI part.
@@ -1736,7 +1408,6 @@ reapchild()
 DEFFUNC_llExecFunc(flushRptdMsgsActions)
 {
 	action_t *pAction = (action_t*) pData;
-
 	assert(pAction != NULL);
 	
 	BEGINfunc
@@ -1759,20 +1430,12 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions)
 }
 
 
-/* This method flushes reapeat messages.
+/* This method flushes repeat messages.
  */
 static void
 doFlushRptdMsgs(void)
 {
-	register selector_t *f;
-
-	/* see if we need to flush any "message repeated n times"... 
-	 * Note that this interferes with objects running on other threads.
-	 * We are using appropriate locking inside the function to handle that.
-	 */
-	for (f = Files; f != NULL ; f = f->f_next) {
-		llExecFunc(&f->llActList, flushRptdMsgsActions, NULL);
-	}
+	ruleset.IterateAllActions(flushRptdMsgsActions, NULL);
 }
 
 
@@ -1963,6 +1626,16 @@ freeAllDynMemForTermination(void)
 }
 
 
+/* Finalize and destruct all actions.
+ */
+static inline void
+destructAllActions(void)
+{
+	ruleset.DestructAllActions();
+	bHaveMainQueue = 0; // flag that internal messages need to be temporarily stored
+}
+
+
 /* die() is called when the program shall end. This typically only occurs
  * during sigterm or during the initialization. 
  * As die() is intended to shutdown rsyslogd, it is
@@ -2013,7 +1686,7 @@ die(int sig)
 	 * repeated msgs.
 	 */
 	DBGPRINTF("Terminating outputs...\n");
-	freeSelectors();
+	destructAllActions();
 
 	DBGPRINTF("all primary multi-thread sources have been terminated - now doing aux cleanup...\n");
 	/* rger 2005-02-22
@@ -2049,7 +1722,7 @@ die(int sig)
 	 * rgerhards, 2007-08-03
 	 * I have added some code now, but all that mod init/de-init should be moved to
 	 * init, so that modules are unloaded and reloaded on HUP to. Eventually it should go
-	 * into freeSelectors() - but that needs to be seen. -- rgerhards, 2007-08-09
+	 * into destructAllActions() - but that needs to be seen. -- rgerhards, 2007-08-09
 	 */
 	module.UnloadAndDestructAll(eMOD_LINK_ALL);
 
@@ -2177,56 +1850,6 @@ static void doDropPrivUid(int iUid)
 }
 
 
-/* helper to freeSelectors(), used with llExecFunc() to flush 
- * pending output.  -- rgerhards, 2007-08-02
- * We do not need to lock the action object here as the processing
- * queue is already empty and no other threads are running when
- * we call this function. -- rgerhards, 2007-12-12
- */
-DEFFUNC_llExecFunc(freeSelectorsActions)
-{
-	action_t *pAction = (action_t*) pData;
-
-	assert(pAction != NULL);
-
-	/* flush any pending output */
-	if(pAction->f_prevcount) {
-		actionWriteToAction(pAction);
-	}
-
-	return RS_RET_OK; /* never fails ;) */
-}
-
-
-/*  Close all open log files and free selector descriptor array.
- */
-static void freeSelectors(void)
-{
-	selector_t *f;
-	selector_t *fPrev;
-
-	if(Files != NULL) {
-		DBGPRINTF("Freeing log structures.\n");
-
-		for(f = Files ; f != NULL ; f = f->f_next) {
-			llExecFunc(&f->llActList, freeSelectorsActions, NULL);
-		}
-
-		/* actions flushed and ready for destruction - so do that... */
-		f = Files;
-		while (f != NULL) {
-			fPrev = f;
-			f = f->f_next;
-			selectorDestruct(fPrev);
-		}
-
-		/* Reflect the deletion of the selectors linked list. */
-		Files = NULL;
-		bHaveMainQueue = 0;
-	}
-}
-
-
 /* helper to generateConfigDAG, to print out all actions via
  * the llExecFunc() facility.
  * rgerhards, 2007-08-02
@@ -2303,14 +1926,14 @@ DEFFUNC_llExecFunc(generateConfigDAGAction)
 static rsRetVal
 generateConfigDAG(uchar *pszDAGFile)
 {
-	selector_t *f;
+	//rule_t *f;
 	FILE *fp;
 	int iActUnit = 1;
-	int bHasFilter = 0;	/* filter associated with this action unit? */
-	int bHadFilter;
-	int i;
+	//int bHasFilter = 0;	/* filter associated with this action unit? */
+	//int bHadFilter;
+	//int i;
 	struct dag_info dagInfo;
-	char *pszFilterName;
+	//char *pszFilterName;
 	char szConnectingNode[64];
 	DEFiRet;
 
@@ -2339,6 +1962,8 @@ generateConfigDAG(uchar *pszDAGFile)
 	strcpy(szConnectingNode, "act0_0");
 	dagInfo.bDiscarded = 0;
 
+/* TODO: re-enable! */
+#if 0
 	for(f = Files; f != NULL ; f = f->f_next) {
 		/* BSD-Style filters are currently ignored */
 		bHadFilter = bHasFilter;
@@ -2394,6 +2019,7 @@ generateConfigDAG(uchar *pszDAGFile)
 
 		++iActUnit;
 	}
+#endif
 
 	fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit);
 	fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n"
@@ -2405,20 +2031,6 @@ finalize_it:
 }
 
 
-/* helper to dbPrintInitInfo, to print out all actions via
- * the llExecFunc() facility.
- * rgerhards, 2007-08-02
- */
-DEFFUNC_llExecFunc(dbgPrintInitInfoAction)
-{
-	DEFiRet;
-	iRet = actionDbgPrint((action_t*) pData);
-	DBGPRINTF("\n");
-
-	RETiRet;
-}
-
-
 /* print debug information as part of init(). This pretty much
  * outputs the whole config of rsyslogd. I've moved this code
  * out of init() to clean it somewhat up.
@@ -2426,47 +2038,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction)
  */
 static void dbgPrintInitInfo(void)
 {
-	selector_t *f;
-	int iSelNbr = 1;
-	int i;
-
-	DBGPRINTF("\nActive selectors:\n");
-	for (f = Files; f != NULL ; f = f->f_next) {
-		DBGPRINTF("Selector %d:\n", iSelNbr++);
-		if(f->pCSProgNameComp != NULL)
-			DBGPRINTF("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp));
-		if(f->eHostnameCmpMode != HN_NO_COMP)
-			DBGPRINTF("hostname: %s '%s'\n",
-				f->eHostnameCmpMode == HN_COMP_MATCH ?
-					"only" : "allbut",
-				rsCStrGetSzStrNoNULL(f->pCSHostnameComp));
-		if(f->f_filter_type == FILTER_PRI) {
-			for (i = 0; i <= LOG_NFACILITIES; i++)
-				if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) {
-					DBGPRINTF(" X ");
-				} else {
-					DBGPRINTF("%2X ", f->f_filterData.f_pmask[i]);
-				}
-		} else if(f->f_filter_type == FILTER_EXPR) {
-			DBGPRINTF("EXPRESSION-BASED Filter: can currently not be displayed");
-		} else {
-			DBGPRINTF("PROPERTY-BASED Filter:\n");
-			DBGPRINTF("\tProperty.: '%s'\n",
-			       rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName));
-			DBGPRINTF("\tOperation: ");
-			if(f->f_filterData.prop.isNegated)
-				DBGPRINTF("NOT ");
-			DBGPRINTF("'%s'\n", getFIOPName(f->f_filterData.prop.operation));
-			DBGPRINTF("\tValue....: '%s'\n",
-			       rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue));
-			DBGPRINTF("\tAction...: ");
-		}
-
-		DBGPRINTF("\nActions:\n");
-		llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */
-
-		DBGPRINTF("\n");
-	}
+	ruleset.DebugPrintAll();
 	DBGPRINTF("\n");
 	if(bDebugPrintTemplateList)
 		tplPrintList();
@@ -2546,13 +2118,14 @@ startInputModules(void)
 static rsRetVal
 init(void)
 {
-	DEFiRet;
 	rsRetVal localRet;
 	int iNbrActions;
 	int bHadConfigErr = 0;
+	ruleset_t *pRuleset;
 	char cbuf[BUFSIZ];
 	char bufStartUpMsg[512];
 	struct sigaction sigAct;
+	DEFiRet;
 
 	thrdTerminateAll(); /* stop all running input threads - TODO: reconsider location! */
 
@@ -2573,7 +2146,7 @@ init(void)
 	/*  Close all open log files and free log descriptor array. This also frees
 	 *  all output-modules instance data.
 	 */
-	freeSelectors();
+	destructAllActions();
 
 	/* Unload all non-static modules */
 	DBGPRINTF("Unloading non-static modules.\n");
@@ -2594,6 +2167,11 @@ init(void)
 
 	conf.ReInitConf();
 
+	/* construct the default ruleset */
+	ruleset.Construct(&pRuleset);
+	ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset"));
+	ruleset.ConstructFinalize(pRuleset);
+
 	/* open the configuration file */
 	localRet = conf.processConfFile(ConfFile);
 	CHKiRet(conf.GetNbrActActions(&iNbrActions));
@@ -2615,23 +2193,23 @@ init(void)
 		 * We ignore any errors while doing this - we would be lost anyhow...
 		 */
 		errmsg.LogError(0, NO_ERRCODE, "EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file!");
-		selector_t *f = NULL;
 
 		/* note: we previously used _POSIY_TTY_NAME_MAX+1, but this turned out to be
 		 * too low on linux... :-S   -- rgerhards, 2008-07-28
 		 */
 		char szTTYNameBuf[128];
-		conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &f);
-		conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &f);
-		conf.cfline((uchar*)"*.PANIC\t*", &f);
-		conf.cfline((uchar*)"syslog.*\troot", &f);
+		rule_t *pRule = NULL; /* initialization to NULL is *vitally* important! */
+		conf.cfline(UCHAR_CONSTANT("*.ERR\t" _PATH_CONSOLE), &pRule);
+		conf.cfline(UCHAR_CONSTANT("syslog.*\t" _PATH_CONSOLE), &pRule);
+		conf.cfline(UCHAR_CONSTANT("*.PANIC\t*"), &pRule);
+		conf.cfline(UCHAR_CONSTANT("syslog.*\troot"), &pRule);
 		if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) {
 			snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf);
-			conf.cfline((uchar*)cbuf, &f);
+			conf.cfline((uchar*)cbuf, &pRule);
 		} else {
 			DBGPRINTF("error %d obtaining controlling terminal, not using that emergency rule\n", errno);
 		}
-		selectorAddList(f);
+		ruleset.AddRule(ruleset.GetCurrent(), &pRule);
 	}
 
 	legacyOptsHook();
@@ -2710,6 +2288,7 @@ init(void)
 	setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace);
 	setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName);
 	setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt);
+	setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles);
 	setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown );
 	setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown);
 	setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown);
@@ -2769,49 +2348,48 @@ finalize_it:
 }
 
 
-/* add a completely-processed selector (after config line parsing) to
- * the linked list of selectors. We now need to check
- * if it has any actions associated and, if so, link it to the linked
- * list. If it has nothing associated with it, we can simply discard
- * it.
- * We have one special case during initialization: then, the current
- * selector is NULL, which means we do not need to care about it at
- * all.  -- rgerhards, 2007-08-01
+/* Switch the default ruleset (that, what servcies bind to if nothing specific
+ * is specified).
+ * rgerhards, 2009-06-12
  */
-rsRetVal
-selectorAddList(selector_t *f)
+static rsRetVal
+setDefaultRuleset(void __attribute__((unused)) *pVal, uchar *pszName)
 {
 	DEFiRet;
-	int iActionCnt;
 
-	static selector_t *nextp = NULL; /* TODO: make this go away (see comment below) */
+	CHKiRet(ruleset.SetDefaultRuleset(pszName));
 
-	if(f != NULL) {
-		CHKiRet(llGetNumElts(&f->llActList, &iActionCnt));
-		if(iActionCnt == 0) {
-			errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded");
-			selectorDestruct(f);
-		} else {
-			/* successfully created an entry */
-			DBGPRINTF("selector line successfully processed\n");
-			/* TODO: we should use the linked list class for the selector list, else we need to add globals
-			 * ... well nextp could be added temporarily...
-			 * Thanks to varmojfekoj for having the idea to just use "Files" to make this
-			 * code work. I had actually forgotten to fix the code here before moving to 1.18.0.
-			 * And, of course, I also did not migrate the selector_t structure to the linked list class.
-			 * However, that should still be one of the very next things to happen.
-			 * rgerhards, 2007-08-06
-			 */
-			if(Files == NULL) {
-				Files = f;
-			} else {
-				nextp->f_next = f;
-			}
-			nextp = f;
-		}
+finalize_it:
+	free(pszName); /* no longer needed */
+	RETiRet;
+}
+
+
+/* Switch to either an already existing rule set or start a new one. The
+ * named rule set becomes the new "current" rule set (what means that new
+ * actions are added to it).
+ * rgerhards, 2009-06-12
+ */
+static rsRetVal
+setCurrRuleset(void __attribute__((unused)) *pVal, uchar *pszName)
+{
+	ruleset_t *pRuleset;
+	rsRetVal localRet;
+	DEFiRet;
+
+	localRet = ruleset.SetCurrRuleset(pszName);
+
+	if(localRet == RS_RET_NOT_FOUND) {
+		DBGPRINTF("begin new current rule set '%s'\n", pszName);
+		CHKiRet(ruleset.Construct(&pRuleset));
+		CHKiRet(ruleset.SetName(pRuleset, pszName));
+		CHKiRet(ruleset.ConstructFinalize(pRuleset));
+	} else {
+		ABORT_FINALIZE(localRet);
 	}
 
 finalize_it:
+	free(pszName); /* no longer needed */
 	RETiRet;
 }
 
@@ -2903,7 +2481,6 @@ DEFFUNC_llExecFunc(doHUPActions)
 static inline void
 doHUP(void)
 {
-	selector_t *f;
 	char buf[512];
 
 	snprintf(buf, sizeof(buf) / sizeof(char),
@@ -2918,9 +2495,7 @@ doHUP(void)
 		init(); /* main queue is stopped as part of init() */
 	} else {
 		DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n");
-		for(f = Files; f != NULL ; f = f->f_next) {
-			llExecFunc(&f->llActList, doHUPActions, NULL);
-		}
+		ruleset.IterateAllActions(doHUPActions, NULL);
 	}
 }
 
@@ -2951,7 +2526,8 @@ mainloop(void)
 		 * powertop, for example). In that case, we primarily wait for a signal,
 		 * but a once-a-day wakeup should be quite acceptable. -- rgerhards, 2008-06-09
 		 */
-		tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/;
+		//tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/;
+		tvSelectTimeout.tv_sec = TIMERINTVL; /* TODO: change this back to the above code when we have a better solution for apc */
 		tvSelectTimeout.tv_usec = 0;
 		select(1, NULL, NULL, NULL, &tvSelectTimeout);
 		if(bFinished)
@@ -2986,49 +2562,11 @@ mainloop(void)
 			bHadHUP = 0;
 			continue;
 		}
+		execScheduled(); /* handle Apc calls (if any) */
 	}
 	ENDfunc
 }
 
-/* If user is not root, prints warnings or even exits 
- * TODO: check all dynafiles for write permission
- * ... but it is probably better to wait here until we have
- * a module interface - rgerhards, 2007-07-23
- */
-static void checkPermissions()
-{
-#if 0
-	/* TODO: this function must either be redone or removed - now with the input modules,
-	 * there is no such simple check we can do. What we can check, however, is if there is
-	 * any input module active and terminate, if not. -- rgerhards, 2007-12-26
-	 */
-	/* we are not root */
-	if (geteuid() != 0)
-	{
-		fputs("WARNING: Local messages will not be logged! If you want to log them, run rsyslog as root.\n",stderr); 
-#ifdef SYSLOG_INET	
-		/* udp enabled and port number less than or equal to 1024 */
-		if ( AcceptRemote && (atoi(LogPort) <= 1024) )
-			fprintf(stderr, "WARNING: Will not listen on UDP port %s. Use port number higher than 1024 or run rsyslog as root!\n", LogPort);
-		
-		/* tcp enabled and port number less or equal to 1024 */
-		if( bEnableTCP   && (atoi(TCPLstnPort) <= 1024) )
-			fprintf(stderr, "WARNING: Will not listen on TCP port %s. Use port number higher than 1024 or run rsyslog as root!\n", TCPLstnPort);
-
-		/* Neither explicit high UDP port nor explicit high TCP port.
-                 * It is useless to run anymore */
-		if( !(AcceptRemote && (atoi(LogPort) > 1024)) && !( bEnableTCP && (atoi(TCPLstnPort) > 1024)) )
-		{
-#endif
-			fprintf(stderr, "ERROR: Nothing to log, no reason to run. Please run rsyslog as root.\n");
-			exit(EXIT_FAILURE);
-#ifdef SYSLOG_INET
-		}
-#endif
-	}
-#endif
-}
-
 
 /* load build-in modules
  * very first version begun on 2007-07-23 by rgerhards
@@ -3037,23 +2575,23 @@ static rsRetVal loadBuildInModules(void)
 {
 	DEFiRet;
 
-	if((iRet = module.doModInit(modInitFile, (uchar*) "builtin-file", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitFile, UCHAR_CONSTANT("builtin-file"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
 #ifdef SYSLOG_INET
-	if((iRet = module.doModInit(modInitFwd, (uchar*) "builtin-fwd", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitFwd, UCHAR_CONSTANT("builtin-fwd"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
 #endif
-	if((iRet = module.doModInit(modInitShell, (uchar*) "builtin-shell", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitShell, UCHAR_CONSTANT("builtin-shell"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
-	if((iRet = module.doModInit(modInitDiscard, (uchar*) "builtin-discard", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitDiscard, UCHAR_CONSTANT("builtin-discard"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
 
 	/* dirty, but this must be for the time being: the usrmsg module must always be
-	 * loaded as last module. This is because it processes any time of action selector.
+	 * loaded as last module. This is because it processes any type of action selector.
 	 * If we load it before other modules, these others will never have a chance of
 	 * working with the config file. We may change that implementation so that a user name
 	 * must start with an alnum, that would definitely help (but would it break backwards
@@ -3061,8 +2599,7 @@ static rsRetVal loadBuildInModules(void)
 	 * User names now must begin with:
 	 *   [a-zA-Z0-9_.]
 	 */
-	if((iRet = module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)) != RS_RET_OK)
-		RETiRet;
+	CHKiRet(module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL));
 
 	/* ok, initialization of the command handler probably does not 100% belong right in
 	 * this space here. However, with the current design, this is actually quite a good
@@ -3072,6 +2609,8 @@ static rsRetVal loadBuildInModules(void)
 	 * This, I think, is the right thing to do. -- rgerhards, 2007-07-31
 	 */
 	CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"defaultruleset", 0, eCmdHdlrGetWord, setDefaultRuleset, NULL, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, setCurrRuleset, NULL, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL));
@@ -3079,6 +2618,7 @@ static rsRetVal loadBuildInModules(void)
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardmark", 0, eCmdHdlrInt, NULL, &iMainMsgQDiscardMark, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardseverity", 0, eCmdHdlrSeverity, NULL, &iMainMsgQDiscardSeverity, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iMainMsgQPersistUpdCnt, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bMainMsgQSyncQeueFiles, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetype", 0, eCmdHdlrGetWord, setMainMsgQueType, NULL, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iMainMsgQueueNumWorkers, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iMainMsgQtoQShutdown, NULL));
@@ -3276,14 +2816,14 @@ InitGlobalClasses(void)
 	CHKiRet(objUse(errmsg,   CORE_COMPONENT));
 	pErrObj = "module";
 	CHKiRet(objUse(module,   CORE_COMPONENT));
-	pErrObj = "var";
-	CHKiRet(objUse(var,      CORE_COMPONENT));
 	pErrObj = "datetime";
 	CHKiRet(objUse(datetime, CORE_COMPONENT));
-	pErrObj = "vm";
-	CHKiRet(objUse(vm,       CORE_COMPONENT));
 	pErrObj = "expr";
 	CHKiRet(objUse(expr,     CORE_COMPONENT));
+	pErrObj = "rule";
+	CHKiRet(objUse(rule,     CORE_COMPONENT));
+	pErrObj = "ruleset";
+	CHKiRet(objUse(ruleset,  CORE_COMPONENT));
 	pErrObj = "conf";
 	CHKiRet(objUse(conf,     CORE_COMPONENT));
 
@@ -3327,33 +2867,13 @@ GlobalClassExit(void)
 	/* first, release everything we used ourself */
 	objRelease(net,      LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */
 	objRelease(conf,     CORE_COMPONENT);
+	objRelease(ruleset,  CORE_COMPONENT);
+	objRelease(rule,     CORE_COMPONENT);
 	objRelease(expr,     CORE_COMPONENT);
 	vmClassExit();					/* this is hack, currently core_modules do not get this automatically called */
-	objRelease(vm,       CORE_COMPONENT);
-	objRelease(var,      CORE_COMPONENT);
 	objRelease(datetime, CORE_COMPONENT);
 
 	/* TODO: implement the rest of the deinit */
-#if 0
-	CHKiRet(datetimeClassInit(NULL));
-	CHKiRet(msgClassInit(NULL));
-	CHKiRet(strmClassInit(NULL));
-	CHKiRet(wtiClassInit(NULL));
-	CHKiRet(wtpClassInit(NULL));
-	CHKiRet(qqueueClassInit(NULL));
-	CHKiRet(vmstkClassInit(NULL));
-	CHKiRet(sysvarClassInit(NULL));
-	CHKiRet(vmClassInit(NULL));
-	CHKiRet(vmopClassInit(NULL));
-	CHKiRet(vmprgClassInit(NULL));
-	CHKiRet(ctok_tokenClassInit(NULL));
-	CHKiRet(ctokClassInit(NULL));
-	CHKiRet(exprClassInit(NULL));
-
-	/* dummy "classes" */
-	CHKiRet(actionClassInit());
-	CHKiRet(templateInit());
-#endif
 	/* dummy "classes */
 	strExit();
 
@@ -3447,7 +2967,6 @@ doGlblProcessInit(void)
 	int i;
 	DEFiRet;
 
-	checkPermissions();
 	thrdInit();
 
 	if( !(Debug || NoFork) )
diff --git a/tools/syslogd.h b/tools/syslogd.h
index 8b9bd131..3dfdbe2b 100644
--- a/tools/syslogd.h
+++ b/tools/syslogd.h
@@ -29,66 +29,6 @@
 #include "linkedlist.h"
 #include "expr.h"
 
-
-#ifndef _PATH_CONSOLE
-#define _PATH_CONSOLE	"/dev/console"
-#endif
-
-
-/* This structure represents the files that will have log
- * copies printed.
- * RGerhards 2004-11-08: Each instance of the filed structure 
- * describes what I call an "output channel". This is important
- * to mention as we now allow database connections to be
- * present in the filed structure. If helps immensely, if we
- * think of it as the abstraction of an output channel.
- * rgerhards, 2005-10-26: The structure below provides ample
- * opportunity for non-thread-safety. Each of the variable
- * accesses must be carefully evaluated, many of them probably
- * be guarded by mutexes. But beware of deadlocks...
- * rgerhards, 2007-08-01: as you can see, the structure has shrunk pretty much. I will
- * remove some of the comments some time. It's still the structure that controls much
- * of the processing that goes on in syslogd, but it now has lots of helpers.
- */
-struct filed {
-	struct	filed *f_next;		/* next in linked list */
-	/* filter properties */
-	enum {
-		FILTER_PRI = 0,		/* traditional PRI based filer */
-		FILTER_PROP = 1,	/* extended filter, property based */
-		FILTER_EXPR = 2		/* extended filter, expression based */
-	} f_filter_type;
-	EHostnameCmpMode eHostnameCmpMode;
-	cstr_t *pCSHostnameComp;	/* hostname to check */
-	cstr_t *pCSProgNameComp;	/* tag to check or NULL, if not to be checked */
-	union {
-		u_char	f_pmask[LOG_NFACILITIES+1];	/* priority mask */
-		struct {
-			cstr_t *pCSPropName;
-			enum {
-				FIOP_NOP = 0,		/* do not use - No Operation */
-				FIOP_CONTAINS  = 1,	/* contains string? */
-				FIOP_ISEQUAL  = 2,	/* is (exactly) equal? */
-				FIOP_STARTSWITH = 3,	/* starts with a string? */
- 				FIOP_REGEX = 4,		/* matches a (BRE) regular expression? */
- 				FIOP_EREREGEX = 5	/* matches a ERE regular expression? */
-			} operation;
-			regex_t *regex_cache;		/* cache for compiled REs, if such are used */
-			cstr_t *pCSCompValue;	/* value to "compare" against */
-			char isNegated;			/* actually a boolean ;) */
-		} prop;
-		expr_t *f_expr;				/* expression object */
-	} f_filterData;
-
-	linkedList_t llActList;	/* list of configured actions */
-};
-
-
-#include "net.h" /* TODO: remove when you remoe isAllowedSender from here! */
-void untty(void);
-rsRetVal selectorConstruct(selector_t **ppThis);
-rsRetVal selectorDestruct(void *pVal);
-rsRetVal selectorAddList(selector_t *f);
 /* the following prototypes should go away once we have an input
  * module interface -- rgerhards, 2007-12-12
  */
diff --git a/tools/zpipe.c b/tools/zpipe.c
new file mode 100644
index 00000000..bde6c5c1
--- /dev/null
+++ b/tools/zpipe.c
@@ -0,0 +1,254 @@
+/* zpipe.c: example of proper use of zlib's inflate() and deflate()
+   Not copyrighted -- provided to the public domain
+   Version 1.5  11 December 2005  Mark Adler 
+   Version 2.0  03 June     2009  Rainer Gerhards */
+
+/* RSYSLOG NOTE:
+ * This file is beeing distributed as part of rsyslog, but is just an
+ * add-on. Most importantly, rsyslog's copyright does not apply but
+ * rather the (non-) copyright stated above.
+ */
+
+/* Version history:
+   1.0  30 Oct 2004  First version
+   1.1   8 Nov 2004  Add void casting for unused return values
+                     Use switch statement for inflate() return values
+   1.2   9 Nov 2004  Add assertions to document zlib guarantees
+   1.3   6 Apr 2005  Remove incorrect assertion in inf()
+   1.4  11 Dec 2005  Add hack to avoid MSDOS end-of-line conversions
+                     Avoid some compiler warnings for input and output buffers
+   2.0  03 Jun 2009  Add hack to support multiple deflate records inside a single
+   		     file on inflate. This is needed in order to support reading
+		     files created by rsyslog's zip output writer.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include "zlib.h"
+
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>
+#  include <io.h>
+#  define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+#else
+#  define SET_BINARY_MODE(file)
+#endif
+
+#define CHUNK 16384
+
+/* Compress from file source to file dest until EOF on source.
+   def() returns Z_OK on success, Z_MEM_ERROR if memory could not be
+   allocated for processing, Z_STREAM_ERROR if an invalid compression
+   level is supplied, Z_VERSION_ERROR if the version of zlib.h and the
+   version of the library linked do not match, or Z_ERRNO if there is
+   an error reading or writing the files. */
+int def(FILE *source, FILE *dest, int level)
+{
+    int ret, flush;
+    unsigned have;
+    z_stream strm;
+    unsigned char in[CHUNK];
+    unsigned char out[CHUNK];
+
+    /* allocate deflate state */
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    ret = deflateInit(&strm, level);
+    if (ret != Z_OK)
+        return ret;
+
+    /* compress until end of file */
+    do {
+        strm.avail_in = fread(in, 1, CHUNK, source);
+        if (ferror(source)) {
+            (void)deflateEnd(&strm);
+            return Z_ERRNO;
+        }
+        flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
+        strm.next_in = in;
+
+        /* run deflate() on input until output buffer not full, finish
+           compression if all of source has been read in */
+        do {
+            strm.avail_out = CHUNK;
+            strm.next_out = out;
+            ret = deflate(&strm, flush);    /* no bad return value */
+            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
+            have = CHUNK - strm.avail_out;
+            if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+                (void)deflateEnd(&strm);
+                return Z_ERRNO;
+            }
+        } while (strm.avail_out == 0);
+        assert(strm.avail_in == 0);     /* all input will be used */
+
+        /* done when last data in file processed */
+    } while (flush != Z_FINISH);
+    assert(ret == Z_STREAM_END);        /* stream will be complete */
+
+    /* clean up and return */
+    (void)deflateEnd(&strm);
+    return Z_OK;
+}
+
+
+/* initialize stream for deflating (we need this in case of 
+ * multiple records.
+ * rgerhards, 2009-06-03
+ */
+int doInflateInit(z_stream *strm)
+{
+    int ret;
+
+    /* allocate inflate state */
+    strm->zalloc = Z_NULL;
+    strm->zfree = Z_NULL;
+    strm->opaque = Z_NULL;
+    strm->avail_in = 0;
+    strm->next_in = Z_NULL;
+    ret = inflateInit(strm);
+    return ret;
+}
+
+
+/* Decompress from file source to file dest until stream ends or EOF.
+   inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be
+   allocated for processing, Z_DATA_ERROR if the deflate data is
+   invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and
+   the version of the library linked do not match, or Z_ERRNO if there
+   is an error reading or writing the files. */
+int inf(FILE *source, FILE *dest)
+{
+    int ret;
+    unsigned have;
+    z_stream strm;
+    unsigned char in[CHUNK];
+    int len;
+    unsigned char *next_in_save;
+    unsigned char out[CHUNK];
+
+    ret = doInflateInit(&strm);
+    if (ret != Z_OK)
+        return ret;
+
+    /* decompress until deflate stream ends or end of file */
+    do {
+        len = fread(in, 1, CHUNK, source);
+        if (ferror(source)) {
+            (void)inflateEnd(&strm);
+            return Z_ERRNO;
+        }
+        if (len == 0) {
+            break;
+	}
+        strm.avail_in = len;
+        strm.next_in = in;
+
+        /* run inflate() on input until output buffer not full */
+        strm.avail_out = CHUNK;
+        strm.next_out = out;
+        do {
+	/*	fprintf(stderr, "---inner LOOP---, avail_in %d, avail_out %d Byte 0: %x, 1: %x\n", strm.avail_in, strm.avail_out, *strm.next_in, *(strm.next_in+1));*/
+		do {
+		    ret = inflate(&strm, Z_NO_FLUSH);
+		    assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
+		    switch (ret) {
+		    case Z_NEED_DICT:
+			ret = Z_DATA_ERROR;     /* and fall through */
+		    case Z_DATA_ERROR:
+		    case Z_MEM_ERROR:
+			(void)inflateEnd(&strm);
+			return ret;
+		    }
+		    have = CHUNK - strm.avail_out;
+		    if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+			(void)inflateEnd(&strm);
+			return Z_ERRNO;
+		    }
+		} while (strm.avail_out == 0);
+		/* handle the case that more than one deflate record is contained
+		 * in a single file. -- rgerhards, 2009-06-03
+		 */
+		if(ret == Z_STREAM_END) {
+			len -= strm.total_in;
+			if(len > 0) {
+				next_in_save = strm.next_in;
+				(void)inflateEnd(&strm);
+				ret = doInflateInit(&strm);
+				if (ret != Z_OK)
+				    return ret;
+				strm.avail_in = len;
+				strm.next_in = next_in_save;
+				strm.avail_out = CHUNK;
+				strm.next_out = out;
+				ret = Z_OK; /* continue outer loop */
+			}
+		}
+	} while (strm.avail_in > 0);
+
+        /* done when inflate() says it's done */
+    } while (ret != Z_STREAM_END);
+
+    /* clean up and return */
+    (void)inflateEnd(&strm);
+    return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+}
+
+/* report a zlib or i/o error */
+void zerr(int ret)
+{
+    fputs("zpipe: ", stdout);
+    switch (ret) {
+    case Z_ERRNO:
+        if (ferror(stdin))
+            fputs("error reading stdin\n", stdout);
+        if (ferror(stdout))
+            fputs("error writing stdout\n", stdout);
+        break;
+    case Z_STREAM_ERROR:
+        fputs("invalid compression level\n", stdout);
+        break;
+    case Z_DATA_ERROR:
+        fputs("invalid or incomplete deflate data\n", stdout);
+        break;
+    case Z_MEM_ERROR:
+        fputs("out of memory\n", stdout);
+        break;
+    case Z_VERSION_ERROR:
+        fputs("zlib version mismatch!\n", stdout);
+    }
+}
+
+/* compress or decompress from stdin to stdout */
+int main(int argc, char **argv)
+{
+    int ret;
+
+    /* avoid end-of-line conversions */
+    SET_BINARY_MODE(stdin);
+    SET_BINARY_MODE(stdout);
+
+    /* do compression if no arguments */
+    if (argc == 1) {
+        ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION);
+        if (ret != Z_OK)
+            zerr(ret);
+        return ret;
+    }
+
+    /* do decompression if -d specified */
+    else if (argc == 2 && strcmp(argv[1], "-d") == 0) {
+        ret = inf(stdin, stdout);
+        if (ret != Z_OK)
+            zerr(ret);
+        return ret;
+    }
+
+    /* otherwise, report usage */
+    else {
+        fputs("zpipe usage: zpipe [-d] < source > dest\n", stdout);
+        return 1;
+    }
+}