Merge branch 'omfile' into v5-devel

Note that this was NOT a trivial merge, and there may be
some issues. This needs to be seen when we continue developing.

Conflicts:
	runtime/msg.h
	runtime/obj.h
	runtime/queue.c
	runtime/srUtils.h
	runtime/stream.c
	runtime/stream.h
	runtime/wti.c
	tests/Makefile.am
	tools/omfile.c
	tools/syslogd.c

8 files changed, 646 insertions, 788 deletions

diff --git a/tools/Makefile.am b/tools/Makefile.am
index e523b854..f0f9afab 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -26,8 +26,10 @@ rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS)
 rsyslogd_LDFLAGS = -export-dynamic
 
 if ENABLE_DIAGTOOLS
-sbin_PROGRAMS += rsyslog_diag_hostname msggen
+sbin_PROGRAMS += rsyslog_diag_hostname msggen zpipe
 rsyslog_diag_hostname_SOURCES = gethostn.c
+zpipe_SOURCES = zpipe.c
+zpipe_LDADD = -lz
 msggen_SOURCES = msggen.c
 endif
 
diff --git a/tools/omfile.c b/tools/omfile.c
index 3e845a73..675d313e 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -12,7 +12,13 @@
  * of the "old" message code without any modifications. However, it
  * helps to have things at the right place one we go to the meat of it.
  *
- * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH.
+ * A large re-write of this file was done in June, 2009. The focus was
+ * to introduce many more features (like zipped writing), clean up the code
+ * and make it more reliable. In short, that rewrite tries to provide a new
+ * solid basis for the next three to five years to come. During it, bugs
+ * may have been introduced ;) -- rgerhards, 2009-06-04
+ *
+ * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -41,6 +47,7 @@
 #include <assert.h>
 #include <errno.h>
 #include <ctype.h>
+#include <libgen.h>
 #include <unistd.h>
 #include <sys/file.h>
 
@@ -48,7 +55,7 @@
 #	include <fcntl.h>
 #endif
 
-#include "syslogd.h"
+#include "conf.h"
 #include "syslogd-types.h"
 #include "srUtils.h"
 #include "template.h"
@@ -57,6 +64,9 @@
 #include "cfsysline.h"
 #include "module-template.h"
 #include "errmsg.h"
+#include "stream.h"
+#include "zlibw.h"
+#include "unicode-helper.h"
 
 MODULE_TYPE_OUTPUT
 
@@ -64,17 +74,22 @@ MODULE_TYPE_OUTPUT
  */
 DEF_OMOD_STATIC_DATA
 DEFobjCurrIf(errmsg)
+DEFobjCurrIf(zlibw)
+DEFobjCurrIf(strm)
 
 /* The following structure is a dynafile name cache entry.
  */
 struct s_dynaFileCacheEntry {
-	uchar *pName;	/* name currently open, if dynamic name */
-	short	fd;		/* name associated with file name in cache */
-	time_t	lastUsed;	/* for LRU - last access */
+	uchar *pName;		/* name currently open, if dynamic name */
+	strm_t	*pStrm;		/* our output stream */
+	time_t	lastUsed;	/* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) 
 };
 typedef struct s_dynaFileCacheEntry dynaFileCacheEntry;
 
 
+#define IOBUF_DFLT_SIZE 1024	/* default size for io buffers */
+#define FLUSH_INTRVL_DFLT 1 	/* default buffer flush interval (in seconds) */
+
 /* globals for default values */
 static int iDynaFileCacheSize = 10; /* max cache for dynamic files */
 static int fCreateMode = 0644; /* mode to use when creating files */
@@ -86,19 +101,17 @@ static uid_t	dirUID;		/* UID to be used for newly created directories */
 static uid_t	dirGID;		/* GID to be used for newly created directories */
 static int	bCreateDirs;	/* auto-create directories for dynaFiles: 0 - no, 1 - yes */
 static int	bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */
+static int	iZipLevel = 0;	/* zip compression mode (0..9 as usual) */
+static bool	bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */
+static int	iIOBufSize = IOBUF_DFLT_SIZE;	/* size of an io buffer */
+static int	iFlushInterval = FLUSH_INTRVL_DFLT; 	/* how often flush the output buffer on inactivity? */
 static uchar	*pszTplName = NULL; /* name of the default template to use */
 /* end globals for default values */
 
 
 typedef struct _instanceData {
 	uchar	f_fname[MAXFNAME];/* file or template name (display only) */
-	short	fd;		  /* file descriptor for (current) file */
-	enum {
-		eTypeFILE,
-		eTypeTTY,
-		eTypeCONSOLE,
-		eTypePIPE
-	} fileType;	
+	strm_t	*pStrm;		/* our output stream */
 	char	bDynamicName;	/* 0 - static name, 1 - dynamic name (with properties) */
 	int	fCreateMode;	/* file creation mode for open() */
 	int	fDirCreateMode;	/* creation mode for mkdir() */
@@ -117,8 +130,12 @@ typedef struct _instanceData {
 	 * pointer points to the overall structure.
 	 */
 	dynaFileCacheEntry **dynCache;
-	off_t	f_sizeLimit;		/* file size limit, 0 = no limit */
-	char	*f_sizeLimitCmd;	/* command to carry out when size limit is reached */
+	off_t	iSizeLimit;		/* file size limit, 0 = no limit */
+	uchar	*pszSizeLimitCmd;	/* command to carry out when size limit is reached */
+	int 	iZipLevel;		/* zip mode to use for this selector */
+	int	iIOBufSize;		/* size of associated io buffer */
+	int	iFlushInterval;		/* how fast flush buffer on inactivity? */
+	bool	bFlushOnTXEnd;		/* flush write buffers when transaction has ended? */
 } instanceData;
 
 
@@ -147,7 +164,7 @@ CODESTARTdbgPrintInstInfo
 			);
 	} else { /* regular file */
 		dbgprintf("%s", pData->f_fname);
-		if (pData->fd == -1)
+		if (pData->pStrm == NULL)
 			dbgprintf(" (unused)");
 	}
 ENDdbgPrintInstInfo
@@ -168,13 +185,13 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal)
 		errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg);
 		iRet = RS_RET_VAL_OUT_OF_RANGE;
 		iNewVal = 1;
-	} else if(iNewVal > 10000) {
+	} else if(iNewVal > 1000) {
 		snprintf((char*) errMsg, sizeof(errMsg)/sizeof(uchar),
-		         "DynaFileCacheSize maximum is 10,000 (%d given), changed to 10,000.", iNewVal);
+		         "DynaFileCacheSize maximum is 1,000 (%d given), changed to 1,000.", iNewVal);
 		errno = 0;
 		errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg);
 		iRet = RS_RET_VAL_OUT_OF_RANGE;
-		iNewVal = 10000;
+		iNewVal = 1000;
 	}
 
 	iDynaFileCacheSize = iNewVal;
@@ -199,14 +216,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR
 	struct outchannel *pOch;
 	char szBuf[128];	/* should be more than sufficient */
 
-	/* this must always be a file, because we can not set a size limit
-	 * on a pipe...
-	 * rgerhards 2005-06-21: later, this will be a separate type, but let's
-	 * emulate things for the time being. When everything runs, we can
-	 * extend it...
-	 */
-	pData->fileType = eTypeFILE;
-
 	++p; /* skip '$' */
 	i = 0;
 	/* get outchannel name */
@@ -241,12 +250,12 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR
 	}
 
 	/* OK, we finally got a correct template. So let's use it... */
-	strncpy((char*) pData->f_fname, (char*) pOch->pszFileTemplate, MAXFNAME);
-	pData->f_sizeLimit = pOch->uSizeLimit;
+	ustrncpy(pData->f_fname, pOch->pszFileTemplate, MAXFNAME);
+	pData->iSizeLimit = pOch->uSizeLimit;
 	/* WARNING: It is dangerous "just" to pass the pointer. As we
 	 * never rebuild the output channel description, this is acceptable here.
 	 */
-	pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit;
+	pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit;
 
 	iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts,
 				       (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName);
@@ -256,64 +265,6 @@ finalize_it:
 }
 
 
-/* rgerhards 2005-06-21: Try to resolve a size limit
- * situation. This first runs the command, and then
- * checks if we are still above the treshold.
- * returns 0 if ok, 1 otherwise
- * TODO: consider moving the initial check in here, too
- */
-int resolveFileSizeLimit(instanceData *pData)
-{
-	uchar *pParams;
-	uchar *pCmd;
-	uchar *p;
-	off_t actualFileSize;
-	ASSERT(pData != NULL);
-
-	if(pData->f_sizeLimitCmd == NULL)
-		return 1; /* nothing we can do in this case... */
-	
-	/* the execProg() below is probably not great, but at least is is
-	 * fairly secure now. Once we change the way file size limits are
-	 * handled, we should also revisit how this command is run (and
-	 * with which parameters).   rgerhards, 2007-07-20
-	 */
-	/* we first check if we have command line parameters. We assume this, 
-	 * when we have a space in the program name. If we find it, everything after
-	 * the space is treated as a single argument.
-	 */
-	if((pCmd = (uchar*)strdup((char*)pData->f_sizeLimitCmd)) == NULL) {
-		/* there is not much we can do - we make syslogd close the file in this case */
-		return 1;
-		}
-
-	for(p = pCmd ; *p && *p != ' ' ; ++p) {
-		/* JUST SKIP */
-	}
-
-	if(*p == ' ') {
-		*p = '\0'; /* pretend string-end */
-		pParams = p+1;
-	} else
-		pParams = NULL;
-
-	execProg(pCmd, 1, pParams);
-
-	free(pCmd);
-
-	pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
-			pData->fCreateMode);
-
-	actualFileSize = lseek(pData->fd, 0, SEEK_END);
-	if(actualFileSize >= pData->f_sizeLimit) {
-		/* OK, it didn't work out... */
-		return 1;
-		}
-
-	return 0;
-}
-
-
 /* This function deletes an entry from the dynamic file name
  * cache. A pointer to the cache must be passed in as well
  * as the index of the to-be-deleted entry. This index may
@@ -321,23 +272,24 @@ int resolveFileSizeLimit(instanceData *pData)
  * function immediately returns. Parameter bFreeEntry is 1
  * if the entry should be d_free()ed and 0 if not.
  */
-static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry)
+static rsRetVal
+dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry)
 {
+	DEFiRet;
 	ASSERT(pCache != NULL);
 
-	BEGINfunc;
-
 	if(pCache[iEntry] == NULL)
 		FINALIZE;
 
 	DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry,
-		pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName);
+		pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName);
 	/* if the name is NULL, this is an improperly initilized entry which
 	 * needs to be discarded. In this case, neither the file is to be closed
 	 * not the name to be freed.
 	 */
 	if(pCache[iEntry]->pName != NULL) {
-		close(pCache[iEntry]->fd);
+		if(pCache[iEntry]->pStrm != NULL)
+			strm.Destruct(&pCache[iEntry]->pStrm);
 		d_free(pCache[iEntry]->pName);
 		pCache[iEntry]->pName = NULL;
 	}
@@ -348,7 +300,7 @@ static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int b
 	}
 
 finalize_it:
-	ENDfunc;
+	RETiRet;
 }
 
 
@@ -356,7 +308,8 @@ finalize_it:
  * relevant files. Part of Shutdown and HUP processing.
  * rgerhards, 2008-10-23
  */
-static inline void dynaFileFreeCacheEntries(instanceData *pData)
+static inline void
+dynaFileFreeCacheEntries(instanceData *pData)
 {
 	register int i;
 	ASSERT(pData != NULL);
@@ -392,25 +345,18 @@ static void dynaFileFreeCache(instanceData *pData)
 static rsRetVal
 prepareFile(instanceData *pData, uchar *newFileName)
 {
+	int fd;
 	DEFiRet;
-	if(pData->fileType == eTypePIPE) {
-		pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC);
-		FINALIZE; /* we are done in this case */
-	}
 
-	if(access((char*)newFileName, F_OK) == 0) {
-		/* file already exists */
-		pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
-				pData->fCreateMode);
-	} else {
-		pData->fd = -1;
+	if(access((char*)newFileName, F_OK) != 0) {
 		/* file does not exist, create it (and eventually parent directories */
+		fd = -1;
 		if(pData->bCreateDirs) {
 			/* We first need to create parent dirs if they are missing.
 			 * We do not report any errors here ourselfs but let the code
 			 * fall through to error handler below.
 			 */
-			if(makeFileParentDirs(newFileName, strlen((char*)newFileName),
+			if(makeFileParentDirs(newFileName, ustrlen(newFileName),
 			     pData->fDirCreateMode, pData->dirUID,
 			     pData->dirGID, pData->bFailOnChown) != 0) {
 			     	ABORT_FINALIZE(RS_RET_ERR); /* we give up */
@@ -419,17 +365,17 @@ prepareFile(instanceData *pData, uchar *newFileName)
 		/* no matter if we needed to create directories or not, we now try to create
 		 * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater)
 		 */
-		pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
+		fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
 				pData->fCreateMode);
-		if(pData->fd != -1) {
+		if(fd != -1) {
 			/* check and set uid/gid */
 			if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) {
 				/* we need to set owner/group */
-				if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) {
+				if(fchown(fd, pData->fileUID, pData->fileGID) != 0) {
 					if(pData->bFailOnChown) {
 						int eSave = errno;
-						close(pData->fd);
-						pData->fd = -1;
+						close(fd);
+						fd = -1;
 						errno = eSave;
 					}
 					/* we will silently ignore the chown() failure
@@ -437,19 +383,37 @@ prepareFile(instanceData *pData, uchar *newFileName)
 					 */
 				}
 			}
+			close(fd); /* close again, as we need a stream further on */
 		}
 	}
-finalize_it:
-	/* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean
-	 * non-open file descriptor. Anyhow, I leave this comment for the time being to that if
-	 * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19
-	 */
-	if(pData->fd != -1 && isatty(pData->fd)) {
-		DBGPRINTF("file %d is a tty file\n", pData->fd);
-		pData->fileType = eTypeTTY;
-		untty();
-	}
 
+	/* the copies below are clumpsy, but there is no way around given the
+	 * anomalies in dirname() and basename() [they MODIFY the provided buffer...]
+	 */
+	uchar szNameBuf[MAXFNAME];
+	uchar szDirName[MAXFNAME];
+	uchar szBaseName[MAXFNAME];
+	ustrncpy(szNameBuf, newFileName, MAXFNAME);
+	ustrncpy(szDirName, (uchar*)dirname((char*)szNameBuf), MAXFNAME);
+	ustrncpy(szNameBuf, newFileName, MAXFNAME);
+	ustrncpy(szBaseName, (uchar*)basename((char*)szNameBuf), MAXFNAME);
+
+	CHKiRet(strm.Construct(&pData->pStrm));
+	CHKiRet(strm.SetFName(pData->pStrm, szBaseName, ustrlen(szBaseName)));
+	CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName)));
+	CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel));
+	CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize));
+	CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval));
+	CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND));
+	CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode));
+	CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile));
+	CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE));
+	CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit));
+	if(pData->pszSizeLimitCmd != NULL)
+		CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd)));
+	CHKiRet(strm.ConstructFinalize(pData->pStrm));
+	
+finalize_it:
 	RETiRet;
 }
 
@@ -462,15 +426,16 @@ finalize_it:
  * be written.
  * This is a helper to writeFile(). rgerhards, 2007-07-03
  */
-static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
+static inline rsRetVal
+prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
 {
 	time_t ttOldest; /* timestamp of oldest element */
 	int iOldest;
 	int i;
 	int iFirstFree;
+	rsRetVal localRet;
 	dynaFileCacheEntry **pCache;
-	
-	BEGINfunc
+	DEFiRet;
 
 	ASSERT(pData != NULL);
 	ASSERT(newFileName != NULL);
@@ -481,15 +446,17 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 	 * I *hope* this will be a performance enhancement.
 	 */
 	if(   (pData->iCurrElt != -1)
-	   && !strcmp((char*) newFileName, (char*) pCache[pData->iCurrElt]->pName)) {
+	   && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) {
 	   	/* great, we are all set */
-		pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */
-		return 0;
+		pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call!
+		// LRU needs only a strictly monotonically increasing counter, so such a one could do
+		FINALIZE;
 	}
 
 	/* ok, no luck. Now let's search the table if we find a matching spot.
 	 * While doing so, we also prepare for creation of a new one.
 	 */
+	pData->iCurrElt = -1;	/* invalid current element pointer */
 	iFirstFree = -1; /* not yet found */
 	iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */
 	ttOldest = time(NULL) + 1; /* there must always be an older one */
@@ -498,12 +465,12 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 			if(iFirstFree == -1)
 				iFirstFree = i;
 		} else { /* got an element, let's see if it matches */
-			if(!strcmp((char*) newFileName, (char*) pCache[i]->pName)) {
+			if(!ustrcmp(newFileName, pCache[i]->pName)) {
 				/* we found our element! */
-				pData->fd = pCache[i]->fd;
+				pData->pStrm = pCache[i]->pStrm;
 				pData->iCurrElt = i;
 				pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */
-				return 0;
+				FINALIZE;
 			}
 			/* did not find it - so lets keep track of the counters for LRU */
 			if(pCache[i]->lastUsed < ttOldest) {
@@ -524,18 +491,14 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 		iFirstFree = iOldest; /* this one *is* now free ;) */
 	} else {
 		/* we need to allocate memory for the cache structure */
-		pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry));
-		if(pCache[iFirstFree] == NULL) {
-			DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n");
-			return -1;
-		}
+		CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)));
 	}
 
 	/* Ok, we finally can open the file */
-	prepareFile(pData, newFileName); /* ignore exact error, we check fd below */
+	localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */
 
 	/* file is either open now or an error state set */
-	if(pData->fd == -1) {
+	if(pData->pStrm == NULL) {
 		/* do not report anything if the message is an internally-generated
 		 * message. Otherwise, we could run into a never-ending loop. The bad
 		 * news is that we also lose errors on startup messages, but so it is.
@@ -543,22 +506,43 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
 		if(iMsgOpts & INTERNAL_MSG) {
 			DBGPRINTF("Could not open dynaFile, discarding message\n");
 		} else {
-			errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName);
+			errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName);
 		}
 		dynaFileDelCacheEntry(pCache, iFirstFree, 1);
-		pData->iCurrElt = -1;
-		return -1;
+		ABORT_FINALIZE(localRet);
 	}
 
-	pCache[iFirstFree]->fd = pData->fd;
-	pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */
-	pCache[iFirstFree]->lastUsed = time(NULL);
+	CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName));
+	pCache[iFirstFree]->pStrm = pData->pStrm;
+	pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance
 	pData->iCurrElt = iFirstFree;
 	DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName);
 
-	ENDfunc
+finalize_it:
+	RETiRet;
+}
+
+
+/* do the actual write process. This function is to be called once we are ready for writing.
+ * It will do buffered writes and persist data only when the buffer is full. Note that we must
+ * be careful to detect when the file handle changed.
+ * rgerhards, 2009-06-03
+ */
+static  rsRetVal
+doWrite(instanceData *pData, uchar *pszBuf, int lenBuf)
+{
+	DEFiRet;
+	ASSERT(pData != NULL);
+	ASSERT(pszBuf != NULL);
+
+dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf);
+	if(pData->pStrm != NULL){
+		CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf));
+		FINALIZE;
+	}
 
-	return 0;
+finalize_it:
+	RETiRet;
 }
 
 
@@ -566,10 +550,9 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg
  * will be called for all outputs using file semantics,
  * for example also for pipes.
  */
-static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData)
+static rsRetVal
+writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData)
 {
-	off_t actualFileSize;
-	int iLenWritten;
 	DEFiRet;
 
 	ASSERT(pData != NULL);
@@ -578,106 +561,27 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa
 	 * check if it still is ok or a new file needs to be created
 	 */
 	if(pData->bDynamicName) {
-		if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0)
-			ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */
-	}
-	
-	if(pData->fd == -1) {
-		rsRetVal iRetLocal;
-		iRetLocal = prepareFile(pData, pData->f_fname);
-		if((iRetLocal != RS_RET_OK) || (pData->fd == -1))
-			ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */
-	}
-
-	/* create the message based on format specified */
-again:
-	/* check if we have a file size limit and, if so,
-	 * obey to it.
-	 */
-	if(pData->f_sizeLimit != 0) {
-		actualFileSize = lseek(pData->fd, 0, SEEK_END);
-		if(actualFileSize >= pData->f_sizeLimit) {
-			char errMsg[256];
-			/* for now, we simply disable a file once it is
-			 * beyond the maximum size. This is better than having
-			 * us aborted by the OS... rgerhards 2005-06-21
-			 */
-			(void) close(pData->fd);
-			/* try to resolve the situation */
-			if(resolveFileSizeLimit(pData) != 0) {
-				/* didn't work out, so disable... */
-				snprintf(errMsg, sizeof(errMsg),
-					 "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation",
-					 pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize);
-				errno = 0;
-				errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg);
-				ABORT_FINALIZE(RS_RET_DISABLE_ACTION);
-			} else {
-				snprintf(errMsg, sizeof(errMsg),
-					 "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation",
-					 pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize);
-				errno = 0;
-				errmsg.LogError(0, NO_ERRCODE, "%s", errMsg);
-			}
+		CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts));
+	} else { /* "regular", non-dynafile */
+		if(pData->pStrm == NULL) {
+			CHKiRet(prepareFile(pData, pData->f_fname));
 		}
 	}
 
-	iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0]));
-//dbgprintf("lenwritten: %d\n", iLenWritten);
-	if(iLenWritten < 0) {
-		int e = errno;
-		char errStr[1024];
-		rs_strerror_r(errno, errStr, sizeof(errStr));
-		DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr);
-
-		/* If a named pipe is full, we suspend this action for a while */
-		if(pData->fileType == eTypePIPE && e == EAGAIN)
-			ABORT_FINALIZE(RS_RET_SUSPENDED);
-
-		close(pData->fd);
-		pData->fd = -1; /* tell that fd is no longer open! */
-		if(pData->bDynamicName && pData->iCurrElt != -1) {
-			/* in this case, we need to invalidate the name in the cache, too
-			 * otherwise, an invalid fd may show up if we had a file name change.
-			 * rgerhards, 2009-03-19
-			 */
-			pData->dynCache[pData->iCurrElt]->fd = -1;
-		}
-		/* Check for EBADF on TTY's due to vhangup()
-		 * Linux uses EIO instead (mrn 12 May 96)
-		 */
-		if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE)
-#ifdef linux
-			&& e == EIO
-#else
-			&& e == EBADF
-#endif
-			) {
-			pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC);
-			if (pData->fd < 0) {
-				iRet = RS_RET_SUSPENDED;
-				errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname);
-			} else {
-				untty();
-				goto again;
-			}
-		} else {
-			iRet = RS_RET_SUSPENDED;
-			errno = e;
-			errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname);
-		}
-	} else if (pData->bSyncFile) {
-		fsync(pData->fd);
-	}
+	CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0]))));
 
 finalize_it:
+	if(iRet != RS_RET_OK) {
+		/* in v5, we shall return different states for message-cause failur (but only there!) */
+		iRet = RS_RET_SUSPENDED;
+	}
 	RETiRet;
 }
 
 
 BEGINcreateInstance
 CODESTARTcreateInstance
-	pData->fd = -1;
+	pData->pStrm = NULL;
 ENDcreateInstance
 
 
@@ -685,8 +589,8 @@ BEGINfreeInstance
 CODESTARTfreeInstance
 	if(pData->bDynamicName) {
 		dynaFileFreeCache(pData);
-	} else if(pData->fd != -1)
-		close(pData->fd);
+	} else if(pData->pStrm != NULL)
+		strm.Destruct(&pData->pStrm);
 ENDfreeInstance
 
 
@@ -696,39 +600,32 @@ ENDtryResume
 
 BEGINdoAction
 CODESTARTdoAction
-	DBGPRINTF(" (%s)\n", pData->f_fname);
-	iRet = writeFile(ppString, iMsgOpts, pData);
+	DBGPRINTF("file to log to: %s\n", pData->f_fname);
+	CHKiRet(writeFile(ppString, iMsgOpts, pData));
+	if(pData->bFlushOnTXEnd) {
+		/* TODO v5: do this in endTransaction only! */
+		CHKiRet(strm.Flush(pData->pStrm));
+	}
+finalize_it:
 ENDdoAction
 
 
 BEGINparseSelectorAct
 CODESTARTparseSelectorAct
-	/* yes, the if below is redundant, but I need it now. Will go away as
-	 * the code further changes.  -- rgerhards, 2007-07-25
-	 */
-	if(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-') {
-		if((iRet = createInstance(&pData)) != RS_RET_OK) {
-			ENDfunc
-			return iRet; /* this can not use RET_iRet! */
-		}
-	} else {
-		/* this is not clean, but we need it for the time being
-		 * TODO: remove when cleaning up modularization 
-		 */
-		ENDfunc
-		return RS_RET_CONFLINE_UNPROCESSED;
-	}
+	if(!(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-'))
+		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
+
+	CHKiRet(createInstance(&pData));
 
 	if(*p == '-') {
 		pData->bSyncFile = 0;
 		p++;
 	} else {
-		pData->bSyncFile = bEnableSync ? 1 : 0;
+		pData->bSyncFile = bEnableSync;
 	}
-	pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */
+	pData->iSizeLimit = 0; /* default value, use outchannels to configure! */
 
-	switch (*p)
-	{
+	switch(*p) {
         case '$':
 		CODE_STD_STRING_REQUESTparseSelectorAct(1)
 		/* rgerhards 2005-06-21: this is a special setting for output-channel
@@ -738,13 +635,8 @@ CODESTARTparseSelectorAct
 		 * rgerhards, 2007-07-24: output-channels will go away. We keep them
 		 * for compatibility reasons, but seems to have been a bad idea.
 		 */
-		if((iRet = cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)) == RS_RET_OK) {
-			pData->bDynamicName = 0;
-			pData->fCreateMode = fCreateMode; /* preserve current setting */
-			pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */
-			pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC,
-					 pData->fCreateMode);
-		}
+		CHKiRet(cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS));
+		pData->bDynamicName = 0;
 		break;
 
 	case '?': /* This is much like a regular file handle, but we need to obtain
@@ -752,84 +644,56 @@ CODESTARTparseSelectorAct
 		   */
 		CODE_STD_STRING_REQUESTparseSelectorAct(2)
 		++p; /* eat '?' */
-		if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName))
-		   != RS_RET_OK)
-			break;
+		CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
+				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName));
 		/* "filename" is actually a template name, we need this as string 1. So let's add it
 		 * to the pOMSR. -- rgerhards, 2007-07-27
 		 */
-		if((iRet = OMSRsetEntry(*ppOMSR, 1, (uchar*)strdup((char*) pData->f_fname), OMSR_NO_RQD_TPL_OPTS)) != RS_RET_OK)
-			break;
+		CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pData->f_fname), OMSR_NO_RQD_TPL_OPTS));
 
 		pData->bDynamicName = 1;
 		pData->iCurrElt = -1;		  /* no current element */
-		pData->fCreateMode = fCreateMode; /* freeze current setting */
-		pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */
-		pData->bCreateDirs = bCreateDirs;
-		pData->bFailOnChown = bFailOnChown;
-		pData->fileUID = fileUID;
-		pData->fileGID = fileGID;
-		pData->dirUID = dirUID;
-		pData->dirGID = dirGID;
-		pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */
-		/* we now allocate the cache table. We use calloc() intentionally, as we 
-		 * need all pointers to be initialized to NULL pointers.
-		 */
-		if((pData->dynCache = (dynaFileCacheEntry**)
-		    calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) {
-			iRet = RS_RET_OUT_OF_MEMORY;
-			DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n");
-		}
+		/* we now allocate the cache table */
+		CHKmalloc(pData->dynCache = (dynaFileCacheEntry**)
+				calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*)));
 		break;
 
         case '|':
 	case '/':
 		CODE_STD_STRING_REQUESTparseSelectorAct(1)
-		/* rgerhards, 2007-0726: first check if file or pipe */
-		if(*p == '|') {
-			pData->fileType = eTypePIPE;
-			++p;
-		} else {
-			pData->fileType = eTypeFILE;
-		}
-		/* rgerhards 2004-11-17: from now, we need to have different
-		 * processing, because after the first comma, the template name
-		 * to use is specified. So we need to scan for the first coma first
-		 * and then look at the rest of the line.
-		 */
-		if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName))
-		   != RS_RET_OK)
-			break;
-
+		CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
+				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName));
 		pData->bDynamicName = 0;
-		pData->fCreateMode = fCreateMode; /* preserve current setting */
-		pData->fDirCreateMode = fDirCreateMode;
-		pData->bCreateDirs = bCreateDirs;
-		pData->bFailOnChown = bFailOnChown;
-		pData->fileUID = fileUID;
-		pData->fileGID = fileGID;
-		pData->dirUID = dirUID;
-		pData->dirGID = dirGID;
-
-		/* at this stage, we ignore the return value of prepareFile, this is taken
-		 * care of in later steps. -- rgerhards, 2009-03-19
+		break;
+	default:
+		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
+	}
+
+	/* freeze current paremeters for this action */
+	pData->iDynaFileCacheSize = iDynaFileCacheSize;
+	pData->fCreateMode = fCreateMode;
+	pData->fDirCreateMode = fDirCreateMode;
+	pData->bCreateDirs = bCreateDirs;
+	pData->bFailOnChown = bFailOnChown;
+	pData->fileUID = fileUID;
+	pData->fileGID = fileGID;
+	pData->dirUID = dirUID;
+	pData->dirGID = dirGID;
+	pData->iZipLevel = iZipLevel;
+	pData->bFlushOnTXEnd = bFlushOnTXEnd;
+	pData->iIOBufSize = iIOBufSize;
+	pData->iFlushInterval = iFlushInterval;
+
+	if(pData->bDynamicName == 0) {
+		/* try open and emit error message if not possible. At this stage, we ignore the
+		 * return value of prepareFile, this is taken care of in later steps.
 		 */
 		prepareFile(pData, pData->f_fname);
 		        
-	  	if(pData->fd < 0 ) {
-			pData->fd = -1;
+	  	if(pData->pStrm == NULL) {
 			DBGPRINTF("Error opening log file: %s\n", pData->f_fname);
 			errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname);
-			break;
 		}
-		if(strcmp((char*) p, _PATH_CONSOLE) == 0)
-			pData->fileType = eTypeCONSOLE;
-		break;
-	default:
-		iRet = RS_RET_CONFLINE_UNPROCESSED;
-		break;
 	}
 CODE_STD_FINALIZERparseSelectorAct
 ENDparseSelectorAct
@@ -850,6 +714,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	fDirCreateMode = 0700;
 	bCreateDirs = 1;
 	bEnableSync = 0;
+	iZipLevel = 0;
+	bFlushOnTXEnd = 0;
+	iIOBufSize = IOBUF_DFLT_SIZE;
+	iFlushInterval = FLUSH_INTRVL_DFLT;
 	if(pszTplName != NULL) {
 		free(pszTplName);
 		pszTplName = NULL;
@@ -865,9 +733,9 @@ CODESTARTdoHUP
 		dynaFileFreeCacheEntries(pData);
 		pData->iCurrElt = -1; /* invalidate current element */
 	} else {
-		if(pData->fd != -1) {
-			close(pData->fd);
-			pData->fd = -1;
+		if(pData->pStrm != NULL) {
+			strm.Destruct(&pData->pStrm);
+			pData->pStrm = NULL;
 		}
 	}
 ENDdoHUP
@@ -875,8 +743,10 @@ ENDdoHUP
 
 BEGINmodExit
 CODESTARTmodExit
-	if(pszTplName != NULL)
-		free(pszTplName);
+	objRelease(errmsg, CORE_COMPONENT);
+	objRelease(strm, CORE_COMPONENT);
+	objRelease(zlibw, LM_ZLIBW_FILENAME);
+	free(pszTplName);
 ENDmodExit
 
 
@@ -892,7 +762,13 @@ CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+	CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME));
+	CHKiRet(objUse(strm, CORE_COMPONENT));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID));
diff --git a/tools/omfwd.c b/tools/omfwd.c
index c8fedfc9..2966a5e4 100644
--- a/tools/omfwd.c
+++ b/tools/omfwd.c
@@ -48,6 +48,7 @@
 #endif
 #include <pthread.h>
 #include "syslogd.h"
+#include "conf.h"
 #include "syslogd-types.h"
 #include "srUtils.h"
 #include "net.h"
diff --git a/tools/omshell.c b/tools/omshell.c
index 7b815869..f8a68527 100644
--- a/tools/omshell.c
+++ b/tools/omshell.c
@@ -38,7 +38,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
-#include "syslogd.h"
+#include "conf.h"
 #include "syslogd-types.h"
 #include "srUtils.h"
 #include "omshell.h"
diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c
index 830bbc87..499a11dd 100644
--- a/tools/omusrmsg.c
+++ b/tools/omusrmsg.c
@@ -66,7 +66,7 @@
 #include "srUtils.h"
 #include "stringbuf.h"
 #include "syslogd-types.h"
-#include "syslogd.h"
+#include "conf.h"
 #include "omusrmsg.h"
 #include "module-template.h"
 #include "errmsg.h"
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 3d91e22f..6d6bcea1 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -133,13 +133,16 @@
 #include "queue.h"
 #include "stream.h"
 #include "conf.h"
-#include "vm.h"
 #include "errmsg.h"
 #include "datetime.h"
 #include "parser.h"
-#include "sysvar.h"
+//#include "sysvar.h"
 #include "batch.h"
 #include "unicode-helper.h"
+#include "ruleset.h"
+#include "rule.h"
+#include "net.h"
+#include "vm.h"
 
 /* definitions for objects we access */
 DEFobjCurrIf(obj)
@@ -147,10 +150,10 @@ DEFobjCurrIf(glbl)
 DEFobjCurrIf(datetime)
 DEFobjCurrIf(conf)
 DEFobjCurrIf(expr)
-DEFobjCurrIf(vm)
-DEFobjCurrIf(var)
 DEFobjCurrIf(module)
 DEFobjCurrIf(errmsg)
+DEFobjCurrIf(rule)
+DEFobjCurrIf(ruleset)
 DEFobjCurrIf(net) /* TODO: make go away! */
 
 
@@ -222,7 +225,7 @@ static rsRetVal GlobalClassExit(void);
 #endif
 
 #ifndef _PATH_TTY
-#define _PATH_TTY	"/dev/tty"
+#	define _PATH_TTY	"/dev/tty"
 #endif
 
 static uchar	*ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */
@@ -249,8 +252,6 @@ int	repeatinterval[2] = { 30, 60 };	/* # of secs before flush */
 
 #define LIST_DELIMITER	':'		/* delimiter between two hosts */
 
-struct	filed *Files = NULL; /* read-only after init() (but beware of sigusr1!) */
-
 static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */
 
 typedef struct legacyOptsLL_s {
@@ -300,6 +301,7 @@ static queueType_t MainMsgQueType = QUEUETYPE_FIXED_ARRAY;	/* type of the main m
 static uchar *pszMainMsgQFName = NULL;				/* prefix for the main message queue file */
 static int64 iMainMsgQueMaxFileSize = 1024*1024;
 static int iMainMsgQPersistUpdCnt = 0;				/* persist queue info every n updates */
+static int bMainMsgQSyncQeueFiles = 0;				/* sync queue files on every write? */
 static int iMainMsgQtoQShutdown = 0;				/* queue shutdown */ 
 static int iMainMsgQtoActShutdown = 1000;			/* action shutdown (in phase 2) */ 
 static int iMainMsgQtoEnq = 2000;				/* timeout for queue enque */ 
@@ -316,7 +318,8 @@ static int iMainMsgQueueDeqtWinToHr = 25;			/* hour begin of time frame when que
 /* support for simple textual representation of FIOP names
  * rgerhards, 2005-09-27
  */
-static char* getFIOPName(unsigned iFIOP)
+char*
+getFIOPName(unsigned iFIOP)
 {
 	char *pRet;
 	switch(iFIOP) {
@@ -363,6 +366,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	iMainMsgQueMaxFileSize = 1024 * 1024;
 	iMainMsgQueueNumWorkers = 1;
 	iMainMsgQPersistUpdCnt = 0;
+	bMainMsgQSyncQeueFiles = 0;
 	iMainMsgQtoQShutdown = 0;
 	iMainMsgQtoActShutdown = 1000;
 	iMainMsgQtoEnq = 2000;
@@ -399,7 +403,6 @@ static char **crunch_list(char *list);
 static void reapchild();
 static void debug_switch();
 static void sighup_handler();
-static void freeSelectors(void);
 static void processImInternal(void);
 
 
@@ -434,67 +437,6 @@ diagGetMainMsgQSize(int *piSize)
 /* ------------------------------ end support functions for imdiag  ------------------------------ */
 
 
-/* function to destruct a selector_t object
- * rgerhards, 2007-08-01
- */
-rsRetVal
-selectorDestruct(void *pVal)
-{
-	selector_t *pThis = (selector_t *) pVal;
-
-	assert(pThis != NULL);
-
-	if(pThis->pCSHostnameComp != NULL)
-		rsCStrDestruct(&pThis->pCSHostnameComp);
-	if(pThis->pCSProgNameComp != NULL)
-		rsCStrDestruct(&pThis->pCSProgNameComp);
-
-	if(pThis->f_filter_type == FILTER_PROP) {
-		if(pThis->f_filterData.prop.pCSPropName != NULL)
-			rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName);
-		if(pThis->f_filterData.prop.pCSCompValue != NULL)
-			rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue);
-		if(pThis->f_filterData.prop.regex_cache != NULL)
-			rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache);
-	} else if(pThis->f_filter_type == FILTER_EXPR) {
-		if(pThis->f_filterData.f_expr != NULL)
-			expr.Destruct(&pThis->f_filterData.f_expr);
-	}
-
-	llDestroy(&pThis->llActList);
-	free(pThis);
-	
-	return RS_RET_OK;
-}
-
-
-/* function to construct a selector_t object
- * rgerhards, 2007-08-01
- */
-rsRetVal
-selectorConstruct(selector_t **ppThis)
-{
-	DEFiRet;
-	selector_t *pThis;
-
-	assert(ppThis != NULL);
-	
-	if((pThis = (selector_t*) calloc(1, sizeof(selector_t))) == NULL) {
-		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
-	}
-	CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL));
-
-finalize_it:
-	if(iRet != RS_RET_OK) {
-		if(pThis != NULL) {
-			selectorDestruct(pThis);
-		}
-	}
-	*ppThis = pThis;
-	RETiRet;
-}
-
-
 /* rgerhards, 2005-10-24: crunch_list is called only during option processing. So
  * it is never called once rsyslogd is running (not even when HUPed). This code
  * contains some exits, but they are considered safe because they only happen
@@ -557,7 +499,7 @@ static char **crunch_list(char *list)
 #if 0
 	count=0;
 	while (result[count])
-		dbgprintf("#%d: %s\n", count, StripDomains[count++]);
+		DBGPRINTF("#%d: %s\n", count, StripDomains[count++]);
 #endif
 	return result;
 }
@@ -566,7 +508,7 @@ static char **crunch_list(char *list)
 void untty(void)
 #ifdef HAVE_SETSID
 {
-	if ( !Debug ) {
+	if(!Debug) {
 		setsid();
 	}
 	return;
@@ -575,18 +517,18 @@ void untty(void)
 {
 	int i;
 
-	if ( !Debug ) {
+	if(!Debug) {
 		i = open(_PATH_TTY, O_RDWR|O_CLOEXEC);
 		if (i >= 0) {
 #			if !defined(__hpux)
-				(void) ioctl(i, (int) TIOCNOTTY, (char *)0);
+				(void) ioctl(i, (int) TIOCNOTTY, NULL);
 #			else
 				/* TODO: we need to implement something for HP UX! -- rgerhards, 2008-03-04 */
 				/* actually, HP UX should have setsid, so the code directly above should
 				 * trigger. So the actual question is why it doesn't do that...
 				 */
 #			endif
-			(void) close(i);
+			close(i);
 		}
 	}
 }
@@ -643,7 +585,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f
 		CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime));
 	}
 	if(pszInputName != NULL)
-		MsgSetInputName(pMsg, pszInputName);
+		MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName));
 	MsgSetFlowControlType(pMsg, flowCtlType);
 	MsgSetRawMsg(pMsg, (char*)msg);
 	
@@ -674,16 +616,9 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f
 	if((pMsg->msgFlags & PARSE_HOSTNAME) == 0)
 		MsgSetHOSTNAME(pMsg, hname);
 	MsgSetRcvFrom(pMsg, hname);
+	MsgSetAfterPRIOffs(pMsg, p - msg);
 	CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP));
 
-	/* rgerhards 2004-11-19: well, well... we've now seen that we
-	 * have the "hostname problem" also with the traditional Unix
-	 * message. As we like to emulate it, we need to add the hostname
-	 * to it.
-	 */
-	if(MsgSetUxTradMsg(pMsg, (char*)p) != 0)
-		ABORT_FINALIZE(RS_RET_ERR);
-
 	logmsg(pMsg, flags);
 
 finalize_it:
@@ -784,7 +719,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla
 	 * rgerhards, 2007-09-14
 	 */
 	if(*(msg + len - 1) == '\0') {
-		dbgprintf("dropped NUL at very end of message\n");
+		DBGPRINTF("dropped NUL at very end of message\n");
 		len--;
 	}
 
@@ -794,7 +729,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla
 	 * turn on/off this handling.  rgerhards, 2007-07-23
 	 */
 	if(bDropTrailingLF && *(msg + len - 1) == '\n') {
-		dbgprintf("dropped LF at very end of message (DropTrailingLF is set)\n");
+		DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n");
 		len--;
 	}
 
@@ -819,7 +754,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla
 		iLenDefBuf = iMaxLine;
 		CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1)));
 		ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1);
-		dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n",
+		DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n",
 		        ret, (long) iLenDefBuf, len-1);
 		/* Now check if the uncompression worked. If not, there is not much we can do. In
 		 * that case, we log an error message but ignore the message itself. Storing the
@@ -865,7 +800,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla
 				 * (I couldn't do any more smart things anyway...).
 				 * rgerhards, 2007-9-20
 				 */
-				dbgprintf("internal error: iMsg > max msg size in printchopped()\n");
+				DBGPRINTF("internal error: iMsg > max msg size in printchopped()\n");
 			}
 			FINALIZE; /* in this case, we are done... nothing left we can do */
 		}
@@ -951,8 +886,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags)
 	DEFiRet;
 
 	CHKiRet(msgConstruct(&pMsg));
-	MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"));
-	MsgSetUxTradMsg(pMsg, (char*)msg);
+	MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1);
 	MsgSetRawMsg(pMsg, (char*)msg);
 	MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName());
 	MsgSetRcvFrom(pMsg, glbl.GetLocalHostName());
@@ -996,6 +930,8 @@ finalize_it:
 	RETiRet;
 }
 
+#if 0 // Code obsoleted by merge imfile, but check for changes if there are problems
+//thus I leave it in for the time being TODO: remove
 /* This functions looks at the given message and checks if it matches the
  * provided filter condition. If so, it returns true, else it returns
  * false. This is a helper to logmsg() and meant to drive the decision
@@ -1226,6 +1162,7 @@ processMsg(msg_t *pMsg)
 	ENDfunc
 }
 
+#endif // if 0 from merge omfile
 
 /* The consumer of dequeued messages. This function is called by the
  * queue engine on dequeueing of a message. It runs on a SEPARATE
@@ -1248,7 +1185,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch)
 		if((pMsg->msgFlags & NEEDS_PARSING) != 0) {
 			parseMsg(pMsg);
 		}
-		processMsg(pMsg);
+		ruleset.ProcessMsg(pMsg);
 	}
 
 	RETiRet;
@@ -1382,8 +1319,8 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 
 	BEGINfunc
 	assert(pMsg != NULL);
-	assert(pMsg->pszUxTradMsg != NULL);
-	p2parse = pMsg->pszUxTradMsg;
+	assert(pMsg->pszRawMsg != NULL);
+	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
 
 	/* do a sanity check on the version and eat it */
 	assert(p2parse[0] == '1' && p2parse[1] == ' ');
@@ -1411,7 +1348,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 			memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime));
 		}
 	} else {
-		dbgprintf("no TIMESTAMP detected!\n");
+		DBGPRINTF("no TIMESTAMP detected!\n");
 		bContParse = 0;
 	}
 
@@ -1483,8 +1420,8 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 	BEGINfunc
 
 	assert(pMsg != NULL);
-	assert(pMsg->pszUxTradMsg != NULL);
-	p2parse = pMsg->pszUxTradMsg;
+	assert(pMsg->pszRawMsg != NULL);
+	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
 
 	/* Check to see if msg contains a timestamp. We start by assuming
 	 * that the message timestamp is the time of reciption (which we 
@@ -1573,7 +1510,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 			/* indeed, this smells like a TAG, so lets use it for this. We take
 			 * the HOSTNAME from the sender system instead.
 			 */
-			dbgprintf("HOSTNAME contains invalid characters, assuming it to be a TAG.\n");
+			DBGPRINTF("HOSTNAME contains invalid characters, assuming it to be a TAG.\n");
 			moveHOSTNAMEtoTAG(pMsg);
 			MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 		}
@@ -1598,22 +1535,21 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 		 * the records: the code is currently clean, but we could optimize it! */
 		if(!bTAGCharDetected) {
 			uchar *pszTAG;
-			if(rsCStrConstruct(&pStrB) != RS_RET_OK) 
+			if(cstrConstruct(&pStrB) != RS_RET_OK) 
 				return 1;
 			rsCStrSetAllocIncrement(pStrB, 33);
 			pWork = pBuf;
 			iCnt = 0;
 			while(*p2parse && *p2parse != ':' && *p2parse != ' ') {
-				rsCStrAppendChar(pStrB, *p2parse++);
+				cstrAppendChar(pStrB, *p2parse++);
 				++iCnt;
 			}
 			if(*p2parse == ':') {
 				++p2parse; 
-				rsCStrAppendChar(pStrB, ':');
+				cstrAppendChar(pStrB, ':');
 			}
-			rsCStrFinish(pStrB);
-
-			rsCStrConvSzStrAndDestruct(pStrB, &pszTAG, 1);
+			cstrFinalize(pStrB);
+			cstrConvSzStrAndDestruct(pStrB, &pszTAG, 1);
 			if(pszTAG == NULL)
 			{	/* rger, 2005-11-10: no TAG found - this implies that what
 				 * we have considered to be the HOSTNAME is most probably the
@@ -1623,7 +1559,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 				 * the hostname. This situation is the standard case with
 				 * stock BSD syslogd.
 				 */
-				dbgprintf("No TAG in message, assuming that HOSTNAME is missing.\n");
+				DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n");
 				moveHOSTNAMEtoTAG(pMsg);
 				MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 			} else { /* we have a TAG, so we can happily set it ;) */
@@ -1639,7 +1575,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 		 */
 		if(!(flags & INTERNAL_MSG))
 		{
-			dbgprintf("HOSTNAME and TAG not parsed by user configuraton.\n");
+			DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n");
 			MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg));
 		}
 	}
@@ -1698,9 +1634,10 @@ logmsg(msg_t *pMsg, int flags)
 
 	BEGINfunc
 	assert(pMsg != NULL);
-	assert(pMsg->pszUxTradMsg != NULL);
-	msg = (char*) pMsg->pszUxTradMsg;
-	dbgprintf("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg);
+	assert(pMsg->pszRawMsg != NULL);
+
+	msg = (char*) pMsg->pszRawMsg + pMsg->offAfterPRI;  /* point to start of text, after PRI */
+	DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg);
 
 	/* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have
 	 * a traditional syslog message or one formatted according to syslog-protocol.
@@ -1708,14 +1645,14 @@ logmsg(msg_t *pMsg, int flags)
 	 * -protocol VERSION field for the detection.
 	 */
 	if(msg[0] == '1' && msg[1] == ' ') {
-		dbgprintf("Message has syslog-protocol format.\n");
+		DBGPRINTF("Message has syslog-protocol format.\n");
 		setProtocolVersion(pMsg, 1);
 		if(parseRFCSyslogMsg(pMsg, flags) == 1) {
 			msgDestruct(&pMsg);
 			return;
 		}
 	} else { /* we have legacy syslog */
-		dbgprintf("Message has legacy syslog format.\n");
+		DBGPRINTF("Message has legacy syslog format.\n");
 		setProtocolVersion(pMsg, 0);
 		if(parseLegacySyslogMsg(pMsg, flags) == 1) {
 			msgDestruct(&pMsg);
@@ -1755,7 +1692,6 @@ reapchild()
 DEFFUNC_llExecFunc(flushRptdMsgsActions)
 {
 	action_t *pAction = (action_t*) pData;
-
 	assert(pAction != NULL);
 	
 	BEGINfunc
@@ -1765,7 +1701,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions)
 	 * in an acceptable way. -- rgerhards, 2008-09-16
 	 */
 	if (pAction->f_prevcount && time(NULL) >= REPEATTIME(pAction)) {
-		dbgprintf("flush %s: repeated %d times, %d sec.\n",
+		DBGPRINTF("flush %s: repeated %d times, %d sec.\n",
 		    module.GetStateName(pAction->pMod), pAction->f_prevcount,
 		    repeatinterval[pAction->f_repeatcount]);
 		actionWriteToAction(pAction);
@@ -1778,20 +1714,12 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions)
 }
 
 
-/* This method flushes reapeat messages.
+/* This method flushes repeat messages.
  */
 static void
 doFlushRptdMsgs(void)
 {
-	register selector_t *f;
-
-	/* see if we need to flush any "message repeated n times"... 
-	 * Note that this interferes with objects running on other threads.
-	 * We are using appropriate locking inside the function to handle that.
-	 */
-	for (f = Files; f != NULL ; f = f->f_next) {
-		llExecFunc(&f->llActList, flushRptdMsgsActions, NULL);
-	}
+	ruleset.IterateAllActions(flushRptdMsgsActions, NULL);
 }
 
 
@@ -1801,9 +1729,9 @@ static void debug_switch()
 
 	if(debugging_on == 0) {
 		debugging_on = 1;
-		dbgprintf("Switching debugging_on to true\n");
+		DBGPRINTF("Switching debugging_on to true\n");
 	} else {
-		dbgprintf("Switching debugging_on to false\n");
+		DBGPRINTF("Switching debugging_on to false\n");
 		debugging_on = 0;
 	}
 	
@@ -1947,7 +1875,7 @@ void legacyOptsParseTCP(char ch, char *arg)
  * a minimal delay, but it is much cleaner than the approach of doing everything
  * inside the signal handler.
  * rgerhards, 2005-10-26
- * Note: we do not call dbgprintf() as this may cause us to block in case something
+ * Note: we do not call DBGPRINTF() as this may cause us to block in case something
  * with the threading is wrong.
  */
 static void doDie(int sig)
@@ -1983,6 +1911,16 @@ freeAllDynMemForTermination(void)
 }
 
 
+/* Finalize and destruct all actions.
+ */
+static inline void
+destructAllActions(void)
+{
+	ruleset.DestructAllActions();
+	bHaveMainQueue = 0; // flag that internal messages need to be temporarily stored
+}
+
+
 /* die() is called when the program shall end. This typically only occurs
  * during sigterm or during the initialization. 
  * As die() is intended to shutdown rsyslogd, it is
@@ -1996,7 +1934,7 @@ die(int sig)
 {
 	char buf[256];
 
-	dbgprintf("exiting on signal %d\n", sig);
+	DBGPRINTF("exiting on signal %d\n", sig);
 
 	/* IMPORTANT: we should close the inputs first, and THEN send our termination
 	 * message. If we do it the other way around, logmsgInternal() may block on
@@ -2011,7 +1949,7 @@ die(int sig)
 	 */
 
 	/* close the inputs */
-	dbgprintf("Terminating input threads...\n");
+	DBGPRINTF("Terminating input threads...\n");
 	thrdTerminateAll();
 
 	/* and THEN send the termination log message (see long comment above) */
@@ -2025,17 +1963,17 @@ die(int sig)
 	}
 	
 	/* drain queue (if configured so) and stop main queue worker thread pool */
-	dbgprintf("Terminating main queue...\n");
+	DBGPRINTF("Terminating main queue...\n");
 	qqueueDestruct(&pMsgQueue);
 	pMsgQueue = NULL;
 
 	/* Free ressources and close connections. This includes flushing any remaining
 	 * repeated msgs.
 	 */
-	dbgprintf("Terminating outputs...\n");
-	freeSelectors();
+	DBGPRINTF("Terminating outputs...\n");
+	destructAllActions();
 
-	dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n");
+	DBGPRINTF("all primary multi-thread sources have been terminated - now doing aux cleanup...\n");
 	/* rger 2005-02-22
 	 * now clean up the in-memory structures. OK, the OS
 	 * would also take care of that, but if we do it
@@ -2069,11 +2007,11 @@ die(int sig)
 	 * rgerhards, 2007-08-03
 	 * I have added some code now, but all that mod init/de-init should be moved to
 	 * init, so that modules are unloaded and reloaded on HUP to. Eventually it should go
-	 * into freeSelectors() - but that needs to be seen. -- rgerhards, 2007-08-09
+	 * into destructAllActions() - but that needs to be seen. -- rgerhards, 2007-08-09
 	 */
 	module.UnloadAndDestructAll(eMOD_LINK_ALL);
 
-	dbgprintf("Clean shutdown completed, bye\n");
+	DBGPRINTF("Clean shutdown completed, bye\n");
 	/* dbgClassExit MUST be the last one, because it de-inits the debug system */
 	dbgClassExit();
 
@@ -2130,7 +2068,7 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles)
 				iFiles, errStr, (long) maxFiles.rlim_max);
 		ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE);
 	}
-	dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max);
+	DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max);
 
 finalize_it:
 	RETiRet;
@@ -2141,7 +2079,7 @@ finalize_it:
 static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask)
 {
 	umask(iUmask);
-	dbgprintf("umask set to 0%3.3o.\n", iUmask);
+	DBGPRINTF("umask set to 0%3.3o.\n", iUmask);
 
 	return RS_RET_OK;
 }
@@ -2197,56 +2135,6 @@ static void doDropPrivUid(int iUid)
 }
 
 
-/* helper to freeSelectors(), used with llExecFunc() to flush 
- * pending output.  -- rgerhards, 2007-08-02
- * We do not need to lock the action object here as the processing
- * queue is already empty and no other threads are running when
- * we call this function. -- rgerhards, 2007-12-12
- */
-DEFFUNC_llExecFunc(freeSelectorsActions)
-{
-	action_t *pAction = (action_t*) pData;
-
-	assert(pAction != NULL);
-
-	/* flush any pending output */
-	if(pAction->f_prevcount) {
-		actionWriteToAction(pAction);
-	}
-
-	return RS_RET_OK; /* never fails ;) */
-}
-
-
-/*  Close all open log files and free selector descriptor array.
- */
-static void freeSelectors(void)
-{
-	selector_t *f;
-	selector_t *fPrev;
-
-	if(Files != NULL) {
-		dbgprintf("Freeing log structures.\n");
-
-		for(f = Files ; f != NULL ; f = f->f_next) {
-			llExecFunc(&f->llActList, freeSelectorsActions, NULL);
-		}
-
-		/* actions flushed and ready for destruction - so do that... */
-		f = Files;
-		while (f != NULL) {
-			fPrev = f;
-			f = f->f_next;
-			selectorDestruct(fPrev);
-		}
-
-		/* Reflect the deletion of the selectors linked list. */
-		Files = NULL;
-		bHaveMainQueue = 0;
-	}
-}
-
-
 /* helper to generateConfigDAG, to print out all actions via
  * the llExecFunc() facility.
  * rgerhards, 2007-08-02
@@ -2323,14 +2211,14 @@ DEFFUNC_llExecFunc(generateConfigDAGAction)
 static rsRetVal
 generateConfigDAG(uchar *pszDAGFile)
 {
-	selector_t *f;
+	//rule_t *f;
 	FILE *fp;
 	int iActUnit = 1;
-	int bHasFilter = 0;	/* filter associated with this action unit? */
-	int bHadFilter;
-	int i;
+	//int bHasFilter = 0;	/* filter associated with this action unit? */
+	//int bHadFilter;
+	//int i;
 	struct dag_info dagInfo;
-	char *pszFilterName;
+	//char *pszFilterName;
 	char szConnectingNode[64];
 	DEFiRet;
 
@@ -2359,6 +2247,8 @@ generateConfigDAG(uchar *pszDAGFile)
 	strcpy(szConnectingNode, "act0_0");
 	dagInfo.bDiscarded = 0;
 
+/* TODO: re-enable! */
+#if 0
 	for(f = Files; f != NULL ; f = f->f_next) {
 		/* BSD-Style filters are currently ignored */
 		bHadFilter = bHasFilter;
@@ -2414,6 +2304,7 @@ generateConfigDAG(uchar *pszDAGFile)
 
 		++iActUnit;
 	}
+#endif
 
 	fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit);
 	fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n"
@@ -2425,20 +2316,6 @@ finalize_it:
 }
 
 
-/* helper to dbPrintInitInfo, to print out all actions via
- * the llExecFunc() facility.
- * rgerhards, 2007-08-02
- */
-DEFFUNC_llExecFunc(dbgPrintInitInfoAction)
-{
-	DEFiRet;
-	iRet = actionDbgPrint((action_t*) pData);
-	dbgprintf("\n");
-
-	RETiRet;
-}
-
-
 /* print debug information as part of init(). This pretty much
  * outputs the whole config of rsyslogd. I've moved this code
  * out of init() to clean it somewhat up.
@@ -2446,47 +2323,8 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction)
  */
 static void dbgPrintInitInfo(void)
 {
-	selector_t *f;
-	int iSelNbr = 1;
-	int i;
-
-	dbgprintf("\nActive selectors:\n");
-	for (f = Files; f != NULL ; f = f->f_next) {
-		dbgprintf("Selector %d:\n", iSelNbr++);
-		if(f->pCSProgNameComp != NULL)
-			dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp));
-		if(f->eHostnameCmpMode != HN_NO_COMP)
-			dbgprintf("hostname: %s '%s'\n",
-				f->eHostnameCmpMode == HN_COMP_MATCH ?
-					"only" : "allbut",
-				rsCStrGetSzStrNoNULL(f->pCSHostnameComp));
-		if(f->f_filter_type == FILTER_PRI) {
-			for (i = 0; i <= LOG_NFACILITIES; i++)
-				if (f->f_filterData.f_pmask[i] == TABLE_NOPRI)
-					dbgprintf(" X ");
-				else
-					dbgprintf("%2X ", f->f_filterData.f_pmask[i]);
-		} else if(f->f_filter_type == FILTER_EXPR) {
-			dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed");
-		} else {
-			dbgprintf("PROPERTY-BASED Filter:\n");
-			dbgprintf("\tProperty.: '%s'\n",
-			       rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName));
-			dbgprintf("\tOperation: ");
-			if(f->f_filterData.prop.isNegated)
-				dbgprintf("NOT ");
-			dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation));
-			dbgprintf("\tValue....: '%s'\n",
-			       rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue));
-			dbgprintf("\tAction...: ");
-		}
-
-		dbgprintf("\nActions:\n");
-		llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */
-
-		dbgprintf("\n");
-	}
-	dbgprintf("\n");
+	ruleset.DebugPrintAll();
+	DBGPRINTF("\n");
 	if(bDebugPrintTemplateList)
 		tplPrintList();
 	if(bDebugPrintModuleList)
@@ -2496,24 +2334,24 @@ static void dbgPrintInitInfo(void)
 	if(bDebugPrintCfSysLineHandlerList)
 		dbgPrintCfSysLineHandlers();
 
-	dbgprintf("Messages with malicious PTR DNS Records are %sdropped.\n",
+	DBGPRINTF("Messages with malicious PTR DNS Records are %sdropped.\n",
 		  glbl.GetDropMalPTRMsgs() ? "" : "not ");
 
-	dbgprintf("Control characters are %sreplaced upon reception.\n",
+	DBGPRINTF("Control characters are %sreplaced upon reception.\n",
 		  bEscapeCCOnRcv? "" : "not ");
 
 	if(bEscapeCCOnRcv)
-		dbgprintf("Control character escape sequence prefix is '%c'.\n",
+		DBGPRINTF("Control character escape sequence prefix is '%c'.\n",
 			cCCEscapeChar);
 
-	dbgprintf("Main queue size %d messages.\n", iMainMsgQueueSize);
-	dbgprintf("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n",
+	DBGPRINTF("Main queue size %d messages.\n", iMainMsgQueueSize);
+	DBGPRINTF("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n",
 		  iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt);
-	dbgprintf("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n",
+	DBGPRINTF("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n",
 		   iMainMsgQtoQShutdown, iMainMsgQtoActShutdown, iMainMsgQtoEnq);
-	dbgprintf("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n",
+	DBGPRINTF("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n",
 		   iMainMsgQHighWtrMark, iMainMsgQLowWtrMark, iMainMsgQDiscardMark, iMainMsgQDiscardSeverity);
-	dbgprintf("Main queue save on shutdown %d, max disk space allowed %lld\n",
+	DBGPRINTF("Main queue save on shutdown %d, max disk space allowed %lld\n",
 		   bMainMsgQSaveOnShutdown, iMainMsgQueMaxDiskSpace);
 	/* TODO: add
 	iActionRetryCount = 0;
@@ -2524,7 +2362,7 @@ static void dbgPrintInitInfo(void)
 	setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100);
 	setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1);
 	 */
-	dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir());
+	DBGPRINTF("Work Directory: '%s'.\n", glbl.GetWorkDir());
 }
 
 
@@ -2547,7 +2385,7 @@ startInputModules(void)
 			/* activate here */
 			thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun);
 		} else {
-			dbgprintf("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet);
+			DBGPRINTF("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet);
 		}
 	pMod = module.GetNxtType(pMod, eMOD_IN);
 	}
@@ -2565,13 +2403,14 @@ startInputModules(void)
 static rsRetVal
 init(void)
 {
-	DEFiRet;
 	rsRetVal localRet;
 	int iNbrActions;
 	int bHadConfigErr = 0;
+	ruleset_t *pRuleset;
 	char cbuf[BUFSIZ];
 	char bufStartUpMsg[512];
 	struct sigaction sigAct;
+	DEFiRet;
 
 	thrdTerminateAll(); /* stop all running input threads - TODO: reconsider location! */
 
@@ -2580,11 +2419,11 @@ init(void)
 	pDfltProgNameCmp = NULL;
 	eDfltHostnameCmpMode = HN_NO_COMP;
 
-	dbgprintf("rsyslog %s - called init()\n", VERSION);
+	DBGPRINTF("rsyslog %s - called init()\n", VERSION);
 
 	/* delete the message queue, which also flushes all messages left over */
 	if(pMsgQueue != NULL) {
-		dbgprintf("deleting main message queue\n");
+		DBGPRINTF("deleting main message queue\n");
 		qqueueDestruct(&pMsgQueue); /* delete pThis here! */
 		pMsgQueue = NULL;
 	}
@@ -2592,13 +2431,13 @@ init(void)
 	/*  Close all open log files and free log descriptor array. This also frees
 	 *  all output-modules instance data.
 	 */
-	freeSelectors();
+	destructAllActions();
 
 	/* Unload all non-static modules */
-	dbgprintf("Unloading non-static modules.\n");
+	DBGPRINTF("Unloading non-static modules.\n");
 	module.UnloadAndDestructAll(eMOD_LINK_DYNAMIC_LOADED);
 
-	dbgprintf("Clearing templates.\n");
+	DBGPRINTF("Clearing templates.\n");
 	tplDeleteNew();
 
 	/* re-setting values to defaults (where applicable) */
@@ -2613,6 +2452,11 @@ init(void)
 
 	conf.ReInitConf();
 
+	/* construct the default ruleset */
+	ruleset.Construct(&pRuleset);
+	ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset"));
+	ruleset.ConstructFinalize(pRuleset);
+
 	/* open the configuration file */
 	localRet = conf.processConfFile(ConfFile);
 	CHKiRet(conf.GetNbrActActions(&iNbrActions));
@@ -2634,23 +2478,23 @@ init(void)
 		 * We ignore any errors while doing this - we would be lost anyhow...
 		 */
 		errmsg.LogError(0, NO_ERRCODE, "EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file!");
-		selector_t *f = NULL;
 
 		/* note: we previously used _POSIY_TTY_NAME_MAX+1, but this turned out to be
 		 * too low on linux... :-S   -- rgerhards, 2008-07-28
 		 */
 		char szTTYNameBuf[128];
-		conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &f);
-		conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &f);
-		conf.cfline((uchar*)"*.PANIC\t*", &f);
-		conf.cfline((uchar*)"syslog.*\troot", &f);
+		rule_t *pRule = NULL; /* initialization to NULL is *vitally* important! */
+		conf.cfline(UCHAR_CONSTANT("*.ERR\t" _PATH_CONSOLE), &pRule);
+		conf.cfline(UCHAR_CONSTANT("syslog.*\t" _PATH_CONSOLE), &pRule);
+		conf.cfline(UCHAR_CONSTANT("*.PANIC\t*"), &pRule);
+		conf.cfline(UCHAR_CONSTANT("syslog.*\troot"), &pRule);
 		if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) {
 			snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf);
-			conf.cfline((uchar*)cbuf, &f);
+			conf.cfline((uchar*)cbuf, &pRule);
 		} else {
-			dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno);
+			DBGPRINTF("error %d obtaining controlling terminal, not using that emergency rule\n", errno);
 		}
-		selectorAddList(f);
+		ruleset.AddRule(ruleset.GetCurrent(), &pRule);
 	}
 
 	legacyOptsHook();
@@ -2730,6 +2574,7 @@ init(void)
 	setQPROP(qqueueSetiDeqBatchSize, "$MainMsgQueueDequeueBatchSize", iMainMsgQueDeqBatchSize);
 	setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName);
 	setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt);
+	setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles);
 	setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown );
 	setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown);
 	setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown);
@@ -2755,7 +2600,7 @@ init(void)
 	}
 
 	bHaveMainQueue = (MainMsgQueType == QUEUETYPE_DIRECT) ? 0 : 1;
-	dbgprintf("Main processing queue is initialized and running\n");
+	DBGPRINTF("Main processing queue is initialized and running\n");
 
 	/* the output part and the queue is now ready to run. So it is a good time
 	 * to start the inputs. Please note that the net code above should be
@@ -2782,56 +2627,55 @@ init(void)
 	sigAct.sa_handler = sighup_handler;
 	sigaction(SIGHUP, &sigAct, NULL);
 
-	dbgprintf(" (re)started.\n");
+	DBGPRINTF(" (re)started.\n");
 
 finalize_it:
 	RETiRet;
 }
 
 
-/* add a completely-processed selector (after config line parsing) to
- * the linked list of selectors. We now need to check
- * if it has any actions associated and, if so, link it to the linked
- * list. If it has nothing associated with it, we can simply discard
- * it.
- * We have one special case during initialization: then, the current
- * selector is NULL, which means we do not need to care about it at
- * all.  -- rgerhards, 2007-08-01
+/* Switch the default ruleset (that, what servcies bind to if nothing specific
+ * is specified).
+ * rgerhards, 2009-06-12
  */
-rsRetVal
-selectorAddList(selector_t *f)
+static rsRetVal
+setDefaultRuleset(void __attribute__((unused)) *pVal, uchar *pszName)
 {
 	DEFiRet;
-	int iActionCnt;
 
-	static selector_t *nextp = NULL; /* TODO: make this go away (see comment below) */
+	CHKiRet(ruleset.SetDefaultRuleset(pszName));
 
-	if(f != NULL) {
-		CHKiRet(llGetNumElts(&f->llActList, &iActionCnt));
-		if(iActionCnt == 0) {
-			errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded");
-			selectorDestruct(f);
-		} else {
-			/* successfully created an entry */
-			dbgprintf("selector line successfully processed\n");
-			/* TODO: we should use the linked list class for the selector list, else we need to add globals
-			 * ... well nextp could be added temporarily...
-			 * Thanks to varmojfekoj for having the idea to just use "Files" to make this
-			 * code work. I had actually forgotten to fix the code here before moving to 1.18.0.
-			 * And, of course, I also did not migrate the selector_t structure to the linked list class.
-			 * However, that should still be one of the very next things to happen.
-			 * rgerhards, 2007-08-06
-			 */
-			if(Files == NULL) {
-				Files = f;
-			} else {
-				nextp->f_next = f;
-			}
-			nextp = f;
-		}
+finalize_it:
+	free(pszName); /* no longer needed */
+	RETiRet;
+}
+
+
+/* Switch to either an already existing rule set or start a new one. The
+ * named rule set becomes the new "current" rule set (what means that new
+ * actions are added to it).
+ * rgerhards, 2009-06-12
+ */
+static rsRetVal
+setCurrRuleset(void __attribute__((unused)) *pVal, uchar *pszName)
+{
+	ruleset_t *pRuleset;
+	rsRetVal localRet;
+	DEFiRet;
+
+	localRet = ruleset.SetCurrRuleset(pszName);
+
+	if(localRet == RS_RET_NOT_FOUND) {
+		DBGPRINTF("begin new current rule set '%s'\n", pszName);
+		CHKiRet(ruleset.Construct(&pRuleset));
+		CHKiRet(ruleset.SetName(pRuleset, pszName));
+		CHKiRet(ruleset.ConstructFinalize(pRuleset));
+	} else {
+		ABORT_FINALIZE(localRet);
 	}
 
 finalize_it:
+	free(pszName); /* no longer needed */
 	RETiRet;
 }
 
@@ -2845,16 +2689,16 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz
 
 	if (!strcasecmp((char *) pszType, "fixedarray")) {
 		MainMsgQueType = QUEUETYPE_FIXED_ARRAY;
-		dbgprintf("main message queue type set to FIXED_ARRAY\n");
+		DBGPRINTF("main message queue type set to FIXED_ARRAY\n");
 	} else if (!strcasecmp((char *) pszType, "linkedlist")) {
 		MainMsgQueType = QUEUETYPE_LINKEDLIST;
-		dbgprintf("main message queue type set to LINKEDLIST\n");
+		DBGPRINTF("main message queue type set to LINKEDLIST\n");
 	} else if (!strcasecmp((char *) pszType, "disk")) {
 		MainMsgQueType = QUEUETYPE_DISK;
-		dbgprintf("main message queue type set to DISK\n");
+		DBGPRINTF("main message queue type set to DISK\n");
 	} else if (!strcasecmp((char *) pszType, "direct")) {
 		MainMsgQueType = QUEUETYPE_DIRECT;
-		dbgprintf("main message queue type set to DIRECT (no queueing at all)\n");
+		DBGPRINTF("main message queue type set to DIRECT (no queueing at all)\n");
 	} else {
 		errmsg.LogError(0, RS_RET_INVALID_PARAMS, "unknown mainmessagequeuetype parameter: %s", (char *) pszType);
 		iRet = RS_RET_INVALID_PARAMS;
@@ -2923,7 +2767,6 @@ DEFFUNC_llExecFunc(doHUPActions)
 static inline void
 doHUP(void)
 {
-	selector_t *f;
 	char buf[512];
 
 	snprintf(buf, sizeof(buf) / sizeof(char),
@@ -2938,9 +2781,7 @@ doHUP(void)
 		init(); /* main queue is stopped as part of init() */
 	} else {
 		DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n");
-		for(f = Files; f != NULL ; f = f->f_next) {
-			llExecFunc(&f->llActList, doHUPActions, NULL);
-		}
+		ruleset.IterateAllActions(doHUPActions, NULL);
 	}
 }
 
@@ -2971,7 +2812,8 @@ mainloop(void)
 		 * powertop, for example). In that case, we primarily wait for a signal,
 		 * but a once-a-day wakeup should be quite acceptable. -- rgerhards, 2008-06-09
 		 */
-		tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/;
+		//tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/;
+tvSelectTimeout.tv_sec = 5; // TESTING ONLY!!! TODO: change back!!!
 		tvSelectTimeout.tv_usec = 0;
 		select(1, NULL, NULL, NULL, &tvSelectTimeout);
 		if(bFinished)
@@ -3006,49 +2848,11 @@ mainloop(void)
 			bHadHUP = 0;
 			continue;
 		}
+		execScheduled(); /* handle Apc calls (if any) */
 	}
 	ENDfunc
 }
 
-/* If user is not root, prints warnings or even exits 
- * TODO: check all dynafiles for write permission
- * ... but it is probably better to wait here until we have
- * a module interface - rgerhards, 2007-07-23
- */
-static void checkPermissions()
-{
-#if 0
-	/* TODO: this function must either be redone or removed - now with the input modules,
-	 * there is no such simple check we can do. What we can check, however, is if there is
-	 * any input module active and terminate, if not. -- rgerhards, 2007-12-26
-	 */
-	/* we are not root */
-	if (geteuid() != 0)
-	{
-		fputs("WARNING: Local messages will not be logged! If you want to log them, run rsyslog as root.\n",stderr); 
-#ifdef SYSLOG_INET	
-		/* udp enabled and port number less than or equal to 1024 */
-		if ( AcceptRemote && (atoi(LogPort) <= 1024) )
-			fprintf(stderr, "WARNING: Will not listen on UDP port %s. Use port number higher than 1024 or run rsyslog as root!\n", LogPort);
-		
-		/* tcp enabled and port number less or equal to 1024 */
-		if( bEnableTCP   && (atoi(TCPLstnPort) <= 1024) )
-			fprintf(stderr, "WARNING: Will not listen on TCP port %s. Use port number higher than 1024 or run rsyslog as root!\n", TCPLstnPort);
-
-		/* Neither explicit high UDP port nor explicit high TCP port.
-                 * It is useless to run anymore */
-		if( !(AcceptRemote && (atoi(LogPort) > 1024)) && !( bEnableTCP && (atoi(TCPLstnPort) > 1024)) )
-		{
-#endif
-			fprintf(stderr, "ERROR: Nothing to log, no reason to run. Please run rsyslog as root.\n");
-			exit(EXIT_FAILURE);
-#ifdef SYSLOG_INET
-		}
-#endif
-	}
-#endif
-}
-
 
 /* load build-in modules
  * very first version begun on 2007-07-23 by rgerhards
@@ -3057,23 +2861,23 @@ static rsRetVal loadBuildInModules(void)
 {
 	DEFiRet;
 
-	if((iRet = module.doModInit(modInitFile, (uchar*) "builtin-file", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitFile, UCHAR_CONSTANT("builtin-file"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
 #ifdef SYSLOG_INET
-	if((iRet = module.doModInit(modInitFwd, (uchar*) "builtin-fwd", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitFwd, UCHAR_CONSTANT("builtin-fwd"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
 #endif
-	if((iRet = module.doModInit(modInitShell, (uchar*) "builtin-shell", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitShell, UCHAR_CONSTANT("builtin-shell"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
-	if((iRet = module.doModInit(modInitDiscard, (uchar*) "builtin-discard", NULL)) != RS_RET_OK) {
+	if((iRet = module.doModInit(modInitDiscard, UCHAR_CONSTANT("builtin-discard"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
 
 	/* dirty, but this must be for the time being: the usrmsg module must always be
-	 * loaded as last module. This is because it processes any time of action selector.
+	 * loaded as last module. This is because it processes any type of action selector.
 	 * If we load it before other modules, these others will never have a chance of
 	 * working with the config file. We may change that implementation so that a user name
 	 * must start with an alnum, that would definitely help (but would it break backwards
@@ -3081,8 +2885,7 @@ static rsRetVal loadBuildInModules(void)
 	 * User names now must begin with:
 	 *   [a-zA-Z0-9_.]
 	 */
-	if((iRet = module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)) != RS_RET_OK)
-		RETiRet;
+	CHKiRet(module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL));
 
 	/* ok, initialization of the command handler probably does not 100% belong right in
 	 * this space here. However, with the current design, this is actually quite a good
@@ -3092,6 +2895,8 @@ static rsRetVal loadBuildInModules(void)
 	 * This, I think, is the right thing to do. -- rgerhards, 2007-07-31
 	 */
 	CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"defaultruleset", 0, eCmdHdlrGetWord, setDefaultRuleset, NULL, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, setCurrRuleset, NULL, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL));
@@ -3099,6 +2904,7 @@ static rsRetVal loadBuildInModules(void)
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardmark", 0, eCmdHdlrInt, NULL, &iMainMsgQDiscardMark, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardseverity", 0, eCmdHdlrSeverity, NULL, &iMainMsgQDiscardSeverity, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iMainMsgQPersistUpdCnt, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bMainMsgQSyncQeueFiles, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetype", 0, eCmdHdlrGetWord, setMainMsgQueType, NULL, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iMainMsgQueueNumWorkers, NULL));
 	CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iMainMsgQtoQShutdown, NULL));
@@ -3234,7 +3040,7 @@ static rsRetVal mainThread()
 	CHKiRet(init());
 
 	if(Debug && debugging_on) {
-		dbgprintf("Debugging enabled, SIGUSR1 to turn off debugging.\n");
+		DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n");
 	}
 	/* Send a signal to the parent so it can terminate.
 	 */
@@ -3266,7 +3072,7 @@ static rsRetVal mainThread()
 	 * do the init() and then restart things.
 	 * rgerhards, 2005-10-24
 	 */
-	dbgprintf("initialization completed, transitioning to regular run mode\n");
+	DBGPRINTF("initialization completed, transitioning to regular run mode\n");
 
 	mainloop();
 
@@ -3297,14 +3103,14 @@ InitGlobalClasses(void)
 	CHKiRet(objUse(errmsg,   CORE_COMPONENT));
 	pErrObj = "module";
 	CHKiRet(objUse(module,   CORE_COMPONENT));
-	pErrObj = "var";
-	CHKiRet(objUse(var,      CORE_COMPONENT));
 	pErrObj = "datetime";
 	CHKiRet(objUse(datetime, CORE_COMPONENT));
-	pErrObj = "vm";
-	CHKiRet(objUse(vm,       CORE_COMPONENT));
 	pErrObj = "expr";
 	CHKiRet(objUse(expr,     CORE_COMPONENT));
+	pErrObj = "rule";
+	CHKiRet(objUse(rule,     CORE_COMPONENT));
+	pErrObj = "ruleset";
+	CHKiRet(objUse(ruleset,  CORE_COMPONENT));
 	pErrObj = "conf";
 	CHKiRet(objUse(conf,     CORE_COMPONENT));
 
@@ -3348,33 +3154,13 @@ GlobalClassExit(void)
 	/* first, release everything we used ourself */
 	objRelease(net,      LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */
 	objRelease(conf,     CORE_COMPONENT);
+	objRelease(ruleset,  CORE_COMPONENT);
+	objRelease(rule,     CORE_COMPONENT);
 	objRelease(expr,     CORE_COMPONENT);
 	vmClassExit();					/* this is hack, currently core_modules do not get this automatically called */
-	objRelease(vm,       CORE_COMPONENT);
-	objRelease(var,      CORE_COMPONENT);
 	objRelease(datetime, CORE_COMPONENT);
 
 	/* TODO: implement the rest of the deinit */
-#if 0
-	CHKiRet(datetimeClassInit(NULL));
-	CHKiRet(msgClassInit(NULL));
-	CHKiRet(strmClassInit(NULL));
-	CHKiRet(wtiClassInit(NULL));
-	CHKiRet(wtpClassInit(NULL));
-	CHKiRet(qqueueClassInit(NULL));
-	CHKiRet(vmstkClassInit(NULL));
-	CHKiRet(sysvarClassInit(NULL));
-	CHKiRet(vmClassInit(NULL));
-	CHKiRet(vmopClassInit(NULL));
-	CHKiRet(vmprgClassInit(NULL));
-	CHKiRet(ctok_tokenClassInit(NULL));
-	CHKiRet(ctokClassInit(NULL));
-	CHKiRet(exprClassInit(NULL));
-
-	/* dummy "classes" */
-	CHKiRet(actionClassInit());
-	CHKiRet(templateInit());
-#endif
 	/* dummy "classes */
 	strExit();
 
@@ -3468,12 +3254,11 @@ doGlblProcessInit(void)
 	int i;
 	DEFiRet;
 
-	checkPermissions();
 	thrdInit();
 
 	if( !(Debug || NoFork) )
 	{
-		dbgprintf("Checking pidfile.\n");
+		DBGPRINTF("Checking pidfile.\n");
 		if (!check_pid(PidFile))
 		{
 			memset(&sigAct, 0, sizeof (sigAct));
@@ -3509,7 +3294,7 @@ doGlblProcessInit(void)
 	}
 
 	/* tuck my process id away */
-	dbgprintf("Writing pidfile %s.\n", PidFile);
+	DBGPRINTF("Writing pidfile %s.\n", PidFile);
 	if (!check_pid(PidFile))
 	{
 		if (!write_pid(PidFile))
@@ -3655,7 +3440,7 @@ int realMain(int argc, char **argv)
 	if ((argc -= optind))
 		usage();
 
-	dbgprintf("rsyslogd %s startup, compatibility mode %d, module path '%s'\n",
+	DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s'\n",
 		  VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath);
 
 	/* we are done with the initial option parsing and processing. Now we init the system. */
@@ -3737,7 +3522,7 @@ int realMain(int argc, char **argv)
 	/* END core initializations - we now come back to carrying out command line options*/
 
 	while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) {
-		dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg);
+		DBGPRINTF("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg);
 		switch((char)ch) {
                 case '4':
 	                glbl.SetDefPFFamily(PF_INET);
diff --git a/tools/syslogd.h b/tools/syslogd.h
index 8b9bd131..3dfdbe2b 100644
--- a/tools/syslogd.h
+++ b/tools/syslogd.h
@@ -29,66 +29,6 @@
 #include "linkedlist.h"
 #include "expr.h"
 
-
-#ifndef _PATH_CONSOLE
-#define _PATH_CONSOLE	"/dev/console"
-#endif
-
-
-/* This structure represents the files that will have log
- * copies printed.
- * RGerhards 2004-11-08: Each instance of the filed structure 
- * describes what I call an "output channel". This is important
- * to mention as we now allow database connections to be
- * present in the filed structure. If helps immensely, if we
- * think of it as the abstraction of an output channel.
- * rgerhards, 2005-10-26: The structure below provides ample
- * opportunity for non-thread-safety. Each of the variable
- * accesses must be carefully evaluated, many of them probably
- * be guarded by mutexes. But beware of deadlocks...
- * rgerhards, 2007-08-01: as you can see, the structure has shrunk pretty much. I will
- * remove some of the comments some time. It's still the structure that controls much
- * of the processing that goes on in syslogd, but it now has lots of helpers.
- */
-struct filed {
-	struct	filed *f_next;		/* next in linked list */
-	/* filter properties */
-	enum {
-		FILTER_PRI = 0,		/* traditional PRI based filer */
-		FILTER_PROP = 1,	/* extended filter, property based */
-		FILTER_EXPR = 2		/* extended filter, expression based */
-	} f_filter_type;
-	EHostnameCmpMode eHostnameCmpMode;
-	cstr_t *pCSHostnameComp;	/* hostname to check */
-	cstr_t *pCSProgNameComp;	/* tag to check or NULL, if not to be checked */
-	union {
-		u_char	f_pmask[LOG_NFACILITIES+1];	/* priority mask */
-		struct {
-			cstr_t *pCSPropName;
-			enum {
-				FIOP_NOP = 0,		/* do not use - No Operation */
-				FIOP_CONTAINS  = 1,	/* contains string? */
-				FIOP_ISEQUAL  = 2,	/* is (exactly) equal? */
-				FIOP_STARTSWITH = 3,	/* starts with a string? */
- 				FIOP_REGEX = 4,		/* matches a (BRE) regular expression? */
- 				FIOP_EREREGEX = 5	/* matches a ERE regular expression? */
-			} operation;
-			regex_t *regex_cache;		/* cache for compiled REs, if such are used */
-			cstr_t *pCSCompValue;	/* value to "compare" against */
-			char isNegated;			/* actually a boolean ;) */
-		} prop;
-		expr_t *f_expr;				/* expression object */
-	} f_filterData;
-
-	linkedList_t llActList;	/* list of configured actions */
-};
-
-
-#include "net.h" /* TODO: remove when you remoe isAllowedSender from here! */
-void untty(void);
-rsRetVal selectorConstruct(selector_t **ppThis);
-rsRetVal selectorDestruct(void *pVal);
-rsRetVal selectorAddList(selector_t *f);
 /* the following prototypes should go away once we have an input
  * module interface -- rgerhards, 2007-12-12
  */
diff --git a/tools/zpipe.c b/tools/zpipe.c
new file mode 100644
index 00000000..bde6c5c1
--- /dev/null
+++ b/tools/zpipe.c
@@ -0,0 +1,254 @@
+/* zpipe.c: example of proper use of zlib's inflate() and deflate()
+   Not copyrighted -- provided to the public domain
+   Version 1.5  11 December 2005  Mark Adler 
+   Version 2.0  03 June     2009  Rainer Gerhards */
+
+/* RSYSLOG NOTE:
+ * This file is beeing distributed as part of rsyslog, but is just an
+ * add-on. Most importantly, rsyslog's copyright does not apply but
+ * rather the (non-) copyright stated above.
+ */
+
+/* Version history:
+   1.0  30 Oct 2004  First version
+   1.1   8 Nov 2004  Add void casting for unused return values
+                     Use switch statement for inflate() return values
+   1.2   9 Nov 2004  Add assertions to document zlib guarantees
+   1.3   6 Apr 2005  Remove incorrect assertion in inf()
+   1.4  11 Dec 2005  Add hack to avoid MSDOS end-of-line conversions
+                     Avoid some compiler warnings for input and output buffers
+   2.0  03 Jun 2009  Add hack to support multiple deflate records inside a single
+   		     file on inflate. This is needed in order to support reading
+		     files created by rsyslog's zip output writer.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include "zlib.h"
+
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>
+#  include <io.h>
+#  define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+#else
+#  define SET_BINARY_MODE(file)
+#endif
+
+#define CHUNK 16384
+
+/* Compress from file source to file dest until EOF on source.
+   def() returns Z_OK on success, Z_MEM_ERROR if memory could not be
+   allocated for processing, Z_STREAM_ERROR if an invalid compression
+   level is supplied, Z_VERSION_ERROR if the version of zlib.h and the
+   version of the library linked do not match, or Z_ERRNO if there is
+   an error reading or writing the files. */
+int def(FILE *source, FILE *dest, int level)
+{
+    int ret, flush;
+    unsigned have;
+    z_stream strm;
+    unsigned char in[CHUNK];
+    unsigned char out[CHUNK];
+
+    /* allocate deflate state */
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    ret = deflateInit(&strm, level);
+    if (ret != Z_OK)
+        return ret;
+
+    /* compress until end of file */
+    do {
+        strm.avail_in = fread(in, 1, CHUNK, source);
+        if (ferror(source)) {
+            (void)deflateEnd(&strm);
+            return Z_ERRNO;
+        }
+        flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
+        strm.next_in = in;
+
+        /* run deflate() on input until output buffer not full, finish
+           compression if all of source has been read in */
+        do {
+            strm.avail_out = CHUNK;
+            strm.next_out = out;
+            ret = deflate(&strm, flush);    /* no bad return value */
+            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
+            have = CHUNK - strm.avail_out;
+            if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+                (void)deflateEnd(&strm);
+                return Z_ERRNO;
+            }
+        } while (strm.avail_out == 0);
+        assert(strm.avail_in == 0);     /* all input will be used */
+
+        /* done when last data in file processed */
+    } while (flush != Z_FINISH);
+    assert(ret == Z_STREAM_END);        /* stream will be complete */
+
+    /* clean up and return */
+    (void)deflateEnd(&strm);
+    return Z_OK;
+}
+
+
+/* initialize stream for deflating (we need this in case of 
+ * multiple records.
+ * rgerhards, 2009-06-03
+ */
+int doInflateInit(z_stream *strm)
+{
+    int ret;
+
+    /* allocate inflate state */
+    strm->zalloc = Z_NULL;
+    strm->zfree = Z_NULL;
+    strm->opaque = Z_NULL;
+    strm->avail_in = 0;
+    strm->next_in = Z_NULL;
+    ret = inflateInit(strm);
+    return ret;
+}
+
+
+/* Decompress from file source to file dest until stream ends or EOF.
+   inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be
+   allocated for processing, Z_DATA_ERROR if the deflate data is
+   invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and
+   the version of the library linked do not match, or Z_ERRNO if there
+   is an error reading or writing the files. */
+int inf(FILE *source, FILE *dest)
+{
+    int ret;
+    unsigned have;
+    z_stream strm;
+    unsigned char in[CHUNK];
+    int len;
+    unsigned char *next_in_save;
+    unsigned char out[CHUNK];
+
+    ret = doInflateInit(&strm);
+    if (ret != Z_OK)
+        return ret;
+
+    /* decompress until deflate stream ends or end of file */
+    do {
+        len = fread(in, 1, CHUNK, source);
+        if (ferror(source)) {
+            (void)inflateEnd(&strm);
+            return Z_ERRNO;
+        }
+        if (len == 0) {
+            break;
+	}
+        strm.avail_in = len;
+        strm.next_in = in;
+
+        /* run inflate() on input until output buffer not full */
+        strm.avail_out = CHUNK;
+        strm.next_out = out;
+        do {
+	/*	fprintf(stderr, "---inner LOOP---, avail_in %d, avail_out %d Byte 0: %x, 1: %x\n", strm.avail_in, strm.avail_out, *strm.next_in, *(strm.next_in+1));*/
+		do {
+		    ret = inflate(&strm, Z_NO_FLUSH);
+		    assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
+		    switch (ret) {
+		    case Z_NEED_DICT:
+			ret = Z_DATA_ERROR;     /* and fall through */
+		    case Z_DATA_ERROR:
+		    case Z_MEM_ERROR:
+			(void)inflateEnd(&strm);
+			return ret;
+		    }
+		    have = CHUNK - strm.avail_out;
+		    if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
+			(void)inflateEnd(&strm);
+			return Z_ERRNO;
+		    }
+		} while (strm.avail_out == 0);
+		/* handle the case that more than one deflate record is contained
+		 * in a single file. -- rgerhards, 2009-06-03
+		 */
+		if(ret == Z_STREAM_END) {
+			len -= strm.total_in;
+			if(len > 0) {
+				next_in_save = strm.next_in;
+				(void)inflateEnd(&strm);
+				ret = doInflateInit(&strm);
+				if (ret != Z_OK)
+				    return ret;
+				strm.avail_in = len;
+				strm.next_in = next_in_save;
+				strm.avail_out = CHUNK;
+				strm.next_out = out;
+				ret = Z_OK; /* continue outer loop */
+			}
+		}
+	} while (strm.avail_in > 0);
+
+        /* done when inflate() says it's done */
+    } while (ret != Z_STREAM_END);
+
+    /* clean up and return */
+    (void)inflateEnd(&strm);
+    return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+}
+
+/* report a zlib or i/o error */
+void zerr(int ret)
+{
+    fputs("zpipe: ", stdout);
+    switch (ret) {
+    case Z_ERRNO:
+        if (ferror(stdin))
+            fputs("error reading stdin\n", stdout);
+        if (ferror(stdout))
+            fputs("error writing stdout\n", stdout);
+        break;
+    case Z_STREAM_ERROR:
+        fputs("invalid compression level\n", stdout);
+        break;
+    case Z_DATA_ERROR:
+        fputs("invalid or incomplete deflate data\n", stdout);
+        break;
+    case Z_MEM_ERROR:
+        fputs("out of memory\n", stdout);
+        break;
+    case Z_VERSION_ERROR:
+        fputs("zlib version mismatch!\n", stdout);
+    }
+}
+
+/* compress or decompress from stdin to stdout */
+int main(int argc, char **argv)
+{
+    int ret;
+
+    /* avoid end-of-line conversions */
+    SET_BINARY_MODE(stdin);
+    SET_BINARY_MODE(stdout);
+
+    /* do compression if no arguments */
+    if (argc == 1) {
+        ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION);
+        if (ret != Z_OK)
+            zerr(ret);
+        return ret;
+    }
+
+    /* do decompression if -d specified */
+    else if (argc == 2 && strcmp(argv[1], "-d") == 0) {
+        ret = inf(stdin, stdout);
+        if (ret != Z_OK)
+            zerr(ret);
+        return ret;
+    }
+
+    /* otherwise, report usage */
+    else {
+        fputs("zpipe usage: zpipe [-d] < source > dest\n", stdout);
+        return 1;
+    }
+}