Merge branch 'v4-stable-solaris' into v4-devel

Conflicts:
	ChangeLog
	configure.ac
	doc/manual.html
	tools/omfile.c
	tools/syslogd.c

12 files changed, 431 insertions, 78 deletions

diff --git a/tools/Makefile.am b/tools/Makefile.am
index f0f9afab..1497d3be 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -13,6 +13,8 @@ rsyslogd_SOURCES = \
 	omfwd.h \
 	omfile.c \
 	omfile.h \
+	ompipe.c \
+	ompipe.h \
 	omdiscard.c \
 	omdiscard.h \
 	iminternal.c \
diff --git a/tools/msggen.c b/tools/msggen.c
index 06244c18..29ade3a7 100644
--- a/tools/msggen.c
+++ b/tools/msggen.c
@@ -21,6 +21,7 @@
  * A copy of the GPL can be found in the file "COPYING" in this distribution.
  */
 
+#include "config.h"
 #include <stdio.h>
 #include <syslog.h>
 
diff --git a/tools/omfile.c b/tools/omfile.c
index efdaead4..74b22019 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -5,10 +5,6 @@
  *       works!
  *
  * File begun on 2007-07-21 by RGerhards (extracted from syslogd.c)
- * This file is under development and has not yet arrived at being fully
- * self-contained and a real object. So far, it is mostly an excerpt
- * of the "old" message code without any modifications. However, it
- * helps to have things at the right place one we go to the meat of it.
  *
  * A large re-write of this file was done in June, 2009. The focus was
  * to introduce many more features (like zipped writing), clean up the code
@@ -16,6 +12,10 @@
  * solid basis for the next three to five years to come. During it, bugs
  * may have been introduced ;) -- rgerhards, 2009-06-04
  *
+ * Note that as of 2010-02-28 this module does no longer handle
+ * pipes. These have been moved to ompipe, to reduced the entanglement
+ * between the two different functionalities. -- rgerhards
+ *
  * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
@@ -79,13 +79,15 @@ DEFobjCurrIf(strm)
 struct s_dynaFileCacheEntry {
 	uchar *pName;		/* name currently open, if dynamic name */
 	strm_t	*pStrm;		/* our output stream */
-	time_t	lastUsed;	/* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) 
+	time_t	lastUsed;	/* for LRU - last access */
 };
 typedef struct s_dynaFileCacheEntry dynaFileCacheEntry;
 
 
 #define IOBUF_DFLT_SIZE 1024	/* default size for io buffers */
 #define FLUSH_INTRVL_DFLT 1 	/* default buffer flush interval (in seconds) */
+#define USE_ASYNCWRITER_DFLT 0 	/* default buffer use async writer */
+#define FLUSHONTX_DFLT 1 	/* default for flush on TX end */
 
 #define DFLT_bForceChown 0
 /* globals for default values */
@@ -101,10 +103,11 @@ static uid_t	dirGID;		/* GID to be used for newly created directories */
 static int	bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */
 static int	bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */
 static int	iZipLevel = 0;	/* zip compression mode (0..9 as usual) */
-static bool	bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */
+static bool	bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */
 static int64	iIOBufSize = IOBUF_DFLT_SIZE;	/* size of an io buffer */
 static int	iFlushInterval = FLUSH_INTRVL_DFLT; 	/* how often flush the output buffer on inactivity? */
-static uchar	*pszTplName = NULL; /* name of the default template to use */
+static int	bUseAsyncWriter = USE_ASYNCWRITER_DFLT;	/* should we enable asynchronous writing? */
+uchar	*pszFileDfltTplName = NULL; /* name of the default template to use */
 /* end globals for default values */
 
 
@@ -137,6 +140,7 @@ typedef struct _instanceData {
 	int	iIOBufSize;		/* size of associated io buffer */
 	int	iFlushInterval;		/* how fast flush buffer on inactivity? */
 	bool	bFlushOnTXEnd;		/* flush write buffers when transaction has ended? */
+	bool	bUseAsyncWriter;	/* use async stream writer? */
 } instanceData;
 
 
@@ -150,28 +154,24 @@ ENDisCompatibleWithFeature
 BEGINdbgPrintInstInfo
 CODESTARTdbgPrintInstInfo
 	if(pData->bDynamicName) {
-		dbgprintf("[dynamic]\n\ttemplate='%s'"
-		       "\tfile cache size=%d\n"
-		       "\tcreate directories: %s\n"
-		       "\tfile owner %d, group %d\n"
-		       "\tdirectory owner %d, group %d\n"
-		       "\tforce chown() for all files: %s\n"
-		       "\tdir create mode 0%3.3o, file create mode 0%3.3o\n"
-		       "\tfail if owner/group can not be set: %s\n",
-		        pData->f_fname,
-			pData->iDynaFileCacheSize,
-			pData->bCreateDirs ? "yes" : "no",
-			pData->fileUID, pData->fileGID,
-			pData->dirUID, pData->dirGID,
-			pData->bForceChown ? "yes" : "no",
-			pData->fDirCreateMode, pData->fCreateMode,
-			pData->bFailOnChown ? "yes" : "no"
-			);
+		dbgprintf("[dynamic]\n");
 	} else { /* regular file */
-		dbgprintf("%s", pData->f_fname);
-		if (pData->pStrm == NULL)
-			dbgprintf(" (unused)");
+		dbgprintf("%s%s\n", pData->f_fname,
+			  (pData->pStrm == NULL) ? " (unused)" : "");
 	}
+
+	dbgprintf("\ttemplate='%s'\n", pData->f_fname);
+	dbgprintf("\tuse async writer=%d\n", pData->bUseAsyncWriter);
+	dbgprintf("\tflush on TX end=%d\n", pData->bFlushOnTXEnd);
+	dbgprintf("\tflush interval=%d\n", pData->iFlushInterval);
+	dbgprintf("\tfile cache size=%d\n", pData->iDynaFileCacheSize);
+	dbgprintf("\tcreate directories: %s\n", pData->bCreateDirs ? "yes" : "no");
+	dbgprintf("\tfile owner %d, group %d\n", pData->fileUID, pData->fileGID);
+	dbgprintf("\tforce chown() for all files: %s\n", pData->bForceChown ? "yes" : "no"); 
+	dbgprintf("\tdirectory owner %d, group %d\n", pData->dirUID, pData->dirGID);
+	dbgprintf("\tdir create mode 0%3.3o, file create mode 0%3.3o\n",
+		  pData->fDirCreateMode, pData->fCreateMode);
+	dbgprintf("\tfail if owner/group can not be set: %s\n", pData->bFailOnChown ? "yes" : "no");
 ENDdbgPrintInstInfo
 
 
@@ -263,7 +263,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR
 	pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit;
 
 	iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts,
-				       (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName);
+				       (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName);
 
 finalize_it:
 	RETiRet;
@@ -288,17 +288,18 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry)
 
 	DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry,
 		pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName);
-	/* if the name is NULL, this is an improperly initilized entry which
-	 * needs to be discarded. In this case, neither the file is to be closed
-	 * not the name to be freed.
-	 */
+
 	if(pCache[iEntry]->pName != NULL) {
-		if(pCache[iEntry]->pStrm != NULL)
-			strm.Destruct(&pCache[iEntry]->pStrm);
 		d_free(pCache[iEntry]->pName);
 		pCache[iEntry]->pName = NULL;
 	}
 
+	if(pCache[iEntry]->pStrm != NULL) {
+		strm.Destruct(&pCache[iEntry]->pStrm);
+		if(pCache[iEntry]->pStrm != NULL) /* safety check -- TODO: remove if no longer necessary */
+			abort();
+	}
+
 	if(bFreeEntry) {
 		d_free(pCache[iEntry]);
 		pCache[iEntry] = NULL;
@@ -323,6 +324,7 @@ dynaFileFreeCacheEntries(instanceData *pData)
 	for(i = 0 ; i < pData->iCurrCacheSize ; ++i) {
 		dynaFileDelCacheEntry(pData->dynCache, i, 1);
 	}
+	pData->iCurrElt = -1; /* invalidate current element */
 	ENDfunc;
 }
 
@@ -432,7 +434,7 @@ prepareFile(instanceData *pData, uchar *newFileName)
 	 * async processing, which is a real performance waste if we do not do buffered
 	 * writes! -- rgerhards, 2009-07-06
 	 */
-	if(!pData->bFlushOnTXEnd)
+	if(pData->bUseAsyncWriter)
 		CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval));
 	if(pData->pszSizeLimitCmd != NULL)
 		CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd)));
@@ -486,11 +488,11 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
 	iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */
 	ttOldest = time(NULL) + 1; /* there must always be an older one */
 	for(i = 0 ; i < pData->iCurrCacheSize ; ++i) {
-		if(pCache[i] == NULL) {
+		if(pCache[i] == NULL || pCache[i]->pName == NULL) {
 			if(iFirstFree == -1)
 				iFirstFree = i;
 		} else { /* got an element, let's see if it matches */
-			if(!ustrcmp(newFileName, pCache[i]->pName)) {
+			if(!ustrcmp(newFileName, pCache[i]->pName)) {  // RG:  name == NULL?
 				/* we found our element! */
 				pData->pStrm = pCache[i]->pStrm;
 				pData->iCurrElt = i;
@@ -506,23 +508,45 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
 	}
 
 	/* we have not found an entry */
+
+	/* invalidate iCurrElt as we may error-exit out of this function when the currrent
+	 * iCurrElt has been freed or otherwise become unusable. This is a precaution, and
+	 * performance-wise it may be better to do that in each of the exits. However, that
+	 * is error-prone, so I prefer to do it here. -- rgerhards, 2010-03-02
+	 */
+	pData->iCurrElt = -1;
+	/* similarly, we need to set the current pStrm to NULL, because otherwise, if prepareFile() fails,
+	 * we may end up using an old stream. This bug depends on how exactly prepareFile fails,
+	 * but it* could be triggered in the common case of a failed open() system call.
+	 * rgerhards, 2010-03-22
+	 */
+	pData->pStrm = NULL;
+
 	if(iFirstFree == -1 && (pData->iCurrCacheSize < pData->iDynaFileCacheSize)) {
 		/* there is space left, so set it to that index */
 		iFirstFree = pData->iCurrCacheSize++;
 	}
 
+// RG: this is the begin of a potential problem area
+	/* Note that the following code sequence does not work with the cache entry itself,
+	 * but rather with pData->pStrm, the (sole) stream pointer in the non-dynafile case.
+	 * The cache array is only updated after the open was successful. -- rgerhards, 2010-03-21
+	 */
 	if(iFirstFree == -1) {
 		dynaFileDelCacheEntry(pCache, iOldest, 0);
 		iFirstFree = iOldest; /* this one *is* now free ;) */
 	} else {
 		/* we need to allocate memory for the cache structure */
+		/* TODO: performance note: we could alloc all entries on startup, thus saving malloc
+		 *       overhead -- this may be something to consider in v5...
+		 */
 		CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)));
 	}
 
 	/* Ok, we finally can open the file */
 	localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */
 
-	/* file is either open now or an error state set */
+	/* file is either open now or an error state set */ // RG: better check localRet?
 	if(pData->pStrm == NULL) {
 		/* do not report anything if the message is an internally-generated
 		 * message. Otherwise, we could run into a never-ending loop. The bad
@@ -533,13 +557,15 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts)
 		} else {
 			errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName);
 		}
-		dynaFileDelCacheEntry(pCache, iFirstFree, 1);
 		ABORT_FINALIZE(localRet);
 	}
 
-	CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName));
+	if((pCache[iFirstFree]->pName = ustrdup(newFileName)) == NULL) {
+		strm.Destruct(&pData->pStrm); /* need to free failed entry! */
+		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
+	}
 	pCache[iFirstFree]->pStrm = pData->pStrm;
-	pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance
+	pCache[iFirstFree]->lastUsed = time(NULL);
 	pData->iCurrElt = iFirstFree;
 	DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName);
 
@@ -640,7 +666,7 @@ ENDdoAction
 
 BEGINparseSelectorAct
 CODESTARTparseSelectorAct
-	if(!(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-'))
+	if(!(*p == '$' || *p == '?' || *p == '/' || *p == '.' || *p == '-'))
 		ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED);
 
 	CHKiRet(createInstance(&pData));
@@ -673,7 +699,7 @@ CODESTARTparseSelectorAct
 		CODE_STD_STRING_REQUESTparseSelectorAct(2)
 		++p; /* eat '?' */
 		CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName));
+				               (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName));
 		/* "filename" is actually a template name, we need this as string 1. So let's add it
 		 * to the pOMSR. -- rgerhards, 2007-07-27
 		 */
@@ -686,8 +712,11 @@ CODESTARTparseSelectorAct
 				calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*)));
 		break;
 
-        case '|':
+        /* case '|': while pipe support has been removed, I leave the code in in case we 
+	 *           need high-performance pipes at a later stage (unlikely). -- rgerhards, 2010-02-28
+	 */
 	case '/':
+	case '.':
 		CODE_STD_STRING_REQUESTparseSelectorAct(1)
 		/* we now have *almost* the same semantics for files and pipes, but we still need
 		 * to know we deal with a pipe, because we must do non-blocking opens in that case
@@ -701,7 +730,7 @@ CODESTARTparseSelectorAct
 		}
 
 		CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
-				               (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName));
+				               (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName));
 		pData->bDynamicName = 0;
 		break;
 	default:
@@ -723,6 +752,7 @@ CODESTARTparseSelectorAct
 	pData->bFlushOnTXEnd = bFlushOnTXEnd;
 	pData->iIOBufSize = (int) iIOBufSize;
 	pData->iFlushInterval = iFlushInterval;
+	pData->bUseAsyncWriter = bUseAsyncWriter;
 
 	if(pData->bDynamicName == 0) {
 		/* try open and emit error message if not possible. At this stage, we ignore the
@@ -756,12 +786,13 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a
 	bCreateDirs = 1;
 	bEnableSync = 0;
 	iZipLevel = 0;
-	bFlushOnTXEnd = 1;
+	bFlushOnTXEnd = FLUSHONTX_DFLT;
 	iIOBufSize = IOBUF_DFLT_SIZE;
 	iFlushInterval = FLUSH_INTRVL_DFLT;
-	if(pszTplName != NULL) {
-		free(pszTplName);
-		pszTplName = NULL;
+	bUseAsyncWriter = USE_ASYNCWRITER_DFLT;
+	if(pszFileDfltTplName != NULL) {
+		free(pszFileDfltTplName);
+		pszFileDfltTplName = NULL;
 	}
 
 	return RS_RET_OK;
@@ -772,7 +803,6 @@ BEGINdoHUP
 CODESTARTdoHUP
 	if(pData->bDynamicName) {
 		dynaFileFreeCacheEntries(pData);
-		pData->iCurrElt = -1; /* invalidate current element */
 	} else {
 		if(pData->pStrm != NULL) {
 			strm.Destruct(&pData->pStrm);
@@ -786,7 +816,7 @@ BEGINmodExit
 CODESTARTmodExit
 	objRelease(errmsg, CORE_COMPONENT);
 	objRelease(strm, CORE_COMPONENT);
-	free(pszTplName);
+	free(pszFileDfltTplName);
 ENDmodExit
 
 
@@ -806,6 +836,7 @@ CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID));
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileasyncwriting", 0, eCmdHdlrBinary, NULL, &bUseAsyncWriter, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID));
@@ -818,7 +849,7 @@ CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"failonchownfailure", 0, eCmdHdlrBinary, NULL, &bFailOnChown, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileForceChown", 0, eCmdHdlrBinary, NULL, &bForceChown, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionfileenablesync", 0, eCmdHdlrBinary, NULL, &bEnableSync, STD_LOADABLE_MODULE_ID));
-	CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL));
+	CHKiRet(regCfSysLineHdlr((uchar *)"actionfiledefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszFileDfltTplName, NULL));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID));
 ENDmodInit
 /* vi:set ai:
diff --git a/tools/omfile.h b/tools/omfile.h
index 03e081f3..8dca6a88 100644
--- a/tools/omfile.h
+++ b/tools/omfile.h
@@ -3,7 +3,7 @@
  *
  * File begun on 2007-07-21 by RGerhards (extracted from syslogd.c)
  *
- * Copyright 2007 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -28,7 +28,11 @@
 /* prototypes */
 rsRetVal modInitFile(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*);
 
+/* the define below is dirty, but we need it for ompipe integration. There is no
+ * other way to have the functionality (well, one way would be to go through the
+ * globals, but that seems not yet justified. -- rgerhards, 2010-03-01
+ */
+uchar	*pszFileDfltTplName;
 #endif /* #ifndef OMFILE_H_INCLUDED */
-/*
- * vi:set ai:
+/* vi:set ai:
  */
diff --git a/tools/omfwd.c b/tools/omfwd.c
index 02f19eac..cbfc36a4 100644
--- a/tools/omfwd.c
+++ b/tools/omfwd.c
@@ -411,7 +411,7 @@ CODESTARTtryResume
 ENDtryResume
 
 BEGINdoAction
-	char *psz; /* temporary buffering */
+	char *psz = NULL; /* temporary buffering */
 	register unsigned l;
 	int iMaxLine;
 CODESTARTdoAction
@@ -458,11 +458,14 @@ CODESTARTdoAction
 			 * rgerhards, 2006-11-30
 			 */
 			dbgprintf("Compression failed, sending uncompressed message\n");
+			free(out);
 		} else if(destLen+1 < l) {
 			/* only use compression if there is a gain in using it! */
 			dbgprintf("there is gain in compression, so we do it\n");
 			psz = (char*) out;
 			l = destLen + 1; /* take care for the "z" at message start! */
+		} else {
+			free(out);
 		}
 		++destLen;
 	}
@@ -484,7 +487,7 @@ CODESTARTdoAction
 	}
 finalize_it:
 #	ifdef USE_NETZIP
-	if(psz != (char*) ppString[0])  {
+	if((psz != NULL) && (psz != (char*) ppString[0]))  {
 		/* we need to free temporary buffer, alloced above - Naoya Nakazawa, 2010-01-11 */
 		free(psz);
 	}
diff --git a/tools/ompipe.c b/tools/ompipe.c
new file mode 100644
index 00000000..7cf61822
--- /dev/null
+++ b/tools/ompipe.c
@@ -0,0 +1,240 @@
+/* ompipe.c
+ * This is the implementation of the build-in pipe output module.
+ * Note that this module stems back to the "old" (4.4.2 and below)
+ * omfile. There were some issues with the new omfile code and pipes
+ * (namely in regard to xconsole), so we took out the pipe code and moved
+ * that to a separate module. That a) immediately solves the issue for a
+ * less common use case and probably makes it much easier to enhance
+ * file and pipe support (now independently) in the future (we always
+ * needed to think about pipes in omfile so far, what we now no longer
+ * need to, hopefully resulting in reduction of complexity).
+ *
+ * NOTE: read comments in module-template.h to understand how this pipe
+ *       works!
+ *
+ * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/file.h>
+
+#include "syslogd.h"
+#include "syslogd-types.h"
+#include "srUtils.h"
+#include "template.h"
+#include "ompipe.h"
+#include "omfile.h" /* for dirty trick: access to $ActionFileDefaultTemplate value */
+#include "cfsysline.h"
+#include "module-template.h"
+#include "conf.h"
+#include "errmsg.h"
+
+MODULE_TYPE_OUTPUT
+
+/* internal structures
+ */
+DEF_OMOD_STATIC_DATA
+DEFobjCurrIf(errmsg)
+
+
+/* globals for default values */
+/* end globals for default values */
+
+
+typedef struct _instanceData {
+	uchar	f_fname[MAXFNAME];/* pipe or template name (display only) */
+	short	fd;		  /* pipe descriptor for (current) pipe */
+} instanceData;
+
+
+BEGINisCompatibleWithFeature
+CODESTARTisCompatibleWithFeature
+	if(eFeat == sFEATURERepeatedMsgReduction)
+		iRet = RS_RET_OK;
+ENDisCompatibleWithFeature
+
+
+BEGINdbgPrintInstInfo
+CODESTARTdbgPrintInstInfo
+	dbgprintf("pipe %s", pData->f_fname);
+	if (pData->fd == -1)
+		dbgprintf(" (unused)");
+ENDdbgPrintInstInfo
+
+
+/* This is now shared code for all types of files. It simply prepares
+ * pipe access, which, among others, means the the pipe wil be opened
+ * and any directories in between will be created (based on config, of
+ * course). -- rgerhards, 2008-10-22
+ * changed to iRet interface - 2009-03-19
+ */
+static inline rsRetVal
+preparePipe(instanceData *pData)
+{
+	DEFiRet;
+	pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC);
+	RETiRet;
+}
+
+
+/* rgerhards 2004-11-11: write to a pipe output. This
+ * will be called for all outputs using pipe semantics,
+ * for example also for pipes.
+ */
+static rsRetVal writePipe(uchar **ppString, instanceData *pData)
+{
+	int iLenWritten;
+	DEFiRet;
+
+	ASSERT(pData != NULL);
+
+	if(pData->fd == -1) {
+		rsRetVal iRetLocal;
+		iRetLocal = preparePipe(pData);
+		if((iRetLocal != RS_RET_OK) || (pData->fd == -1))
+			ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */
+	}
+
+	/* create the message based on format specified */
+	iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0]));
+	if(iLenWritten < 0) {
+		int e = errno;
+		char errStr[1024];
+		rs_strerror_r(errno, errStr, sizeof(errStr));
+		DBGPRINTF("pipe (%d) write error %d: %s\n", pData->fd, e, errStr);
+
+		/* If a named pipe is full, we suspend this action for a while */
+		if(e == EAGAIN)
+			ABORT_FINALIZE(RS_RET_SUSPENDED);
+
+		close(pData->fd);
+		pData->fd = -1; /* tell that fd is no longer open! */
+		iRet = RS_RET_SUSPENDED;
+		errno = e;
+		errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname);
+	}
+
+finalize_it:
+	RETiRet;
+}
+
+
+BEGINcreateInstance
+CODESTARTcreateInstance
+	pData->fd = -1;
+ENDcreateInstance
+
+
+BEGINfreeInstance
+CODESTARTfreeInstance
+	if(pData->fd != -1)
+		close(pData->fd);
+ENDfreeInstance
+
+
+BEGINtryResume
+CODESTARTtryResume
+ENDtryResume
+
+BEGINdoAction
+CODESTARTdoAction
+	DBGPRINTF(" (%s)\n", pData->f_fname);
+	iRet = writePipe(ppString, pData);
+ENDdoAction
+
+
+BEGINparseSelectorAct
+CODESTARTparseSelectorAct
+	/* yes, the if below is redundant, but I need it now. Will go away as
+	 * the code further changes.  -- rgerhards, 2007-07-25
+	 */
+	if(*p == '|') {
+		if((iRet = createInstance(&pData)) != RS_RET_OK) {
+			ENDfunc
+			return iRet; /* this can not use RET_iRet! */
+		}
+	} else {
+		/* this is not clean, but we need it for the time being
+		 * TODO: remove when cleaning up modularization 
+		 */
+		ENDfunc
+		return RS_RET_CONFLINE_UNPROCESSED;
+	}
+
+	CODE_STD_STRING_REQUESTparseSelectorAct(1)
+	++p;
+	/* rgerhards 2004-11-17: from now, we need to have different
+	 * processing, because after the first comma, the template name
+	 * to use is specified. So we need to scan for the first coma first
+	 * and then look at the rest of the line.
+	 */
+	CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS,
+				       (pszFileDfltTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszFileDfltTplName));
+
+	/* at this stage, we ignore the return value of preparePipe, this is taken
+	 * care of in later steps. -- rgerhards, 2009-03-19
+	 */
+	preparePipe(pData);
+		
+	if(pData->fd < 0 ) {
+		pData->fd = -1;
+		DBGPRINTF("Error opening log pipe: %s\n", pData->f_fname);
+		errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output pipe '%s'", pData->f_fname);
+	}
+CODE_STD_FINALIZERparseSelectorAct
+ENDparseSelectorAct
+
+
+BEGINdoHUP
+CODESTARTdoHUP
+	if(pData->fd != -1) {
+		close(pData->fd);
+		pData->fd = -1;
+	}
+ENDdoHUP
+
+
+BEGINmodExit
+CODESTARTmodExit
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_OMOD_QUERIES
+CODEqueryEtryPt_doHUP
+ENDqueryEtryPt
+
+
+BEGINmodInit(Pipe)
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
+ENDmodInit
+/* vi:set ai:
+ */
diff --git a/tools/ompipe.h b/tools/ompipe.h
new file mode 100644
index 00000000..d17346c5
--- /dev/null
+++ b/tools/ompipe.h
@@ -0,0 +1,31 @@
+/* ompipe.h
+ * These are the definitions for the build-in pipe output module.
+ *
+ * Copyright 2007-2010 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This pipe is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the pipe "COPYING" in this distribution.
+ */
+#ifndef	OMPIPE_H_INCLUDED
+#define	OMPIPE_H_INCLUDED 1
+
+/* prototypes */
+rsRetVal modInitPipe(int iIFVersRequested __attribute__((unused)), int *ipIFVersProvided, rsRetVal (**pQueryEtryPt)(), rsRetVal (*pHostQueryEtryPt)(uchar*, rsRetVal (**)()), modInfo_t*);
+
+#endif /* #ifndef OMPIPE_H_INCLUDED */
+/* vi:set ai:
+ */
diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c
index 499a11dd..e788a006 100644
--- a/tools/omusrmsg.c
+++ b/tools/omusrmsg.c
@@ -50,7 +50,15 @@
 #include <assert.h>
 #include <signal.h>
 #include <sys/param.h>
-#include <utmp.h>
+#ifdef HAVE_UTMP_H
+#  include <utmp.h>
+#  define STRUCTUTMP struct utmp
+#  define UTNAME ut_name
+#else
+#  include <utmpx.h>
+#  define STRUCTUTMP struct utmpx
+#  define UTNAME ut_user
+#endif
 #include <unistd.h>
 #include <sys/uio.h>
 #include <sys/stat.h>
@@ -124,6 +132,12 @@ ENDdbgPrintInstInfo
  * need! rgerhards 2005-03-18
  */
 #ifdef OS_BSD
+/* Since version 900007, FreeBSD has a POSIX compliant <utmpx.h> */
+#if defined(__FreeBSD__) && (__FreeBSD_version >= 900007)
+#  define setutent(void) setutxent(void)
+#  define getutent(void) getutxent(void)
+#  define endutent(void) endutxent(void)
+#else
 static FILE *BSD_uf = NULL;
 void setutent(void)
 {
@@ -134,9 +148,9 @@ void setutent(void)
 	}
 }
 
-struct utmp* getutent(void)
+STRUCTUTMP* getutent(void)
 {
-	static struct utmp st_utmp;
+	static STRUCTUTMP st_utmp;
 
 	if(fread((char *)&st_utmp, sizeof(st_utmp), 1, BSD_uf) != 1)
 		return NULL;
@@ -149,6 +163,7 @@ void endutent(void)
 	fclose(BSD_uf);
 	BSD_uf = NULL;
 }
+#endif /* if defined(__FreeBSD__) */
 #endif  /* #ifdef OS_BSD */
 
 
@@ -173,8 +188,8 @@ static rsRetVal wallmsg(uchar* pMsg, instanceData *pData)
 	int errnoSave;
 	int ttyf;
 	int wrRet;
-	struct utmp ut;
-	struct utmp *uptr;
+	STRUCTUTMP ut;
+	STRUCTUTMP *uptr;
 	struct stat statb;
 	DEFiRet;
 
@@ -187,13 +202,13 @@ static rsRetVal wallmsg(uchar* pMsg, instanceData *pData)
 	while((uptr = getutent())) {
 		memcpy(&ut, uptr, sizeof(ut));
 		/* is this slot used? */
-		if(ut.ut_name[0] == '\0')
+		if(ut.UTNAME[0] == '\0')
 			continue;
 #ifndef OS_BSD
 		if(ut.ut_type != USER_PROCESS)
 			continue;
 #endif
-		if(!(strncmp (ut.ut_name,"LOGIN", 6))) /* paranoia */
+		if(!(strncmp (ut.UTNAME,"LOGIN", 6))) /* paranoia */
 			continue;
 
 		/* should we send the message to this user? */
@@ -203,7 +218,7 @@ static rsRetVal wallmsg(uchar* pMsg, instanceData *pData)
 					i = MAXUNAMES;
 					break;
 				}
-				if(strncmp(pData->uname[i], ut.ut_name, UNAMESZ) == 0)
+				if(strncmp(pData->uname[i], ut.UTNAME, UNAMESZ) == 0)
 					break;
 			}
 			if(i == MAXUNAMES) /* user not found? */
diff --git a/tools/regexp.c b/tools/regexp.c
index c8e4c681..e8bba4f4 100644
--- a/tools/regexp.c
+++ b/tools/regexp.c
@@ -26,6 +26,7 @@
  *
  * A copy of the GPL can be found in the file "COPYING" in this distribution.
  */
+#include "config.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/tools/rsyslog.conf.5 b/tools/rsyslog.conf.5
index f2b915e2..e8a4ab92 100644
--- a/tools/rsyslog.conf.5
+++ b/tools/rsyslog.conf.5
@@ -80,7 +80,7 @@ used like this:
 .IP
 $ModLoad imudp
 .IP
-$InputUDPServerRun 514
+$UDPServerRun 514
 .TP
 .I imtcp
 Input plugin for plain TCP syslog. Replaces the deprecated -t
diff --git a/tools/syslogd.c b/tools/syslogd.c
index 8fcb8dd0..e318fd8e 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -127,6 +127,7 @@
 #include "omusrmsg.h"
 #include "omfwd.h"
 #include "omfile.h"
+#include "ompipe.h"
 #include "omdiscard.h"
 #include "threads.h"
 #include "queue.h"
@@ -177,7 +178,11 @@ static rsRetVal GlobalClassExit(void);
 #endif
 
 #ifndef _PATH_MODDIR
-#define _PATH_MODDIR	"/lib/rsyslog/"
+#       if defined(__FreeBSD__)
+#               define _PATH_MODDIR     "/usr/local/lib/rsyslog/"
+#       else
+#               define _PATH_MODDIR     "/lib/rsyslog/"
+#       endif
 #endif
 
 #if defined(SYSLOGD_PIDNAME)
@@ -792,7 +797,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla
 				 * (I couldn't do any more smart things anyway...).
 				 * rgerhards, 2007-9-20
 				 */
-				DBGPRINTF("internal error: iMsg > max msg size in printchopped()\n");
+				DBGPRINTF("internal error: iMsg > max msg size in parseAndSubmitMessage()\n");
 			}
 			FINALIZE; /* in this case, we are done... nothing left we can do */
 		}
@@ -1193,13 +1198,12 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 	int bTAGCharDetected;
 	int i;	/* general index for parsing */
 	uchar bufParseTAG[CONF_TAG_MAXSIZE];
-	uchar bufParseHOSTNAME[CONF_TAG_HOSTNAME];
+	uchar bufParseHOSTNAME[CONF_HOSTNAME_MAXSIZE];
 	BEGINfunc
 
 	assert(pMsg != NULL);
 	assert(pMsg->pszRawMsg != NULL);
-	lenMsg = pMsg->iLenRawMsg - (pMsg->offAfterPRI + 1);
-
+	lenMsg = pMsg->iLenRawMsg - pMsg->offAfterPRI; /* note: offAfterPRI is already the number of PRI chars (do not add one!) */
 	p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */
 
 	/* Check to see if msg contains a timestamp. We start by assuming
@@ -1255,16 +1259,24 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags)
 		bTAGCharDetected = 0;
 		if(lenMsg > 0 && flags & PARSE_HOSTNAME) {
 			i = 0;
-			while(lenMsg > 0 && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.'
-				|| p2parse[i] == '_' || p2parse[i] == '-') && i < CONF_TAG_MAXSIZE) {
+			while(i < lenMsg && (isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.'
+				|| p2parse[i] == '_' || p2parse[i] == '-') && i < (CONF_HOSTNAME_MAXSIZE - 1)) {
 				bufParseHOSTNAME[i] = p2parse[i];
 				++i;
-				--lenMsg;
 			}
 
-			if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) {
+			if(i == lenMsg) {
+				/* we have a message that is empty immediately after the hostname,
+				 * but the hostname thus is valid! -- rgerhards, 2010-02-22
+				 */
+				p2parse += i;
+				lenMsg -= i;
+				bufParseHOSTNAME[i] = '\0';
+				MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i);
+			} else if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) {
 				/* we got a hostname! */
 				p2parse += i + 1; /* "eat" it (including SP delimiter) */
+				lenMsg -= i + 1;
 				bufParseHOSTNAME[i] = '\0';
 				MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i);
 			}
@@ -2664,6 +2676,9 @@ static rsRetVal loadBuildInModules(void)
 	if((iRet = module.doModInit(modInitFile, UCHAR_CONSTANT("builtin-file"), NULL)) != RS_RET_OK) {
 		RETiRet;
 	}
+	if((iRet = module.doModInit(modInitPipe, UCHAR_CONSTANT("builtin-pipe"), NULL)) != RS_RET_OK) {
+		RETiRet;
+	}
 #ifdef SYSLOG_INET
 	if((iRet = module.doModInit(modInitFwd, UCHAR_CONSTANT("builtin-fwd"), NULL)) != RS_RET_OK) {
 		RETiRet;
@@ -2769,7 +2784,7 @@ static void printVersion(void)
 #else
 	printf("\tFEATURE_REGEXP:\t\t\t\tNo\n");
 #endif
-#ifndef	NOLARGEFILE
+#if defined(_LARGE_FILES) || (defined (_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS >= 64)
 	printf("\tFEATURE_LARGEFILE:\t\t\tYes\n");
 #else
 	printf("\tFEATURE_LARGEFILE:\t\t\tNo\n");
@@ -3173,6 +3188,7 @@ int realMain(int argc, char **argv)
 	uchar *LocalHostName;
 	uchar *LocalDomain;
 	uchar *LocalFQDNName;
+	char cwdbuf[128]; /* buffer to obtain/display current working directory */
 
 	/* first, parse the command line options. We do not carry out any actual work, just
 	 * see what we should do. This relieves us from certain anomalies and we can process
@@ -3259,8 +3275,9 @@ int realMain(int argc, char **argv)
 	if ((argc -= optind))
 		usage();
 
-	DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s'\n",
-		  VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath);
+	DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s', cwd:%s\n",
+		  VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath,
+		  getcwd(cwdbuf, sizeof(cwdbuf)));
 
 	/* we are done with the initial option parsing and processing. Now we init the system. */
 
diff --git a/tools/zpipe.c b/tools/zpipe.c
index bde6c5c1..38069425 100644
--- a/tools/zpipe.c
+++ b/tools/zpipe.c
@@ -4,6 +4,13 @@
    Version 2.0  03 June     2009  Rainer Gerhards */
 
 /* RSYSLOG NOTE:
+ * This file is primarily been used as a testing aid for rsyslog. We do NOT
+ * properly maintain it and it has been brought to our attention that it may
+ * have some security issues. However, we prefer not to remove the file as it
+ * may turn out to be useful for further testing. All users are advised NOT
+ * to base any development on this version here, but rather look for the
+ * original zpipe.c by the authors mentioned above.
+ *
  * This file is beeing distributed as part of rsyslog, but is just an
  * add-on. Most importantly, rsyslog's copyright does not apply but
  * rather the (non-) copyright stated above.
@@ -22,6 +29,7 @@
 		     files created by rsyslog's zip output writer.
  */
 
+#include "config.h"
 #include <stdio.h>
 #include <string.h>
 #include <assert.h>