From 9e0426da5fae3f565e44411ed8b9796acc81040d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Mar 2010 17:52:01 +0100 Subject: bugfix: $omfileFlushOnTXEnd was turned on when set to off and vice versa --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index eb56201c..9b8f8e37 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -413,7 +413,7 @@ prepareFile(instanceData *pData, uchar *newFileName) * async processing, which is a real performance waste if we do not do buffered * writes! -- rgerhards, 2009-07-06 */ - if(!pData->bFlushOnTXEnd) + if(pData->bFlushOnTXEnd) CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); if(pData->pszSizeLimitCmd != NULL) CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); -- cgit From 7916735ba6890f67d49077975b0d635dc9931380 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Mar 2010 17:57:46 +0100 Subject: retain old $OMFileFlushOnTXEnd semantics required because due to bug the default was actually different than specified (or better said: spec was inconsistent in doc as well). --- tools/omfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 9b8f8e37..8fbf8fea 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -99,7 +99,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */ +static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ uchar *pszFileDfltTplName = NULL; /* name of the default template to use */ @@ -749,7 +749,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; - bFlushOnTXEnd = 1; + bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; iFlushInterval = FLUSH_INTRVL_DFLT; if(pszFileDfltTplName != NULL) { -- cgit From 3d80d6ba301e4d26b646c84d621ebe880ebc513f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Mar 2010 18:07:31 +0100 Subject: bugfix: potential problem (loop, abort) when file write error occured When a write error occured in stream.c, variable iWritten had the error code but this was handled as if it were the actual number of bytes written. That was used in pointer arithmetic later on, and thus could lead to all sorts of problems. However, this could only happen if the error was EINTR or the file in question was a tty. All other cases were handled properly. Now, iWritten is reset to zero in such cases, resulting in proper retries. --- tools/omfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 8fbf8fea..daefa680 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -79,7 +79,7 @@ DEFobjCurrIf(strm) struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ strm_t *pStrm; /* our output stream */ - time_t lastUsed; /* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) + time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; @@ -532,7 +532,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } pCache[iFirstFree]->pStrm = pData->pStrm; - pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance + pCache[iFirstFree]->lastUsed = time(NULL); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); -- cgit From ba0f23182a55b26b2265d2138c707cbc7ddbb72e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Mar 2010 16:25:24 +0100 Subject: new feature: "." action type added to support writing files to relative pathes (this is primarily meant as a debug aid) --- tools/omfile.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index daefa680..438f8c4a 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -633,7 +633,7 @@ ENDdoAction BEGINparseSelectorAct CODESTARTparseSelectorAct - if(!(*p == '$' || *p == '?' || *p == '/' || *p == '-')) + if(!(*p == '$' || *p == '?' || *p == '/' || *p == '.' || *p == '-')) ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); CHKiRet(createInstance(&pData)); @@ -683,6 +683,7 @@ CODESTARTparseSelectorAct * need high-performance pipes at a later stage (unlikely). -- rgerhards, 2010-02-28 */ case '/': + case '.': CODE_STD_STRING_REQUESTparseSelectorAct(1) /* we now have *almost* the same semantics for files and pipes, but we still need * to know we deal with a pipe, because we must do non-blocking opens in that case -- cgit From 89216d6a96ea5f6d1fa9893d56fa877a2131d390 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Mar 2010 09:42:46 +0100 Subject: fixed regression from previos (yet unrelease) $omfileFlushOnTXEnd fix The previous fix fixed an issue with on/off bying used in the exact wrong semantic. It corrected the situation, but failed to fix one spot where the wrong semantics were used. This is done with this commit. Note that this is NOT a bug seen in any released version. --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 438f8c4a..c4e834cc 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -413,7 +413,7 @@ prepareFile(instanceData *pData, uchar *newFileName) * async processing, which is a real performance waste if we do not do buffered * writes! -- rgerhards, 2009-07-06 */ - if(pData->bFlushOnTXEnd) + if(!pData->bFlushOnTXEnd) CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); if(pData->pszSizeLimitCmd != NULL) CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); -- cgit From dccadb677c5a6b8379f631e4c1f14c8c4089d4a6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 21 Mar 2010 18:52:51 +0100 Subject: exp: made omfile ensure that it is not called twice in parallel on the same file instance In theory, the rsyslog core should never call in parallel into an output module for the same instance. However, it looks like this seems to happen under (strange?) circumstances. I have now enhanced omfile so that it guards itself against being called in parallel on the same instance data. This is done to help troubleshooting and may stay as an interim solution if it proves to solve an anomaly we see in at least one installation (to trigger this problem, an extremely large traffic volume is needed). --- tools/omfile.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index c4e834cc..f4db7dab 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -48,6 +48,7 @@ #include #include #include +#include #ifdef OS_SOLARIS # include @@ -65,6 +66,7 @@ #include "stream.h" #include "unicode-helper.h" #include "atomic.h" +#include "debug.h" MODULE_TYPE_OUTPUT @@ -134,6 +136,7 @@ typedef struct _instanceData { int iIOBufSize; /* size of associated io buffer */ int iFlushInterval; /* how fast flush buffer on inactivity? */ bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ + pthread_mutex_t mutEx; /* guard ourselves against being called multiple times with the same instance */ } instanceData; @@ -603,6 +606,7 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->pStrm = NULL; + pthread_mutex_init(&pData->mutEx, NULL); ENDcreateInstance @@ -612,6 +616,7 @@ CODESTARTfreeInstance dynaFileFreeCache(pData); } else if(pData->pStrm != NULL) strm.Destruct(&pData->pStrm); + pthread_mutex_destroy(&pData->mutEx); ENDfreeInstance @@ -621,12 +626,14 @@ ENDtryResume BEGINdoAction CODESTARTdoAction + d_pthread_mutex_lock(&pData->mutEx); DBGPRINTF("file to log to: %s\n", pData->f_fname); CHKiRet(writeFile(ppString, iMsgOpts, pData)); if(pData->bFlushOnTXEnd) { /* TODO v5: do this in endTransaction only! */ CHKiRet(strm.Flush(pData->pStrm)); } + d_pthread_mutex_unlock(&pData->mutEx); finalize_it: ENDdoAction @@ -764,6 +771,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINdoHUP CODESTARTdoHUP + d_pthread_mutex_lock(&pData->mutEx); if(pData->bDynamicName) { dynaFileFreeCacheEntries(pData); } else { @@ -772,6 +780,7 @@ CODESTARTdoHUP pData->pStrm = NULL; } } + d_pthread_mutex_unlock(&pData->mutEx); ENDdoHUP -- cgit From a28a31feeab1bec5f4c74e3b6121f866f68eac54 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Mar 2010 14:22:22 +0100 Subject: previous (experimental) patch was somewhat incorrect, fixed This is what caused the new test to fail... --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index f4db7dab..0f27f1e9 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -633,8 +633,8 @@ CODESTARTdoAction /* TODO v5: do this in endTransaction only! */ CHKiRet(strm.Flush(pData->pStrm)); } - d_pthread_mutex_unlock(&pData->mutEx); finalize_it: + d_pthread_mutex_unlock(&pData->mutEx); ENDdoAction -- cgit From f8dee56243d7378864fdcdcc21262fc563639827 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Mar 2010 14:11:50 +0100 Subject: bugfix: potential re-use of free()ed file stream object in omfile when dynaCache is enabled, the cache is full, a new entry needs to be allocated, thus the LRU discarded, then a new entry is opend and that fails. In that case, it looks like the discarded stream may be reused improperly (based on code analysis, test case and confirmation pending) --- tools/omfile.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 0f27f1e9..ae5f350d 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -286,13 +286,15 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName); - /* if the name is NULL, this is an improperly initilized entry which +// RG: check the "open failed" case -- can this cause trouble (but do we have that situation?) + /* if the name is NULL, this is an improperly initialized entry which * needs to be discarded. In this case, neither the file is to be closed - * not the name to be freed. + * nor the name to be freed. */ if(pCache[iEntry]->pName != NULL) { if(pCache[iEntry]->pStrm != NULL) strm.Destruct(&pCache[iEntry]->pStrm); +// RG: pStrm should now be NULL... d_free(pCache[iEntry]->pName); pCache[iEntry]->pName = NULL; } @@ -474,7 +476,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) if(iFirstFree == -1) iFirstFree = i; } else { /* got an element, let's see if it matches */ - if(!ustrcmp(newFileName, pCache[i]->pName)) { + if(!ustrcmp(newFileName, pCache[i]->pName)) { // RG: name == NULL? /* we found our element! */ pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; @@ -497,24 +499,38 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) * is error-prone, so I prefer to do it here. -- rgerhards, 2010-03-02 */ pData->iCurrElt = -1; + /* similarly, we need to set the current pStrm to NULL, because otherwise, if prepareFile() fails, + * we may end up using an old stream. This bug depends on how exactly prepareFile fails, + * but it* could be triggered in the common case of a failed open() system call. + * rgerhards, 2010-03-22 + */ + pData->pStrm = NULL; if(iFirstFree == -1 && (pData->iCurrCacheSize < pData->iDynaFileCacheSize)) { /* there is space left, so set it to that index */ iFirstFree = pData->iCurrCacheSize++; } +// RG: this is the begin of a potential problem area + /* Note that the following code sequence does not work with the cache entry itself, + * but rather with pData->pStrm, the (sole) stream pointer in the non-dynafile case. + * The cache array is only updated after the open was successful. -- rgerhards, 2010-03-21 + */ if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ } else { /* we need to allocate memory for the cache structure */ + /* TODO: performance note: we could alloc all entries on startup, thus saving malloc + * overhead -- this may be something to consider in v5... + */ CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry))); } /* Ok, we finally can open the file */ localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ - /* file is either open now or an error state set */ + /* file is either open now or an error state set */ // RG: better check localRet? if(pData->pStrm == NULL) { /* do not report anything if the message is an internally-generated * message. Otherwise, we could run into a never-ending loop. The bad @@ -525,13 +541,11 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) } else { errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } - dynaFileDelCacheEntry(pCache, iFirstFree, 1); ABORT_FINALIZE(localRet); } if((pCache[iFirstFree]->pName = ustrdup(newFileName)) == NULL) { - /* we need to discard the entry, otherwise things could lead to a segfault! */ - dynaFileDelCacheEntry(pCache, iFirstFree, 1); + strm.Destruct(&pData->pStrm); /* need to free failed entry! */ ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } pCache[iFirstFree]->pStrm = pData->pStrm; -- cgit From 5d58774813d4ecd4fc9f8230f8d5446457eb2ed5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Mar 2010 14:21:33 +0100 Subject: streamline dynafile cache entry deletion a bit The old code looks a bit "strange", though not necessarily incorrect. The new code looks correct and is probably less irritating during bug hunting. --- tools/omfile.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index ae5f350d..91bcf0c2 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -286,19 +286,18 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName); -// RG: check the "open failed" case -- can this cause trouble (but do we have that situation?) - /* if the name is NULL, this is an improperly initialized entry which - * needs to be discarded. In this case, neither the file is to be closed - * nor the name to be freed. - */ + if(pCache[iEntry]->pName != NULL) { - if(pCache[iEntry]->pStrm != NULL) - strm.Destruct(&pCache[iEntry]->pStrm); -// RG: pStrm should now be NULL... d_free(pCache[iEntry]->pName); pCache[iEntry]->pName = NULL; } + if(pCache[iEntry]->pStrm != NULL) { + strm.Destruct(&pCache[iEntry]->pStrm); + if(pCache[iEntry]->pStrm != NULL) /* safety check -- TODO: remove if no longer necessary */ + abort(); + } + if(bFreeEntry) { d_free(pCache[iEntry]); pCache[iEntry] = NULL; -- cgit From 28b3703c95cb06642ff245f4d7e265c4591c128f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Mar 2010 07:56:03 +0100 Subject: bugfix: potential segfault in dynafile cache This bug was triggered by an open failure. The the cache was full and a new entry needed to be placed inside it, a victim for eviction was selected. That victim was freed, then the open of the new file tried. If the open failed, the victim entry was still freed, and the function exited. However, on next invocation and cache search, the victim entry was used as if it were populated, most probably resulting in a segfault. --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 91bcf0c2..2a14f45b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -471,7 +471,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ ttOldest = time(NULL) + 1; /* there must always be an older one */ for(i = 0 ; i < pData->iCurrCacheSize ; ++i) { - if(pCache[i] == NULL) { + if(pCache[i] == NULL || pCache[i]->pName == NULL) { if(iFirstFree == -1) iFirstFree = i; } else { /* got an element, let's see if it matches */ -- cgit From bcddd30c2e9d8594a67a853c389c4424aa0c6524 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Mar 2010 08:13:58 +0100 Subject: undo experimental commit dccadb677c5a6b8379f631e4c1f14c8c4089d4a6 Further testing turned out that the rsyslog core works correctly and this fix is not needed. The concurrency we saw was actually caused by other actions (even processes) during directory creation. See commit 9e5b31fc44136dbcc1e443cfe7714e9daf97d844 for further details. --- tools/omfile.c | 9 --------- 1 file changed, 9 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 2a14f45b..774f0b96 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -48,7 +48,6 @@ #include #include #include -#include #ifdef OS_SOLARIS # include @@ -66,7 +65,6 @@ #include "stream.h" #include "unicode-helper.h" #include "atomic.h" -#include "debug.h" MODULE_TYPE_OUTPUT @@ -136,7 +134,6 @@ typedef struct _instanceData { int iIOBufSize; /* size of associated io buffer */ int iFlushInterval; /* how fast flush buffer on inactivity? */ bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ - pthread_mutex_t mutEx; /* guard ourselves against being called multiple times with the same instance */ } instanceData; @@ -619,7 +616,6 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->pStrm = NULL; - pthread_mutex_init(&pData->mutEx, NULL); ENDcreateInstance @@ -629,7 +625,6 @@ CODESTARTfreeInstance dynaFileFreeCache(pData); } else if(pData->pStrm != NULL) strm.Destruct(&pData->pStrm); - pthread_mutex_destroy(&pData->mutEx); ENDfreeInstance @@ -639,7 +634,6 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - d_pthread_mutex_lock(&pData->mutEx); DBGPRINTF("file to log to: %s\n", pData->f_fname); CHKiRet(writeFile(ppString, iMsgOpts, pData)); if(pData->bFlushOnTXEnd) { @@ -647,7 +641,6 @@ CODESTARTdoAction CHKiRet(strm.Flush(pData->pStrm)); } finalize_it: - d_pthread_mutex_unlock(&pData->mutEx); ENDdoAction @@ -784,7 +777,6 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a BEGINdoHUP CODESTARTdoHUP - d_pthread_mutex_lock(&pData->mutEx); if(pData->bDynamicName) { dynaFileFreeCacheEntries(pData); } else { @@ -793,7 +785,6 @@ CODESTARTdoHUP pData->pStrm = NULL; } } - d_pthread_mutex_unlock(&pData->mutEx); ENDdoHUP -- cgit From d102aab886fcde914677146caf20bd68ca41bfff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Mar 2010 15:23:39 +0100 Subject: new feature: $OMFileAsyncWriting directive added it permits to specifiy if asynchronous writing should be done or not --- tools/omfile.c | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 774f0b96..dd944946 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -86,6 +86,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ #define FLUSH_INTRVL_DFLT 1 /* default buffer flush interval (in seconds) */ +#define USE_ASYNCWRITER_DFLT 0 /* default buffer use async writer */ /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -102,6 +103,7 @@ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ +static int bUseAsyncWriter = USE_ASYNCWRITER_DFLT; /* should we enable asynchronous writing? */ uchar *pszFileDfltTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -134,6 +136,7 @@ typedef struct _instanceData { int iIOBufSize; /* size of associated io buffer */ int iFlushInterval; /* how fast flush buffer on inactivity? */ bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ + bool bUseAsyncWriter; /* use async stream writer? */ } instanceData; @@ -147,26 +150,23 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - dbgprintf("[dynamic]\n\ttemplate='%s'" - "\tfile cache size=%d\n" - "\tcreate directories: %s\n" - "\tfile owner %d, group %d\n" - "\tdirectory owner %d, group %d\n" - "\tdir create mode 0%3.3o, file create mode 0%3.3o\n" - "\tfail if owner/group can not be set: %s\n", - pData->f_fname, - pData->iDynaFileCacheSize, - pData->bCreateDirs ? "yes" : "no", - pData->fileUID, pData->fileGID, - pData->dirUID, pData->dirGID, - pData->fDirCreateMode, pData->fCreateMode, - pData->bFailOnChown ? "yes" : "no" - ); + dbgprintf("[dynamic]\n"); } else { /* regular file */ - dbgprintf("%s", pData->f_fname); - if (pData->pStrm == NULL) - dbgprintf(" (unused)"); + dbgprintf("%s%s\n", pData->f_fname, + (pData->pStrm == NULL) ? " (unused)" : ""); } + + dbgprintf("\ttemplate='%s'\n", pData->f_fname); + dbgprintf("\tuse async writer=%d\n", pData->bUseAsyncWriter); + dbgprintf("\tflush on TX end=%d\n", pData->bFlushOnTXEnd); + dbgprintf("\tflush interval=%d\n", pData->iFlushInterval); + dbgprintf("\tfile cache size=%d\n", pData->iDynaFileCacheSize); + dbgprintf("\tcreate directories: %s\n", pData->bCreateDirs ? "yes" : "no"); + dbgprintf("\tfile owner %d, group %d\n", pData->fileUID, pData->fileGID); + dbgprintf("\tdirectory owner %d, group %d\n", pData->dirUID, pData->dirGID); + dbgprintf("\tdir create mode 0%3.3o, file create mode 0%3.3o\n", + pData->fDirCreateMode, pData->fCreateMode); + dbgprintf("\tfail if owner/group can not be set: %s\n", pData->bFailOnChown ? "yes" : "no"); ENDdbgPrintInstInfo @@ -414,7 +414,7 @@ prepareFile(instanceData *pData, uchar *newFileName) * async processing, which is a real performance waste if we do not do buffered * writes! -- rgerhards, 2009-07-06 */ - if(!pData->bFlushOnTXEnd) + if(pData->bUseAsyncWriter) CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); if(pData->pszSizeLimitCmd != NULL) CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); @@ -731,6 +731,7 @@ CODESTARTparseSelectorAct pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = (int) iIOBufSize; pData->iFlushInterval = iFlushInterval; + pData->bUseAsyncWriter = bUseAsyncWriter; if(pData->bDynamicName == 0) { /* try open and emit error message if not possible. At this stage, we ignore the @@ -766,6 +767,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; iFlushInterval = FLUSH_INTRVL_DFLT; + bUseAsyncWriter = USE_ASYNCWRITER_DFLT; if(pszFileDfltTplName != NULL) { free(pszFileDfltTplName); pszFileDfltTplName = NULL; @@ -812,6 +814,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileasyncwriting", 0, eCmdHdlrInt, NULL, &bUseAsyncWriter, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); -- cgit From 77f4230c10f9f2a1c9605f0b4208d7209b2ccf27 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Mar 2010 17:36:50 +0100 Subject: $OMFileAsyncWriting directive did not work properly --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index dd944946..0f476000 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -814,7 +814,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileasyncwriting", 0, eCmdHdlrInt, NULL, &bUseAsyncWriter, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileasyncwriting", 0, eCmdHdlrBinary, NULL, &bUseAsyncWriter, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); -- cgit From e2ceb7247cbd9dff1bfb0efe22d6bc68e70a3fd8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 7 Apr 2010 14:25:40 +0200 Subject: bugfix: default for $OMFileFlushOnTXEnd was wrong ("off"). This, in default mode, caused buffered writing to be used, what means that it looked like no output were written or partial lines. Thanks to Michael Biebl for pointing out this bug. --- tools/omfile.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 0f476000..24de052c 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -87,6 +87,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ #define FLUSH_INTRVL_DFLT 1 /* default buffer flush interval (in seconds) */ #define USE_ASYNCWRITER_DFLT 0 /* default buffer use async writer */ +#define FLUSHONTX_DFLT 1 /* default for flush on TX end */ /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -100,7 +101,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ +static bool bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */ static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static int bUseAsyncWriter = USE_ASYNCWRITER_DFLT; /* should we enable asynchronous writing? */ @@ -764,7 +765,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; - bFlushOnTXEnd = 0; + bFlushOnTXEnd = FLUSHONTX_DFLT; iIOBufSize = IOBUF_DFLT_SIZE; iFlushInterval = FLUSH_INTRVL_DFLT; bUseAsyncWriter = USE_ASYNCWRITER_DFLT; -- cgit