diff options
Diffstat (limited to 'tools/omfile.c')
-rw-r--r-- | tools/omfile.c | 97 |
1 files changed, 76 insertions, 21 deletions
diff --git a/tools/omfile.c b/tools/omfile.c index 109bfcc0..fbd263c7 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -48,10 +48,13 @@ #include <libgen.h> #include <unistd.h> #include <sys/file.h> - #ifdef OS_SOLARIS # include <fcntl.h> #endif +#ifdef HAVE_ATOMIC_BUILTINS +# include <pthread.h> +#endif + #include "conf.h" #include "syslogd-types.h" @@ -67,6 +70,7 @@ #include "atomic.h" MODULE_TYPE_OUTPUT +MODULE_TYPE_NOKEEP /* internal structures */ @@ -74,12 +78,41 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(strm) +/* for our current LRU mechanism, we need a monotonically increasing counters. We use + * it much like a "Lamport logical clock": we do not need the actual time, we just need + * to know the sequence in which files were accessed. So we use a simple counter to + * create that sequence. We use an unsigned 64 bit value which is extremely unlike to + * wrap within the lifetime of a process. If we process 1,000,000 file writes per + * second, the process could still exist over 500,000 years before a wrap to 0 happens. + * That should be sufficient (and even than, there would no really bad effect ;)). + * The variable below is the global counter/clock. + */ +#if HAVE_ATOMIC_BUILTINS_64BIT +static uint64 clockFileAccess = 0; +#else +static unsigned clockFileAccess = 0; +#endif +/* and the "tick" function */ +#ifndef HAVE_ATOMIC_BUILTINS +static pthread_mutex_t mutClock; +#endif +static inline uint64 +getClockFileAccess(void) +{ +#if HAVE_ATOMIC_BUILTINS_64BIT + return ATOMIC_INC_AND_FETCH_uint64(&clockFileAccess, &mutClock); +#else + return ATOMIC_INC_AND_FETCH_unsigned(&clockFileAccess, &mutClock); +#endif +} + + /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ strm_t *pStrm; /* our output stream */ - time_t lastUsed; /* for LRU - last access */ + uint64 clkTickAccessed;/* for LRU - based on clockFileAccess */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; @@ -103,7 +136,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs = 1;/* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */ +static sbool bFlushOnTXEnd = FLUSHONTX_DFLT;/* flush write buffers when transaction has ended? */ static int64 iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static int bUseAsyncWriter = USE_ASYNCWRITER_DFLT; /* should we enable asynchronous writing? */ @@ -120,7 +153,7 @@ typedef struct _instanceData { int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ - bool bForceChown; /* force chown() on existing files? */ + sbool bForceChown; /* force chown() on existing files? */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -139,8 +172,8 @@ typedef struct _instanceData { int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ int iFlushInterval; /* how fast flush buffer on inactivity? */ - bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ - bool bUseAsyncWriter; /* use async stream writer? */ + sbool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ + sbool bUseAsyncWriter; /* use async stream writer? */ } instanceData; @@ -461,7 +494,7 @@ finalize_it: static inline rsRetVal prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) { - time_t ttOldest; /* timestamp of oldest element */ + uint64 ctOldest; /* "timestamp" of oldest element */ int iOldest; int i; int iFirstFree; @@ -480,7 +513,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) if( (pData->iCurrElt != -1) && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ - pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call! + pCache[pData->iCurrElt]->clkTickAccessed = getClockFileAccess(); // LRU needs only a strictly monotonically increasing counter, so such a one could do FINALIZE; } @@ -491,7 +524,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) pData->iCurrElt = -1; /* invalid current element pointer */ iFirstFree = -1; /* not yet found */ iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ - ttOldest = time(NULL) + 1; /* there must always be an older one */ + ctOldest = getClockFileAccess(); /* there must always be an older one */ for(i = 0 ; i < pData->iCurrCacheSize ; ++i) { if(pCache[i] == NULL || pCache[i]->pName == NULL) { if(iFirstFree == -1) @@ -501,12 +534,12 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) /* we found our element! */ pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; - pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */ + pCache[i]->clkTickAccessed = getClockFileAccess(); /* update "timestamp" for LRU */ FINALIZE; } /* did not find it - so lets keep track of the counters for LRU */ - if(pCache[i]->lastUsed < ttOldest) { - ttOldest = pCache[i]->lastUsed; + if(pCache[i]->clkTickAccessed < ctOldest) { + ctOldest = pCache[i]->clkTickAccessed; iOldest = i; } } @@ -532,7 +565,6 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) iFirstFree = pData->iCurrCacheSize++; } -// RG: this is the begin of a potential problem area /* Note that the following code sequence does not work with the cache entry itself, * but rather with pData->pStrm, the (sole) stream pointer in the non-dynafile case. * The cache array is only updated after the open was successful. -- rgerhards, 2010-03-21 @@ -558,9 +590,10 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) * news is that we also lose errors on startup messages, but so it is. */ if(iMsgOpts & INTERNAL_MSG) { - DBGPRINTF("Could not open dynaFile, discarding message\n"); + DBGPRINTF("Could not open dynaFile '%s', state %d, discarding message\n", + newFileName, localRet); } else { - errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' [state %d] - discarding message", newFileName, localRet); + errmsg.LogError(0, localRet, "Could not open dynamic file '%s' [state %d] - discarding message", newFileName, localRet); } ABORT_FINALIZE(localRet); } @@ -570,7 +603,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } pCache[iFirstFree]->pStrm = pData->pStrm; - pCache[iFirstFree]->lastUsed = time(NULL); + pCache[iFirstFree]->clkTickAccessed = getClockFileAccess(); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); @@ -591,7 +624,7 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pData != NULL); ASSERT(pszBuf != NULL); -dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); +dbgprintf("write to stream, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); FINALIZE; @@ -631,8 +664,6 @@ finalize_it: /* in v5, we shall return different states for message-caused failure (but only there!) */ if(pData->strmType == STREAMTYPE_NAMED_PIPE) iRet = RS_RET_DISABLE_ACTION; /* this is the traditional semantic -- rgerhards, 2010-01-15 */ - else - iRet = RS_RET_SUSPENDED; } RETiRet; } @@ -657,15 +688,32 @@ BEGINtryResume CODESTARTtryResume ENDtryResume +BEGINbeginTransaction +CODESTARTbeginTransaction + /* we have nothing to do to begin a transaction */ +ENDbeginTransaction + + +BEGINendTransaction +CODESTARTendTransaction + /* Note: pStrm may be NULL if there was an error opening the stream */ + if(pData->bFlushOnTXEnd && pData->pStrm != NULL) { + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: +ENDendTransaction + + BEGINdoAction CODESTARTdoAction DBGPRINTF("file to log to: %s\n", pData->f_fname); CHKiRet(writeFile(ppString, iMsgOpts, pData)); - if(pData->bFlushOnTXEnd) { - /* TODO v5: do this in endTransaction only! */ + if(!bCoreSupportsBatching && pData->bFlushOnTXEnd) { CHKiRet(strm.Flush(pData->pStrm)); } finalize_it: + if(iRet == RS_RET_OK) + iRet = RS_RET_DEFER_COMMIT; ENDdoAction @@ -837,12 +885,14 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(strm, CORE_COMPONENT); free(pszFileDfltTplName); + DESTROY_ATOMIC_HELPER_MUT(mutClock); ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ CODEqueryEtryPt_doHUP ENDqueryEtryPt @@ -853,6 +903,11 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + + INIT_ATOMIC_HELPER_MUT(mutClock); + + INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + DBGPRINTF("omfile: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); |