From f54e72cec06f21f4af939c70541e8a339b7e56ff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 14:45:09 +0200 Subject: first proof of concept on zipped file writer This DOES NOT work sufficiently well, I just wanted to verify that zip writing is possible and files are readable. Will be refined soon. --- tools/omfile.c | 234 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 196 insertions(+), 38 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index c7283e4d..62a044d8 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,7 +12,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -57,6 +57,8 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "unicode-helper.h" +#include "zlibw.h" MODULE_TYPE_OUTPUT @@ -64,16 +66,30 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +DEFobjCurrIf(zlibw) /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { - uchar *pName; /* name currently open, if dynamic name */ + uchar *pName; /* name currently open, if dynamic name */ short fd; /* name associated with file name in cache */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; +/* the output buffer structure */ +// TODO: later make this part of the dynafile cache! +#define OUTBUF_LEN 128 // TODO: make dynamic! +typedef struct { + uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ + size_t lenBuf; /* max size of buffer */ + size_t iBuf; /* current buffer index */ + int fd; /* which file descriptor is this buf for? */ + /* elements for zip writing */ + z_stream zStrm; + char zipBuf[OUTBUF_LEN]; +} outbuf_t; + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -92,7 +108,8 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ - short fd; /* file descriptor for (current) file */ + short fd; /* file descriptor for (current) file */ + outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -132,7 +149,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" @@ -146,9 +163,9 @@ CODESTARTdbgPrintInstInfo pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); + dbgprintf("%s", pData->f_fname); if (pData->fd == -1) - printf(" (unused)"); + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -321,7 +338,8 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) +static void +dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { ASSERT(pCache != NULL); @@ -356,7 +374,8 @@ finalize_it: * relevant files. Part of Shutdown and HUP processing. * rgerhards, 2008-10-23 */ -static inline void dynaFileFreeCacheEntries(instanceData *pData) +static inline void +dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -562,48 +581,32 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } -/* rgerhards 2004-11-11: write to a file output. This - * will be called for all outputs using file semantics, - * for example also for pipes. +/* physically write the file */ -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal +doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) { off_t actualFileSize; int iLenWritten; DEFiRet; - ASSERT(pData != NULL); - /* first check if we have a dynamic file name and, if so, - * check if it still is ok or a new file needs to be created - */ - if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - /* create the message based on format specified */ +dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); again: /* check if we have a file size limit and, if so, * obey to it. */ if(pData->f_sizeLimit != 0) { - actualFileSize = lseek(pData->fd, 0, SEEK_END); + actualFileSize = lseek(fd, 0, SEEK_END); if(actualFileSize >= pData->f_sizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having * us aborted by the OS... rgerhards 2005-06-21 */ - (void) close(pData->fd); + (void) close(fd); /* try to resolve the situation */ + // TODO: *doesn't work, will need to use new fd ! if(resolveFileSizeLimit(pData) != 0) { /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), @@ -622,13 +625,12 @@ again: } } - iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); -//dbgprintf("lenwritten: %d\n", iLenWritten); + iLenWritten = write(fd, pszBuf, lenBuf); if(iLenWritten < 0) { int e = errno; char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); + DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); /* If a named pipe is full, we suspend this action for a while */ if(pData->fileType == eTypePIPE && e == EAGAIN) @@ -667,9 +669,159 @@ again: errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } } else if (pData->bSyncFile) { - fsync(pData->fd); + fsync(fd); + } + + pData->poBuf->iBuf = 0; + +finalize_it: + RETiRet; +} + + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + */ +static rsRetVal +doZipWrite(instanceData *pData) +{ + outbuf_t *poBuf; + z_stream strm; + int zRet; /* zlib return state */ + DEFiRet; + assert(pData != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ + strm = poBuf->zStrm; /* another shortcut */ + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + zRet = zlibw.DeflateInit(&strm, 9); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateInit() done successfully\n"); + + /* now doing the compression */ + strm.avail_in = poBuf->iBuf; + strm.next_in = (Bytef*) poBuf->pszBuf; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); + strm.avail_out = OUTBUF_LEN; + strm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + +RUNLOG_STR("deflate() should be done successfully\n"); + + zRet = zlibw.DeflateEnd(&strm); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateEnd() done successfully\n"); + +finalize_it: + RETiRet; +} + + +/* flush the output buffer + */ +static rsRetVal +doFlush(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + + if(pData->poBuf->iBuf == 0) + FINALIZE; /* nothing to write, but make this a valid case */ + + if(1) { // zlib enabled! + CHKiRet(doZipWrite(pData)); + } else { + CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); + } + +finalize_it: + RETiRet; +} + + +/* do the actual write process. This function is to be called once we are ready for writing. + * It will do buffered writes and persist data only when the buffer is full. Note that we must + * be careful to detect when the file handle changed. + * rgerhards, 2009-06-03 + */ +static rsRetVal +doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) +{ + int i; + outbuf_t *poBuf; + DEFiRet; + ASSERT(pData != NULL); + ASSERT(pszBuf != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ +dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + + if(pData->fd != poBuf->fd) { + // TODO: more efficient use for dynafiles + CHKiRet(doFlush(pData)); + poBuf->fd = pData->fd; + } + + for(i = 0 ; i < lenBuf ; ++i) { + poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; + if(poBuf->iBuf == poBuf->lenBuf) { + CHKiRet(doFlush(pData)); + } + } + +finalize_it: + RETiRet; +} + + +/* rgerhards 2004-11-11: write to a file output. This + * will be called for all outputs using file semantics, + * for example also for pipes. + */ +static rsRetVal +writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + DEFiRet; + + ASSERT(pData != NULL); + + /* first check if we have a dynamic file name and, if so, + * check if it still is ok or a new file needs to be created + */ + if(pData->bDynamicName) { + if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + } + + if(pData->fd == -1) { + rsRetVal iRetLocal; + iRetLocal = prepareFile(pData, pData->f_fname); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } + /* create the message based on format specified */ + CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); + finalize_it: RETiRet; } @@ -678,15 +830,20 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->fd = -1; + CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + pData->poBuf->lenBuf = OUTBUF_LEN; +finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance + doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) close(pData->fd); + free(pData->poBuf); ENDfreeInstance @@ -696,7 +853,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->f_fname); + DBGPRINTF("file to log to: %s\n", pData->f_fname); iRet = writeFile(ppString, iMsgOpts, pData); ENDdoAction @@ -875,8 +1032,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit - if(pszTplName != NULL) - free(pszTplName); + objRelease(zlibw, LM_ZLIBW_FILENAME); + free(pszTplName); ENDmodExit @@ -891,6 +1048,7 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); -- cgit From 69eac5341d49c5c7278e6d86a77dc9d6b4a4edde Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 15:32:39 +0200 Subject: added gzip header to output files so they can now be processed with the "regular" gzip tools --- tools/omfile.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 62a044d8..08c33a7f 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -14,6 +14,29 @@ * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * * This file is part of rsyslog. * * Rsyslog is free software: you can redistribute it and/or modify @@ -698,12 +721,13 @@ doZipWrite(instanceData *pData) strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; - zRet = zlibw.DeflateInit(&strm, 9); + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&strm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateInit() done successfully\n"); +RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ strm.avail_in = poBuf->iBuf; -- cgit From 768836ab79268e1091fc3af8bb920c03287db91a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 11:05:41 +0200 Subject: reduced max value for $DynaFileCacheSize to 1,000 The former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). Also, some general cleanup was done. --- tools/omfile.c | 66 +++++++++++++++++++++++++--------------------------------- 1 file changed, 28 insertions(+), 38 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 08c33a7f..d1274f1b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,6 +12,12 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * + * A large re-write of this file was done in June, 2009. The focus was + * to introduce many more features (like zipped writing), clean up the code + * and make it more reliable. In short, that rewrite tries to provide a new + * solid basis for the next three to five years to come. During it, bugs + * may have been introduced ;) -- rgerhards, 2009-06-04 + * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * An important note on writing gzip format via zlib (kept anonymous @@ -158,7 +164,7 @@ typedef struct _instanceData { */ dynaFileCacheEntry **dynCache; off_t f_sizeLimit; /* file size limit, 0 = no limit */ - char *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ } instanceData; @@ -208,13 +214,13 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; iNewVal = 1; - } else if(iNewVal > 10000) { + } else if(iNewVal > 1000) { snprintf((char*) errMsg, sizeof(errMsg)/sizeof(uchar), - "DynaFileCacheSize maximum is 10,000 (%d given), changed to 10,000.", iNewVal); + "DynaFileCacheSize maximum is 1,000 (%d given), changed to 1,000.", iNewVal); errno = 0; errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; - iNewVal = 10000; + iNewVal = 1000; } iDynaFileCacheSize = iNewVal; @@ -286,7 +292,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit; + pData->f_sizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -322,7 +328,7 @@ int resolveFileSizeLimit(instanceData *pData) * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. */ - if((pCmd = (uchar*)strdup((char*)pData->f_sizeLimitCmd)) == NULL) { + if((pCmd = ustrdup(pData->f_sizeLimitCmd)) == NULL) { /* there is not much we can do - we make syslogd close the file in this case */ return 1; } @@ -372,7 +378,7 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) FINALIZE; DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, - pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName); + pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName); /* if the name is NULL, this is an improperly initilized entry which * needs to be discarded. In this case, neither the file is to be closed * not the name to be freed. @@ -452,7 +458,7 @@ prepareFile(instanceData *pData, uchar *newFileName) * We do not report any errors here ourselfs but let the code * fall through to error handler below. */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + if(makeFileParentDirs(newFileName, ustrlen(newFileName), pData->fDirCreateMode, pData->dirUID, pData->dirGID, pData->bFailOnChown) != 0) { ABORT_FINALIZE(RS_RET_ERR); /* we give up */ @@ -523,7 +529,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * I *hope* this will be a performance enhancement. */ if( (pData->iCurrElt != -1) - && !strcmp((char*) newFileName, (char*) pCache[pData->iCurrElt]->pName)) { + && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ return 0; @@ -540,7 +546,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iFirstFree == -1) iFirstFree = i; } else { /* got an element, let's see if it matches */ - if(!strcmp((char*) newFileName, (char*) pCache[i]->pName)) { + if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ pData->fd = pCache[i]->fd; pData->iCurrElt = i; @@ -585,7 +591,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iMsgOpts & INTERNAL_MSG) { DBGPRINTF("Could not open dynaFile, discarding message\n"); } else { - errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName); + errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); pData->iCurrElt = -1; @@ -593,7 +599,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */ + pCache[iFirstFree]->pName = ustrdup(newFileName); /* TODO: check for NULL (very unlikely) */ pCache[iFirstFree]->lastUsed = time(NULL); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); @@ -884,21 +890,10 @@ ENDdoAction BEGINparseSelectorAct CODESTARTparseSelectorAct - /* yes, the if below is redundant, but I need it now. Will go away as - * the code further changes. -- rgerhards, 2007-07-25 - */ - if(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-') { - if((iRet = createInstance(&pData)) != RS_RET_OK) { - ENDfunc - return iRet; /* this can not use RET_iRet! */ - } - } else { - /* this is not clean, but we need it for the time being - * TODO: remove when cleaning up modularization - */ - ENDfunc - return RS_RET_CONFLINE_UNPROCESSED; - } + if(!(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-')) + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + + CHKiRet(createInstance(&pData)); if(*p == '-') { pData->bSyncFile = 0; @@ -933,15 +928,12 @@ CODESTARTparseSelectorAct */ CODE_STD_STRING_REQUESTparseSelectorAct(2) ++p; /* eat '?' */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); /* "filename" is actually a template name, we need this as string 1. So let's add it * to the pOMSR. -- rgerhards, 2007-07-27 */ - if((iRet = OMSRsetEntry(*ppOMSR, 1, (uchar*)strdup((char*) pData->f_fname), OMSR_NO_RQD_TPL_OPTS)) != RS_RET_OK) - break; + CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pData->f_fname), OMSR_NO_RQD_TPL_OPTS)); pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ @@ -979,10 +971,8 @@ CODESTARTparseSelectorAct * to use is specified. So we need to scan for the first coma first * and then look at the rest of the line. */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); pData->bDynamicName = 0; pData->fCreateMode = fCreateMode; /* preserve current setting */ @@ -1005,7 +995,7 @@ CODESTARTparseSelectorAct errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); break; } - if(strcmp((char*) p, _PATH_CONSOLE) == 0) + if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) pData->fileType = eTypeCONSOLE; break; default: -- cgit From 9e434f19a9baa4a6f411808b5cb6bc22d6a32781 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 12:15:59 +0200 Subject: cleaned up stream class ... ... and also made it callable via an rsyslog interface rather then relying on the OS loader (important if we go for using it inside loadbale modules, which we soon possible will) --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index d1274f1b..689bad0c 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -776,7 +776,7 @@ doFlush(instanceData *pData) if(pData->poBuf->iBuf == 0) FINALIZE; /* nothing to write, but make this a valid case */ - if(1) { // zlib enabled! + if(0) { // zlib enabled! CHKiRet(doZipWrite(pData)); } else { CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); -- cgit From f2800ba261d2fb7466cbdebbf80afe92f0bffd3d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 15:10:24 +0200 Subject: modified stream class and omfile to work with it now some basic operations are carried out via the stream class. --- tools/omfile.c | 94 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 26 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 689bad0c..3f517906 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -70,6 +70,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,7 @@ #include "module-template.h" #include "errmsg.h" #include "unicode-helper.h" +#include "stream.h" #include "zlibw.h" MODULE_TYPE_OUTPUT @@ -96,6 +98,7 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(zlibw) +DEFobjCurrIf(strm) /* The following structure is a dynafile name cache entry. */ @@ -131,12 +134,14 @@ static uid_t dirUID; /* UID to be used for newly created directories */ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ +static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ + strm_t *pStrm; /* our output stream */ short fd; /* file descriptor for (current) file */ outbuf_t *poBuf; /* output buffer */ enum { @@ -165,6 +170,7 @@ typedef struct _instanceData { dynaFileCacheEntry **dynCache; off_t f_sizeLimit; /* file size limit, 0 = no limit */ uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + int iZipLevel; /* zip mode to use for this selector */ } instanceData; @@ -446,13 +452,11 @@ prepareFile(instanceData *pData, uchar *newFileName) FINALIZE; /* we are done in this case */ } - if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } else { - pData->fd = -1; + // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 + + if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ + pData->fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code @@ -485,8 +489,29 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } + close(pData->fd); /* close again, as we need a stream further on */ } } + + char szNameBuf[MAXFNAME]; + char szDirName[MAXFNAME]; + char szBaseName[MAXFNAME]; + strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szDirName, dirname(szNameBuf)); + strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szBaseName, basename(szNameBuf)); +DBGPRINTF("XXX: name to set: '%s', dirname '%s'\n", pData->f_fname, szDirName); + + CHKiRet(strm.Construct(&pData->pStrm)); + CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); + CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); + CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.ConstructFinalize(pData->pStrm)); + + pData->fd = 2; /* TODO: dummy to keep current inconsistent code happy - remove later */ + finalize_it: /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean * non-open file descriptor. Anyhow, I leave this comment for the time being to that if @@ -715,20 +740,20 @@ static rsRetVal doZipWrite(instanceData *pData) { outbuf_t *poBuf; - z_stream strm; + z_stream zstrm; int zRet; /* zlib return state */ DEFiRet; assert(pData != NULL); poBuf = pData->poBuf; /* use as a shortcut */ - strm = poBuf->zStrm; /* another shortcut */ + zstrm = poBuf->zStrm; /* another shortcut */ /* allocate deflate state */ - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&strm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + zRet = zlibw.DeflateInit2(&zstrm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); @@ -736,24 +761,24 @@ doZipWrite(instanceData *pData) RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ - strm.avail_in = poBuf->iBuf; - strm.next_in = (Bytef*) poBuf->pszBuf; + zstrm.avail_in = poBuf->iBuf; + zstrm.next_in = (Bytef*) poBuf->pszBuf; /* run deflate() on input until output buffer not full, finish compression if all of source has been read in */ do { - dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); - strm.avail_out = OUTBUF_LEN; - strm.next_out = (Bytef*) poBuf->zipBuf; - zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ - dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = OUTBUF_LEN; + zstrm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); - } while (strm.avail_out == 0); - assert(strm.avail_in == 0); /* all input will be used */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ RUNLOG_STR("deflate() should be done successfully\n"); - zRet = zlibw.DeflateEnd(&strm); + zRet = zlibw.DeflateEnd(&zstrm); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); @@ -802,8 +827,14 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pszBuf != NULL); poBuf = pData->poBuf; /* use as a shortcut */ -dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", -pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); +dbgprintf("doWrite, pData->fd %d, pData->pStrm %p, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->pStrm, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + +RUNLOG_VAR("%p", pData->pStrm); + if(pData->pStrm != NULL){ + CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); + FINALIZE; // TODO: clean up later + } if(pData->fd != poBuf->fd) { // TODO: more efficient use for dynafiles @@ -869,6 +900,10 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ + if(pData->pStrm != NULL) { +RUNLOG_STR("XXX: destructing stream"); + strm.Destruct(&pData->pStrm); + } if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) @@ -945,6 +980,7 @@ CODESTARTparseSelectorAct pData->fileGID = fileGID; pData->dirUID = dirUID; pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ /* we now allocate the cache table. We use calloc() intentionally, as we * need all pointers to be initialized to NULL pointers. @@ -983,6 +1019,7 @@ CODESTARTparseSelectorAct pData->fileGID = fileGID; pData->dirUID = dirUID; pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; /* at this stage, we ignore the return value of prepareFile, this is taken * care of in later steps. -- rgerhards, 2009-03-19 @@ -1021,6 +1058,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a fDirCreateMode = 0700; bCreateDirs = 1; bEnableSync = 0; + iZipLevel = 0; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -1046,6 +1084,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); + objRelease(strm, CORE_COMPONENT); objRelease(zlibw, LM_ZLIBW_FILENAME); free(pszTplName); ENDmodExit @@ -1062,9 +1102,11 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr - CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); + CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); -- cgit From 76da7f9f4e3dc900046a1956153d10532f2b1ae0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 15:59:37 +0200 Subject: added $OMFileIOBufferSize config directive and plumbing --- tools/omfile.c | 114 ++++++++++++++++++++++++++------------------------------- 1 file changed, 52 insertions(+), 62 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 3f517906..85f04f13 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -123,6 +123,8 @@ typedef struct { } outbuf_t; +#define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ static int fCreateMode = 0644; /* mode to use when creating files */ @@ -135,6 +137,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -168,9 +171,10 @@ typedef struct _instanceData { * pointer points to the overall structure. */ dynaFileCacheEntry **dynCache; - off_t f_sizeLimit; /* file size limit, 0 = no limit */ - uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *iSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ + int iIOBufSize; /* size of associated io buffer */ } instanceData; @@ -293,12 +297,12 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR } /* OK, we finally got a correct template. So let's use it... */ - strncpy((char*) pData->f_fname, (char*) pOch->pszFileTemplate, MAXFNAME); - pData->f_sizeLimit = pOch->uSizeLimit; + ustrncpy(pData->f_fname, pOch->pszFileTemplate, MAXFNAME); + pData->iSizeLimit = pOch->uSizeLimit; /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = pOch->cmdOnSizeLimit; + pData->iSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -322,7 +326,7 @@ int resolveFileSizeLimit(instanceData *pData) off_t actualFileSize; ASSERT(pData != NULL); - if(pData->f_sizeLimitCmd == NULL) + if(pData->iSizeLimitCmd == NULL) return 1; /* nothing we can do in this case... */ /* the execProg() below is probably not great, but at least is is @@ -334,7 +338,7 @@ int resolveFileSizeLimit(instanceData *pData) * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. */ - if((pCmd = ustrdup(pData->f_sizeLimitCmd)) == NULL) { + if((pCmd = ustrdup(pData->iSizeLimitCmd)) == NULL) { /* there is not much we can do - we make syslogd close the file in this case */ return 1; } @@ -357,7 +361,7 @@ int resolveFileSizeLimit(instanceData *pData) pData->fCreateMode); actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { + if(actualFileSize >= pData->iSizeLimit) { /* OK, it didn't work out... */ return 1; } @@ -500,12 +504,14 @@ prepareFile(instanceData *pData, uchar *newFileName) strcpy(szDirName, dirname(szNameBuf)); strcpy(szNameBuf, (char*)pData->f_fname); strcpy(szBaseName, basename(szNameBuf)); -DBGPRINTF("XXX: name to set: '%s', dirname '%s'\n", pData->f_fname, szDirName); CHKiRet(strm.Construct(&pData->pStrm)); CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); +dbgprintf("static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); +RUNLOG_VAR("%d", pData->iIOBufSize); + CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); @@ -650,9 +656,9 @@ again: /* check if we have a file size limit and, if so, * obey to it. */ - if(pData->f_sizeLimit != 0) { + if(pData->iSizeLimit != 0) { actualFileSize = lseek(fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { + if(actualFileSize >= pData->iSizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having @@ -665,14 +671,14 @@ again: /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); + pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); errno = 0; errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } else { snprintf(errMsg, sizeof(errMsg), "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); + pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); errno = 0; errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); } @@ -901,7 +907,6 @@ BEGINfreeInstance CODESTARTfreeInstance doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->pStrm != NULL) { -RUNLOG_STR("XXX: destructing stream"); strm.Destruct(&pData->pStrm); } if(pData->bDynamicName) { @@ -934,12 +939,11 @@ CODESTARTparseSelectorAct pData->bSyncFile = 0; p++; } else { - pData->bSyncFile = bEnableSync ? 1 : 0; + pData->bSyncFile = bEnableSync; } - pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */ + pData->iSizeLimit = 0; /* default value, use outchannels to configure! */ - switch (*p) - { + switch(*p) { case '$': CODE_STD_STRING_REQUESTparseSelectorAct(1) /* rgerhards 2005-06-21: this is a special setting for output-channel @@ -949,13 +953,8 @@ CODESTARTparseSelectorAct * rgerhards, 2007-07-24: output-channels will go away. We keep them * for compatibility reasons, but seems to have been a bad idea. */ - if((iRet = cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)) == RS_RET_OK) { - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } + CHKiRet(cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)); + pData->bDynamicName = 0; break; case '?': /* This is much like a regular file handle, but we need to obtain @@ -972,16 +971,6 @@ CODESTARTparseSelectorAct pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - pData->fCreateMode = fCreateMode; /* freeze current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iZipLevel = iZipLevel; - pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ /* we now allocate the cache table. We use calloc() intentionally, as we * need all pointers to be initialized to NULL pointers. */ @@ -995,50 +984,49 @@ CODESTARTparseSelectorAct case '|': case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards, 2007-0726: first check if file or pipe */ if(*p == '|') { pData->fileType = eTypePIPE; ++p; } else { pData->fileType = eTypeFILE; } - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iZipLevel = iZipLevel; - - /* at this stage, we ignore the return value of prepareFile, this is taken - * care of in later steps. -- rgerhards, 2009-03-19 + break; + default: + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* freeze current paremeters for this action */ + pData->iDynaFileCacheSize = iDynaFileCacheSize; + pData->fCreateMode = fCreateMode; + pData->fDirCreateMode = fDirCreateMode; + pData->bCreateDirs = bCreateDirs; + pData->bFailOnChown = bFailOnChown; + pData->fileUID = fileUID; + pData->fileGID = fileGID; + pData->dirUID = dirUID; + pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; + pData->iIOBufSize = iIOBufSize; + + if(pData->bDynamicName == 0) { + if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) + pData->fileType = eTypeCONSOLE; + /* try open and emit error message if not possible. At this stage, we ignore the + * return value of prepareFile, this is taken care of in later steps. */ prepareFile(pData, pData->f_fname); if(pData->fd < 0 ) { - pData->fd = -1; DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); - break; } - if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) - pData->fileType = eTypeCONSOLE; - break; - default: - iRet = RS_RET_CONFLINE_UNPROCESSED; - break; } + +dbgprintf("in init static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); +RUNLOG_VAR("%d", pData->iIOBufSize); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -1059,6 +1047,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; + iIOBufSize = IOBUF_DFLT_SIZE; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -1107,6 +1096,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); -- cgit From 68f0ffb29acf5ff07ccd8f30581d96d6915a834e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jun 2009 15:41:05 +0200 Subject: made dynafiles working again now only stream class is utilized. ttys, pipes and outchannel functionality is currently disabled. But the testbench worked again. Cleanup needed, will do this with next commit (it may break things and I like to have this milestone here). --- tools/omfile.c | 148 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 73 insertions(+), 75 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 85f04f13..dd585893 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -104,7 +104,7 @@ DEFobjCurrIf(strm) */ struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ - short fd; /* name associated with file name in cache */ + strm_t *pStrm; /* our output stream */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; @@ -116,7 +116,6 @@ typedef struct { uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ size_t lenBuf; /* max size of buffer */ size_t iBuf; /* current buffer index */ - int fd; /* which file descriptor is this buf for? */ /* elements for zip writing */ z_stream zStrm; char zipBuf[OUTBUF_LEN]; @@ -145,8 +144,7 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - short fd; /* file descriptor for (current) file */ - outbuf_t *poBuf; /* output buffer */ + //outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -203,7 +201,7 @@ CODESTARTdbgPrintInstInfo ); } else { /* regular file */ dbgprintf("%s", pData->f_fname); - if (pData->fd == -1) + if (pData->pStrm == NULL) dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -312,6 +310,7 @@ finalize_it: } +#if 0 /* rgerhards 2005-06-21: Try to resolve a size limit * situation. This first runs the command, and then * checks if we are still above the treshold. @@ -368,6 +367,7 @@ int resolveFileSizeLimit(instanceData *pData) return 0; } +#endif /* This function deletes an entry from the dynamic file name @@ -377,13 +377,12 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void +static rsRetVal dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { + DEFiRet; ASSERT(pCache != NULL); - BEGINfunc; - if(pCache[iEntry] == NULL) FINALIZE; @@ -394,7 +393,8 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) * not the name to be freed. */ if(pCache[iEntry]->pName != NULL) { - close(pCache[iEntry]->fd); + if(pCache[iEntry]->pStrm != NULL) + strm.Destruct(&pCache[iEntry]->pStrm); d_free(pCache[iEntry]->pName); pCache[iEntry]->pName = NULL; } @@ -405,7 +405,7 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) } finalize_it: - ENDfunc; + RETiRet; } @@ -450,17 +450,20 @@ static void dynaFileFreeCache(instanceData *pData) static rsRetVal prepareFile(instanceData *pData, uchar *newFileName) { + int fd; DEFiRet; +#if 0 if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); + fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); FINALIZE; /* we are done in this case */ } +#endif // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ - pData->fd = -1; + fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code @@ -475,17 +478,17 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, + fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); - if(pData->fd != -1) { + if(fd != -1) { /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { + if(fchown(fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(fd); + fd = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -493,16 +496,16 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } - close(pData->fd); /* close again, as we need a stream further on */ + close(fd); /* close again, as we need a stream further on */ } } char szNameBuf[MAXFNAME]; char szDirName[MAXFNAME]; char szBaseName[MAXFNAME]; - strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szNameBuf, (char*)newFileName); strcpy(szDirName, dirname(szNameBuf)); - strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szNameBuf, (char*)newFileName); strcpy(szBaseName, basename(szNameBuf)); CHKiRet(strm.Construct(&pData->pStrm)); @@ -516,18 +519,18 @@ RUNLOG_VAR("%d", pData->iIOBufSize); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); - pData->fd = 2; /* TODO: dummy to keep current inconsistent code happy - remove later */ - finalize_it: - /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean + /* this was "fd != 0", which I think was a bug. I guess 0 was intended to mean * non-open file descriptor. Anyhow, I leave this comment for the time being to that if * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 */ - if(pData->fd != -1 && isatty(pData->fd)) { - DBGPRINTF("file %d is a tty file\n", pData->fd); +#if 0 // TODO: this must be done by stream class! + if(fd != -1 && isatty(fd)) { + DBGPRINTF("file %d is a tty file\n", fd); pData->fileType = eTypeTTY; untty(); } +#endif RETiRet; } @@ -541,15 +544,16 @@ finalize_it: * be written. * This is a helper to writeFile(). rgerhards, 2007-07-03 */ -static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) +static inline rsRetVal +prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) { time_t ttOldest; /* timestamp of oldest element */ int iOldest; int i; int iFirstFree; + rsRetVal localRet; dynaFileCacheEntry **pCache; - - BEGINfunc + DEFiRet; ASSERT(pData != NULL); ASSERT(newFileName != NULL); @@ -562,8 +566,9 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if( (pData->iCurrElt != -1) && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ - pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call! + // LRU needs only a strictly monotonically increasing counter, so such a one could do + FINALIZE; } /* ok, no luck. Now let's search the table if we find a matching spot. @@ -579,10 +584,10 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } else { /* got an element, let's see if it matches */ if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ - pData->fd = pCache[i]->fd; + pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + FINALIZE; } /* did not find it - so lets keep track of the counters for LRU */ if(pCache[i]->lastUsed < ttOldest) { @@ -598,23 +603,20 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg iFirstFree = pData->iCurrCacheSize++; } + pData->iCurrElt = -1; if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ } else { /* we need to allocate memory for the cache structure */ - pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)); - if(pCache[iFirstFree] == NULL) { - DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n"); - return -1; - } + CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry))); } /* Ok, we finally can open the file */ - prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ + localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ /* file is either open now or an error state set */ - if(pData->fd == -1) { + if(pData->pStrm == NULL) { /* do not report anything if the message is an internally-generated * message. Otherwise, we could run into a never-ending loop. The bad * news is that we also lose errors on startup messages, but so it is. @@ -625,22 +627,21 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); - pData->iCurrElt = -1; - return -1; + ABORT_FINALIZE(localRet); } - pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = ustrdup(newFileName); /* TODO: check for NULL (very unlikely) */ - pCache[iFirstFree]->lastUsed = time(NULL); + CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName)); + pCache[iFirstFree]->pStrm = pData->pStrm; + pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); - ENDfunc - - return 0; +finalize_it: + RETiRet; } +#if 0 /* physically write the file */ static rsRetVal @@ -816,6 +817,7 @@ doFlush(instanceData *pData) finalize_it: RETiRet; } +#endif /* do the actual write process. This function is to be called once we are ready for writing. @@ -826,15 +828,12 @@ finalize_it: static rsRetVal doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) { - int i; - outbuf_t *poBuf; DEFiRet; ASSERT(pData != NULL); ASSERT(pszBuf != NULL); - poBuf = pData->poBuf; /* use as a shortcut */ -dbgprintf("doWrite, pData->fd %d, pData->pStrm %p, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", -pData->fd, pData->pStrm, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", +pData->pStrm, lenBuf); RUNLOG_VAR("%p", pData->pStrm); if(pData->pStrm != NULL){ @@ -842,6 +841,7 @@ RUNLOG_VAR("%p", pData->pStrm); FINALIZE; // TODO: clean up later } +#if 0 if(pData->fd != poBuf->fd) { // TODO: more efficient use for dynafiles CHKiRet(doFlush(pData)); @@ -854,6 +854,7 @@ RUNLOG_VAR("%p", pData->pStrm); CHKiRet(doFlush(pData)); } } +#endif finalize_it: RETiRet; @@ -875,45 +876,42 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) * check if it still is ok or a new file needs to be created */ if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts)); + } else { /* "regular", non-dynafile */ + if(pData->pStrm == NULL) { + CHKiRet(prepareFile(pData, pData->f_fname)); + } } /* create the message based on format specified */ CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: + if(iRet != RS_RET_OK) { + /* in v5, we shall return different states for message-cause failur (but only there!) */ + iRet = RS_RET_SUSPENDED; + } RETiRet; } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; - CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); - pData->poBuf->lenBuf = OUTBUF_LEN; + pData->pStrm = NULL; + //CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + //pData->poBuf->lenBuf = OUTBUF_LEN; finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ - if(pData->pStrm != NULL) { - strm.Destruct(&pData->pStrm); - } + //DEL! doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); - } else if(pData->fd != -1) - close(pData->fd); - free(pData->poBuf); + } else if(pData->pStrm != NULL) + strm.Destruct(&pData->pStrm); + //free(pData->poBuf); ENDfreeInstance @@ -1019,7 +1017,7 @@ CODESTARTparseSelectorAct */ prepareFile(pData, pData->f_fname); - if(pData->fd < 0 ) { + if(pData->pStrm == NULL) { DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); } @@ -1063,9 +1061,9 @@ CODESTARTdoHUP dynaFileFreeCacheEntries(pData); pData->iCurrElt = -1; /* invalidate current element */ } else { - if(pData->fd != -1) { - close(pData->fd); - pData->fd = -1; + if(pData->pStrm != NULL) { + strm.Destruct(&pData->pStrm); + pData->pStrm = NULL; } } ENDdoHUP -- cgit From 84b786753744d9f7a002665571a428283feae248 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jun 2009 18:41:28 +0200 Subject: cleanup in preparation of next steps now cleand up omfile and straighted out some things. The only commented-out code left is code that must be moved/merged to the stream class, my next target. --- tools/omfile.c | 174 ++++----------------------------------------------------- 1 file changed, 10 insertions(+), 164 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index dd585893..f196d602 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -20,29 +20,6 @@ * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * - * An important note on writing gzip format via zlib (kept anonymous - * by request): - * - * -------------------------------------------------------------------------- - * We'd like to make sure the output file is in full gzip format - * (compatible with gzip -d/zcat etc). There is a flag in how the output - * is initialized within zlib to properly add the gzip wrappers to the - * output. (gzip is effectively a small metadata wrapper around raw - * zstream output.) - * - * I had written an old bit of code to do this - the documentation on - * deflatInit2() was pretty tricky to nail down on this specific feature: - * - * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, - * int memLevel, int strategy); - * - * I believe "31" would be the value for the "windowBits" field that you'd - * want to try: - * - * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - * -------------------------------------------------------------------------- - * - * * This file is part of rsyslog. * * Rsyslog is free software: you can redistribute it and/or modify @@ -105,22 +82,10 @@ DEFobjCurrIf(strm) struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ strm_t *pStrm; /* our output stream */ - time_t lastUsed; /* for LRU - last access */ + time_t lastUsed; /* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; -/* the output buffer structure */ -// TODO: later make this part of the dynafile cache! -#define OUTBUF_LEN 128 // TODO: make dynamic! -typedef struct { - uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ - size_t lenBuf; /* max size of buffer */ - size_t iBuf; /* current buffer index */ - /* elements for zip writing */ - z_stream zStrm; - char zipBuf[OUTBUF_LEN]; -} outbuf_t; - #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ @@ -144,7 +109,6 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - //outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -152,10 +116,10 @@ typedef struct _instanceData { eTypePIPE } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ - int fCreateMode; /* file creation mode for open() */ + int fCreateMode; /* file creation mode for open() */ // TODO: shuffle down to stream class int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ - int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ + int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -452,14 +416,8 @@ prepareFile(instanceData *pData, uchar *newFileName) { int fd; DEFiRet; -#if 0 - if(pData->fileType == eTypePIPE) { - fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); - FINALIZE; /* we are done in this case */ - } -#endif - // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 + // TODO: handle TTY/PIPE case! (in stream.c!) 2009-06-04 if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ @@ -512,8 +470,6 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); -dbgprintf("static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); -RUNLOG_VAR("%d", pData->iIOBufSize); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); @@ -574,6 +530,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) /* ok, no luck. Now let's search the table if we find a matching spot. * While doing so, we also prepare for creation of a new one. */ + pData->iCurrElt = -1; /* invalid current element pointer */ iFirstFree = -1; /* not yet found */ iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ ttOldest = time(NULL) + 1; /* there must always be an older one */ @@ -603,7 +560,6 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) iFirstFree = pData->iCurrCacheSize++; } - pData->iCurrElt = -1; if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ @@ -642,6 +598,7 @@ finalize_it: #if 0 +// TODO: mirgrate code below to stream class (update its write handler, which is not great!) /* physically write the file */ static rsRetVal @@ -738,85 +695,6 @@ again: finalize_it: RETiRet; } - - -/* write the output buffer in zip mode - * This means we compress it first and then do a physical write. - */ -static rsRetVal -doZipWrite(instanceData *pData) -{ - outbuf_t *poBuf; - z_stream zstrm; - int zRet; /* zlib return state */ - DEFiRet; - assert(pData != NULL); - - poBuf = pData->poBuf; /* use as a shortcut */ - zstrm = poBuf->zStrm; /* another shortcut */ - - /* allocate deflate state */ - zstrm.zalloc = Z_NULL; - zstrm.zfree = Z_NULL; - zstrm.opaque = Z_NULL; - /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&zstrm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } -RUNLOG_STR("deflateInit2() done successfully\n"); - - /* now doing the compression */ - zstrm.avail_in = poBuf->iBuf; - zstrm.next_in = (Bytef*) poBuf->pszBuf; - /* run deflate() on input until output buffer not full, finish - compression if all of source has been read in */ - do { - dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); - zstrm.avail_out = OUTBUF_LEN; - zstrm.next_out = (Bytef*) poBuf->zipBuf; - zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ - dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); - assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - zstrm.avail_out)); - } while (zstrm.avail_out == 0); - assert(zstrm.avail_in == 0); /* all input will be used */ - -RUNLOG_STR("deflate() should be done successfully\n"); - - zRet = zlibw.DeflateEnd(&zstrm); - if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } -RUNLOG_STR("deflateEnd() done successfully\n"); - -finalize_it: - RETiRet; -} - - -/* flush the output buffer - */ -static rsRetVal -doFlush(instanceData *pData) -{ - DEFiRet; - assert(pData != NULL); - - if(pData->poBuf->iBuf == 0) - FINALIZE; /* nothing to write, but make this a valid case */ - - if(0) { // zlib enabled! - CHKiRet(doZipWrite(pData)); - } else { - CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); - } - -finalize_it: - RETiRet; -} #endif @@ -832,30 +710,12 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pData != NULL); ASSERT(pszBuf != NULL); -dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", -pData->pStrm, lenBuf); - -RUNLOG_VAR("%p", pData->pStrm); +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); FINALIZE; // TODO: clean up later } -#if 0 - if(pData->fd != poBuf->fd) { - // TODO: more efficient use for dynafiles - CHKiRet(doFlush(pData)); - poBuf->fd = pData->fd; - } - - for(i = 0 ; i < lenBuf ; ++i) { - poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; - if(poBuf->iBuf == poBuf->lenBuf) { - CHKiRet(doFlush(pData)); - } - } -#endif - finalize_it: RETiRet; } @@ -883,7 +743,6 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) } } - /* create the message based on format specified */ CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: @@ -898,20 +757,15 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->pStrm = NULL; - //CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); - //pData->poBuf->lenBuf = OUTBUF_LEN; -finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - //DEL! doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->pStrm != NULL) strm.Destruct(&pData->pStrm); - //free(pData->poBuf); ENDfreeInstance @@ -969,14 +823,9 @@ CODESTARTparseSelectorAct pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - /* we now allocate the cache table. We use calloc() intentionally, as we - * need all pointers to be initialized to NULL pointers. - */ - if((pData->dynCache = (dynaFileCacheEntry**) - calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n"); - } + /* we now allocate the cache table */ + CHKmalloc(pData->dynCache = (dynaFileCacheEntry**) + calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))); break; case '|': @@ -1022,9 +871,6 @@ CODESTARTparseSelectorAct errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); } } - -dbgprintf("in init static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); -RUNLOG_VAR("%d", pData->iIOBufSize); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct -- cgit From 8d1e2e496c6a4a4d40d1e8604c746e0d32013536 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Jun 2009 18:39:06 +0200 Subject: improved error handling and added fsync() support ... restoring missing functionality after the restructuring of imfile. As a side-effect, this also lays the foundation for even more reliable queue engine operations (but this is not yet done). --- tools/omfile.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index f196d602..e3f7af19 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -116,7 +116,7 @@ typedef struct _instanceData { eTypePIPE } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ - int fCreateMode; /* file creation mode for open() */ // TODO: shuffle down to stream class + int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! @@ -472,6 +472,8 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); + CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); -- cgit From 0917edf26da9055c6dc160aafca97896daea3e6c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 09:47:44 +0200 Subject: re-enabled outchannel functionality --- tools/omfile.c | 91 ++++++++++++---------------------------------------------- 1 file changed, 19 insertions(+), 72 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index e3f7af19..cf020f7b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -67,6 +67,7 @@ #include "unicode-helper.h" #include "stream.h" #include "zlibw.h" +#include "unicode-helper.h" MODULE_TYPE_OUTPUT @@ -119,7 +120,7 @@ typedef struct _instanceData { int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ - int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! + int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -134,7 +135,7 @@ typedef struct _instanceData { */ dynaFileCacheEntry **dynCache; off_t iSizeLimit; /* file size limit, 0 = no limit */ - uchar *iSizeLimitCmd; /* command to carry out when size limit is reached */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ } instanceData; @@ -264,7 +265,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->iSizeLimitCmd = pOch->cmdOnSizeLimit; + pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -274,66 +275,6 @@ finalize_it: } -#if 0 -/* rgerhards 2005-06-21: Try to resolve a size limit - * situation. This first runs the command, and then - * checks if we are still above the treshold. - * returns 0 if ok, 1 otherwise - * TODO: consider moving the initial check in here, too - */ -int resolveFileSizeLimit(instanceData *pData) -{ - uchar *pParams; - uchar *pCmd; - uchar *p; - off_t actualFileSize; - ASSERT(pData != NULL); - - if(pData->iSizeLimitCmd == NULL) - return 1; /* nothing we can do in this case... */ - - /* the execProg() below is probably not great, but at least is is - * fairly secure now. Once we change the way file size limits are - * handled, we should also revisit how this command is run (and - * with which parameters). rgerhards, 2007-07-20 - */ - /* we first check if we have command line parameters. We assume this, - * when we have a space in the program name. If we find it, everything after - * the space is treated as a single argument. - */ - if((pCmd = ustrdup(pData->iSizeLimitCmd)) == NULL) { - /* there is not much we can do - we make syslogd close the file in this case */ - return 1; - } - - for(p = pCmd ; *p && *p != ' ' ; ++p) { - /* JUST SKIP */ - } - - if(*p == ' ') { - *p = '\0'; /* pretend string-end */ - pParams = p+1; - } else - pParams = NULL; - - execProg(pCmd, 1, pParams); - - free(pCmd); - - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - - actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->iSizeLimit) { - /* OK, it didn't work out... */ - return 1; - } - - return 0; -} -#endif - - /* This function deletes an entry from the dynamic file name * cache. A pointer to the cache must be passed in as well * as the index of the to-be-deleted entry. This index may @@ -458,23 +399,29 @@ prepareFile(instanceData *pData, uchar *newFileName) } } - char szNameBuf[MAXFNAME]; - char szDirName[MAXFNAME]; - char szBaseName[MAXFNAME]; - strcpy(szNameBuf, (char*)newFileName); - strcpy(szDirName, dirname(szNameBuf)); - strcpy(szNameBuf, (char*)newFileName); - strcpy(szBaseName, basename(szNameBuf)); + /* the copies below are clumpsy, but there is no way around given the + * anomalies in dirname() and basename() [they MODIFY the provided buffer...] + */ + uchar szNameBuf[MAXFNAME]; + uchar szDirName[MAXFNAME]; + uchar szBaseName[MAXFNAME]; + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szDirName, (uchar*)dirname((char*)szNameBuf), MAXFNAME); + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szBaseName, (uchar*)basename((char*)szNameBuf), MAXFNAME); CHKiRet(strm.Construct(&pData->pStrm)); - CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); - CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); + CHKiRet(strm.SetFName(pData->pStrm, szBaseName, ustrlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit)); + if(pData->pszSizeLimitCmd != NULL) + CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); CHKiRet(strm.ConstructFinalize(pData->pStrm)); finalize_it: -- cgit From e3d9843c85b1dfddabc937ac6ccb4057d626bf03 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 11:47:00 +0200 Subject: re-enabled pipe, tty and console in omfile ... by moving code to stream.c. Thanks to the new design, new cases are not really needed, resulting in cleaner code. I also did a cleanup of header file usage as a side-activity. --- tools/omfile.c | 142 +-------------------------------------------------------- 1 file changed, 2 insertions(+), 140 deletions(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index cf020f7b..6377268d 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -55,7 +55,7 @@ # include #endif -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" @@ -64,7 +64,6 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" -#include "unicode-helper.h" #include "stream.h" #include "zlibw.h" #include "unicode-helper.h" @@ -110,12 +109,6 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - enum { - eTypeFILE, - eTypeTTY, - eTypeCONSOLE, - eTypePIPE - } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ @@ -218,14 +211,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR struct outchannel *pOch; char szBuf[128]; /* should be more than sufficient */ - /* this must always be a file, because we can not set a size limit - * on a pipe... - * rgerhards 2005-06-21: later, this will be a separate type, but let's - * emulate things for the time being. When everything runs, we can - * extend it... - */ - pData->fileType = eTypeFILE; - ++p; /* skip '$' */ i = 0; /* get outchannel name */ @@ -358,8 +343,6 @@ prepareFile(instanceData *pData, uchar *newFileName) int fd; DEFiRet; - // TODO: handle TTY/PIPE case! (in stream.c!) 2009-06-04 - if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ fd = -1; @@ -425,18 +408,6 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.ConstructFinalize(pData->pStrm)); finalize_it: - /* this was "fd != 0", which I think was a bug. I guess 0 was intended to mean - * non-open file descriptor. Anyhow, I leave this comment for the time being to that if - * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 - */ -#if 0 // TODO: this must be done by stream class! - if(fd != -1 && isatty(fd)) { - DBGPRINTF("file %d is a tty file\n", fd); - pData->fileType = eTypeTTY; - untty(); - } -#endif - RETiRet; } @@ -546,107 +517,6 @@ finalize_it: } -#if 0 -// TODO: mirgrate code below to stream class (update its write handler, which is not great!) -/* physically write the file - */ -static rsRetVal -doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) -{ - off_t actualFileSize; - int iLenWritten; - DEFiRet; - ASSERT(pData != NULL); - -dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); -again: - /* check if we have a file size limit and, if so, - * obey to it. - */ - if(pData->iSizeLimit != 0) { - actualFileSize = lseek(fd, 0, SEEK_END); - if(actualFileSize >= pData->iSizeLimit) { - char errMsg[256]; - /* for now, we simply disable a file once it is - * beyond the maximum size. This is better than having - * us aborted by the OS... rgerhards 2005-06-21 - */ - (void) close(fd); - /* try to resolve the situation */ - // TODO: *doesn't work, will need to use new fd ! - if(resolveFileSizeLimit(pData) != 0) { - /* didn't work out, so disable... */ - snprintf(errMsg, sizeof(errMsg), - "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); - ABORT_FINALIZE(RS_RET_DISABLE_ACTION); - } else { - snprintf(errMsg, sizeof(errMsg), - "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); - } - } - } - - iLenWritten = write(fd, pszBuf, lenBuf); - if(iLenWritten < 0) { - int e = errno; - char errStr[1024]; - rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); - - /* If a named pipe is full, we suspend this action for a while */ - if(pData->fileType == eTypePIPE && e == EAGAIN) - ABORT_FINALIZE(RS_RET_SUSPENDED); - - close(pData->fd); - pData->fd = -1; /* tell that fd is no longer open! */ - if(pData->bDynamicName && pData->iCurrElt != -1) { - /* in this case, we need to invalidate the name in the cache, too - * otherwise, an invalid fd may show up if we had a file name change. - * rgerhards, 2009-03-19 - */ - pData->dynCache[pData->iCurrElt]->fd = -1; - } - /* Check for EBADF on TTY's due to vhangup() - * Linux uses EIO instead (mrn 12 May 96) - */ - if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) -#ifdef linux - && e == EIO -#else - && e == EBADF -#endif - ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); - if (pData->fd < 0) { - iRet = RS_RET_SUSPENDED; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } else { - untty(); - goto again; - } - } else { - iRet = RS_RET_SUSPENDED; - errno = e; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } - } else if (pData->bSyncFile) { - fsync(fd); - } - - pData->poBuf->iBuf = 0; - -finalize_it: - RETiRet; -} -#endif - - /* do the actual write process. This function is to be called once we are ready for writing. * It will do buffered writes and persist data only when the buffer is full. Note that we must * be careful to detect when the file handle changed. @@ -662,7 +532,7 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); - FINALIZE; // TODO: clean up later + FINALIZE; } finalize_it: @@ -780,12 +650,6 @@ CODESTARTparseSelectorAct case '|': case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - if(*p == '|') { - pData->fileType = eTypePIPE; - ++p; - } else { - pData->fileType = eTypeFILE; - } CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); pData->bDynamicName = 0; @@ -808,8 +672,6 @@ CODESTARTparseSelectorAct pData->iIOBufSize = iIOBufSize; if(pData->bDynamicName == 0) { - if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) - pData->fileType = eTypeCONSOLE; /* try open and emit error message if not possible. At this stage, we ignore the * return value of prepareFile, this is taken care of in later steps. */ -- cgit From 21dafea3ee98d16a8fe93d0d5228939dc259aea7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 12:18:17 +0200 Subject: implemented $OMFileFlushOnTXEnd directive plus some cleanup... --- tools/omfile.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 6377268d..6d9eb096 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -101,6 +101,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -131,6 +132,7 @@ typedef struct _instanceData { uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ + bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -595,7 +597,12 @@ ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF("file to log to: %s\n", pData->f_fname); - iRet = writeFile(ppString, iMsgOpts, pData); + CHKiRet(writeFile(ppString, iMsgOpts, pData)); + if(pData->bFlushOnTXEnd) { + /* TODO v5: do this in endTransaction only! */ + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: ENDdoAction @@ -669,6 +676,7 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; pData->iZipLevel = iZipLevel; + pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = iIOBufSize; if(pData->bDynamicName == 0) { @@ -702,6 +710,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; + bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; if(pszTplName != NULL) { free(pszTplName); @@ -751,6 +760,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); -- cgit From 16ecb90c3ac88bb2261c31c990d88f97f1a1b32f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Jun 2009 13:44:51 +0200 Subject: omfile buffers are now synchronized after inactivity This is the first shot at this functionality. Currently, we run off a fixed counter in the rsyslogd mainloop, which needs to be restructured. But this code works, so it is a good time for a commit. --- tools/omfile.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tools/omfile.c') diff --git a/tools/omfile.c b/tools/omfile.c index 6d9eb096..675d313e 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -88,6 +88,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ +#define FLUSH_INTRVL_DFLT 1 /* default buffer flush interval (in seconds) */ /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -103,6 +104,7 @@ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathn static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ +static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -132,6 +134,7 @@ typedef struct _instanceData { uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ + int iFlushInterval; /* how fast flush buffer on inactivity? */ bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -400,6 +403,7 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); + CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); @@ -678,6 +682,7 @@ CODESTARTparseSelectorAct pData->iZipLevel = iZipLevel; pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = iIOBufSize; + pData->iFlushInterval = iFlushInterval; if(pData->bDynamicName == 0) { /* try open and emit error message if not possible. At this stage, we ignore the @@ -712,6 +717,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iZipLevel = 0; bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; + iFlushInterval = FLUSH_INTRVL_DFLT; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -760,6 +766,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); -- cgit